From 7bc0e412df9d7560c69f3048c9b5539bc25c7305 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sat, 3 Aug 2019 13:09:18 +0300 Subject: [PATCH 01/68] KAFKA-8611 / add Repartitioned configuration class --- .../kafka/streams/kstream/Repartitioned.java | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java new file mode 100644 index 0000000000000..af6636dd1975f --- /dev/null +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java @@ -0,0 +1,162 @@ +package org.apache.kafka.streams.kstream; + +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.processor.StreamPartitioner; + +/** + * This class is used to provide the optional parameters for internal repartitioned topics when using: + * - {@link KStream#repartition(Repartitioned)} + * - {@link KStream#repartition(KeyValueMapper, Repartitioned)} + * - {@link KStream#groupByKey(Repartitioned)} + * - {@link KStream#groupBy(KeyValueMapper, Repartitioned)} + * + * @param key type + * @param value type + */ +public class Repartitioned implements NamedOperation> { + + protected final String name; + + protected final Serde keySerde; + + protected final Serde valueSerde; + + protected final Integer numberOfPartitions; + + protected final StreamPartitioner partitioner; + + private Repartitioned(String name, + Serde keySerde, + Serde valueSerde, + Integer numberOfPartitions, + StreamPartitioner partitioner) { + this.name = name; + this.keySerde = keySerde; + this.valueSerde = valueSerde; + this.numberOfPartitions = numberOfPartitions; + this.partitioner = partitioner; + } + + /** + * Create a {@link Repartitioned} instance with the provided name used as part of the repartition topic if required. + * + * @param name the name used as a processor named and part of the repartition topic name if required. + * @param key type + * @param value type + * @return A new {@link Repartitioned} instance configured with processor name and repartition topic name + * @see KStream#repartition(Repartitioned) + * @see KStream#repartition(KeyValueMapper, Repartitioned) + * @see KStream#groupByKey(Repartitioned) + * @see KStream#groupBy(KeyValueMapper, Repartitioned) + */ + public static Repartitioned as(final String name) { + return new Repartitioned<>(name, null, null, null, null); + } + + /** + * Create a {@link Repartitioned} instance with provided key serde and value serde. + * + * @param keySerde Serde to use for serializing the key + * @param valueSerde Serde to use for serializing the value + * @param key type + * @param value type + * @return A new {@link Repartitioned} instance configured with key serde and value serde + * @see KStream#repartition(Repartitioned) + * @see KStream#repartition(KeyValueMapper, Repartitioned) + * @see KStream#groupByKey(Repartitioned) + * @see KStream#groupBy(KeyValueMapper, Repartitioned) + */ + public static Repartitioned with(final Serde keySerde, + final Serde valueSerde) { + return new Repartitioned<>(null, keySerde, valueSerde, null, null); + } + + /** + * Create a {@link Repartitioned} instance with provided partitioner. + * + * @param partitioner the function used to determine how records are distributed among partitions of the topic, + * if not specified and the key serde provides a {@link WindowedSerializer} for the key + * {@link WindowedStreamPartitioner} will be used—otherwise {@link DefaultPartitioner} will be used + * @param key type + * @param value type + * @return A new {@link Repartitioned} instance configured with partitioner + * @see KStream#repartition(Repartitioned) + * @see KStream#repartition(KeyValueMapper, Repartitioned) + * @see KStream#groupByKey(Repartitioned) + * @see KStream#groupBy(KeyValueMapper, Repartitioned) + */ + public static Repartitioned streamPartitioner(final StreamPartitioner partitioner) { + return new Repartitioned<>(null, null, null, null, partitioner); + } + + /** + * Create a {@link Repartitioned} instance with provided number of partitions for repartition topic if required. + * + * @param numberOfPartitions number of partitions used when creating repartition topic if required + * @param key type + * @param value type + * @return A new {@link Repartitioned} instance configured number of partitions + * @see KStream#repartition(Repartitioned) + * @see KStream#repartition(KeyValueMapper, Repartitioned) + * @see KStream#groupByKey(Repartitioned) + * @see KStream#groupBy(KeyValueMapper, Repartitioned) + */ + public static Repartitioned numberOfPartitions(final int numberOfPartitions) { + return new Repartitioned<>(null, null, null, numberOfPartitions, null); + } + + /** + * Create a new instance of {@link Repartitioned} with the provided name used as part of repartition topic and processor name. + * Note that Kafka Streams creates repartition topic only if required. + * + * @param name the name used for the processor name and as part of the repartition topic name if required + * @return a new {@link Repartitioned} instance configured with the name + */ + @Override + public Repartitioned withName(final String name) { + return new Repartitioned<>(name, keySerde, valueSerde, numberOfPartitions, partitioner); + } + + /** + * Create a new instance of {@link Repartitioned} with the provided number of partitions for repartition topic. + * Note that Kafka Streams creates repartition topic only if required. + * + * @param numberOfPartitions the name used for the processor name and as part of the repartition topic name if required + * @return a new {@link Repartitioned} instance configured with the number of partitions + */ + public Repartitioned withNumberOfPartitions(final int numberOfPartitions) { + return new Repartitioned<>(name, keySerde, valueSerde, numberOfPartitions, partitioner); + } + + /** + * Create a new instance of {@link Repartitioned} with the provided key serde. + * + * @param keySerde Serde to use for serializing the key + * @return a new {@link Repartitioned} instance configured with the key serde + */ + public Repartitioned withKeySerde(final Serde keySerde) { + return new Repartitioned<>(name, keySerde, valueSerde, numberOfPartitions, partitioner); + } + + /** + * Create a new instance of {@link Repartitioned} with the provided value serde. + * + * @param valueSerde Serde to use for serializing the value + * @return a new {@link Repartitioned} instance configured with the value serde + */ + public Repartitioned withValueSerde(final Serde valueSerde) { + return new Repartitioned<>(name, keySerde, valueSerde, numberOfPartitions, partitioner); + } + + /** + * Create a new instance of {@link Repartitioned} with the provided partitioner. + * + * @param partitioner the function used to determine how records are distributed among partitions of the topic, + * if not specified and the key serde provides a {@link WindowedSerializer} for the key + * {@link WindowedStreamPartitioner} will be used—otherwise {@link DefaultPartitioner} wil be used + * @return a new {@link Repartitioned} instance configured with provided partitioner + */ + public Repartitioned withStreamPartitioner(final StreamPartitioner partitioner) { + return new Repartitioned<>(name, keySerde, valueSerde, numberOfPartitions, partitioner); + } +} \ No newline at end of file From 597fb18e4d1f13ce9c659d9f1b11f2b0912d81e8 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sat, 3 Aug 2019 13:17:28 +0300 Subject: [PATCH 02/68] KAFKA-8611 / add NO-OP KStream#repartition operations --- .../apache/kafka/streams/kstream/KStream.java | 43 +++++++++++++++++++ .../kafka/streams/kstream/Repartitioned.java | 15 ++----- .../kstream/internals/KStreamImpl.java | 17 ++++++++ 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java index 8313addb6ff68..f2f0c4e35a15f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java @@ -837,6 +837,49 @@ KStream flatMapValues(final ValueMapperWithKey through(final String topic, final Produced produced); + /** + * Materialize this stream to a auto-generated repartition topic and creates a new {@code KStream} + * from the auto-generated topic using default serializers, deserializers, producer's {@link DefaultPartitioner}. + * Number of partitions is inherited from the source topic. + * + * @return a {@code KStream} that contains the exact same (and potentially repartitioned) records as this {@code KStream} + * @see #repartition(Repartitioned) + * @see #repartition(KeyValueMapper, Repartitioned) + */ + KStream repartition(); + + /** + * Materialize this stream to a auto-generated repartition topic and creates a new {@code KStream} + * from the auto-generated topic using {@link Serde key serde}, {@link Serde value serde}, {@link StreamPartitioner}, + * number of partitions and topic name part as defined by {@link Repartitioned}. + * + * @param repartitioned the {@link Repartitioned} instance used to specify {@link org.apache.kafka.common.serialization.Serdes}, + * {@link StreamPartitioner} which determines how records are distributed among partitions of the topic, + * part of the topic name and number of partitions for a repartition topic, if repartitioning is required. + * @return a {@code KStream} that contains the exact same (and potentially repartitioned) records as this {@code KStream} + * @see #repartition() + * @see #repartition(KeyValueMapper, Repartitioned) + */ + KStream repartition(final Repartitioned repartitioned); + + /** + * Materialize this stream to a auto-generated repartition topic and creates a new {@code KStream} + * from the auto-generated topic using {@link Serde key serde}, {@link Serde value serde}, {@link StreamPartitioner}, + * number of partitions and topic name part as defined by {@link Repartitioned}. + * The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. + * Thus, an input record {@code } can be transformed into an output record {@code }. + * + * @param mapper a {@link KeyValueMapper} that computes a new key for each record + * @param repartitioned the {@link Repartitioned} instance used to specify {@link org.apache.kafka.common.serialization.Serdes}, + * {@link StreamPartitioner} which determines how records are distributed among partitions of the topic, + * part of the topic name and number of partitions for a repartition topic, if repartitioning is required. + * @return a {@code KStream} that contains records with new key (possibly of different type) and unmodified value + * @see #repartition() + * @see #repartition(Repartitioned) + */ + KStream repartition(final KeyValueMapper mapper, + final Repartitioned repartitioned); + /** * Materialize this stream to a topic using default serializers specified in the config and producer's * {@link DefaultPartitioner}. diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java index af6636dd1975f..64331e3a06232 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java @@ -1,14 +1,15 @@ package org.apache.kafka.streams.kstream; +import org.apache.kafka.clients.producer.internals.DefaultPartitioner; import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.internals.WindowedSerializer; +import org.apache.kafka.streams.kstream.internals.WindowedStreamPartitioner; import org.apache.kafka.streams.processor.StreamPartitioner; /** * This class is used to provide the optional parameters for internal repartitioned topics when using: * - {@link KStream#repartition(Repartitioned)} * - {@link KStream#repartition(KeyValueMapper, Repartitioned)} - * - {@link KStream#groupByKey(Repartitioned)} - * - {@link KStream#groupBy(KeyValueMapper, Repartitioned)} * * @param key type * @param value type @@ -46,8 +47,6 @@ private Repartitioned(String name, * @return A new {@link Repartitioned} instance configured with processor name and repartition topic name * @see KStream#repartition(Repartitioned) * @see KStream#repartition(KeyValueMapper, Repartitioned) - * @see KStream#groupByKey(Repartitioned) - * @see KStream#groupBy(KeyValueMapper, Repartitioned) */ public static Repartitioned as(final String name) { return new Repartitioned<>(name, null, null, null, null); @@ -63,8 +62,6 @@ public static Repartitioned as(final String name) { * @return A new {@link Repartitioned} instance configured with key serde and value serde * @see KStream#repartition(Repartitioned) * @see KStream#repartition(KeyValueMapper, Repartitioned) - * @see KStream#groupByKey(Repartitioned) - * @see KStream#groupBy(KeyValueMapper, Repartitioned) */ public static Repartitioned with(final Serde keySerde, final Serde valueSerde) { @@ -82,8 +79,6 @@ public static Repartitioned with(final Serde keySerde, * @return A new {@link Repartitioned} instance configured with partitioner * @see KStream#repartition(Repartitioned) * @see KStream#repartition(KeyValueMapper, Repartitioned) - * @see KStream#groupByKey(Repartitioned) - * @see KStream#groupBy(KeyValueMapper, Repartitioned) */ public static Repartitioned streamPartitioner(final StreamPartitioner partitioner) { return new Repartitioned<>(null, null, null, null, partitioner); @@ -98,8 +93,6 @@ public static Repartitioned streamPartitioner(final StreamPartition * @return A new {@link Repartitioned} instance configured number of partitions * @see KStream#repartition(Repartitioned) * @see KStream#repartition(KeyValueMapper, Repartitioned) - * @see KStream#groupByKey(Repartitioned) - * @see KStream#groupBy(KeyValueMapper, Repartitioned) */ public static Repartitioned numberOfPartitions(final int numberOfPartitions) { return new Repartitioned<>(null, null, null, numberOfPartitions, null); @@ -159,4 +152,4 @@ public Repartitioned withValueSerde(final Serde valueSerde) { public Repartitioned withStreamPartitioner(final StreamPartitioner partitioner) { return new Repartitioned<>(name, keySerde, valueSerde, numberOfPartitions, partitioner); } -} \ No newline at end of file +} diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 9d76033e82f3d..b567fbe985668 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -31,6 +31,7 @@ import org.apache.kafka.streams.kstream.Predicate; import org.apache.kafka.streams.kstream.Printed; import org.apache.kafka.streams.kstream.Produced; +import org.apache.kafka.streams.kstream.Repartitioned; import org.apache.kafka.streams.kstream.TransformerSupplier; import org.apache.kafka.streams.kstream.ValueJoiner; import org.apache.kafka.streams.kstream.ValueMapper; @@ -483,6 +484,22 @@ public KStream through(final String topic, final Produced produced) ); } + @Override + public KStream repartition() { + return null; + } + + @Override + public KStream repartition(final Repartitioned repartitioned) { + return null; + } + + @Override + public KStream repartition(final KeyValueMapper mapper, + final Repartitioned repartitioned) { + return null; + } + @Override public void to(final String topic) { to(topic, Produced.with(keySerde, valSerde, null)); From 457d824d1715b67acd6ffbfc75bb335fb1e19b93 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sat, 3 Aug 2019 14:05:38 +0300 Subject: [PATCH 03/68] KAFKA-8611 / fix Repartitioned class checkstyle violations --- .../kafka/streams/kstream/Repartitioned.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java index 64331e3a06232..5aea2f579fd13 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.kafka.streams.kstream; import org.apache.kafka.clients.producer.internals.DefaultPartitioner; @@ -26,11 +42,11 @@ public class Repartitioned implements NamedOperation> protected final StreamPartitioner partitioner; - private Repartitioned(String name, - Serde keySerde, - Serde valueSerde, - Integer numberOfPartitions, - StreamPartitioner partitioner) { + private Repartitioned(final String name, + final Serde keySerde, + final Serde valueSerde, + final Integer numberOfPartitions, + final StreamPartitioner partitioner) { this.name = name; this.keySerde = keySerde; this.valueSerde = valueSerde; From 771e8e6890e2292b797e9259f39811b163484380 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 6 Aug 2019 00:39:35 +0300 Subject: [PATCH 04/68] KAFKA-8611 / introduce InternalTopicProperties class; enrich topic number of partitions based on InternalTopicProperties --- .../kafka/streams/kstream/Repartitioned.java | 10 +++ .../kstream/internals/KStreamImpl.java | 82 ++++++++++++++++--- .../internals/RepartitionedInternal.java | 64 +++++++++++++++ .../internals/graph/BaseRepartitionNode.java | 7 +- .../GroupedTableOperationRepartitionNode.java | 3 +- .../graph/OptimizableRepartitionNode.java | 36 +++++--- .../internals/InternalTopologyBuilder.java | 33 ++++++-- 7 files changed, 206 insertions(+), 29 deletions(-) create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java index 5aea2f579fd13..d289f3423bbc2 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java @@ -54,6 +54,16 @@ private Repartitioned(final String name, this.partitioner = partitioner; } + protected Repartitioned(final Repartitioned repartitioned) { + this( + repartitioned.name, + repartitioned.keySerde, + repartitioned.valueSerde, + repartitioned.numberOfPartitions, + repartitioned.partitioner + ); + } + /** * Create a {@link Repartitioned} instance with the provided name used as part of the repartition topic if required. * diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index b567fbe985668..ebed17ff596ac 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -38,6 +38,7 @@ import org.apache.kafka.streams.kstream.ValueMapperWithKey; import org.apache.kafka.streams.kstream.ValueTransformerSupplier; import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; +import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; import org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode; import org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode.OptimizableRepartitionNodeBuilder; import org.apache.kafka.streams.kstream.internals.graph.ProcessorGraphNode; @@ -115,6 +116,8 @@ public class KStreamImpl extends AbstractStream implements KStream through(final String topic, final Produced produced) @Override public KStream repartition() { - return null; + return repartition(null); } @Override public KStream repartition(final Repartitioned repartitioned) { - return null; + final RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); + final String name = new NamedInternal(repartitionedInternal.name()) + .orElseGenerateWithPrefix(builder, REPARTITION_NAME); + + //TODO: I don't like this. ideally it should be check against source topic + if (!repartitionRequired && repartitionedInternal.numberOfPartitions() == null) { + return new KStreamImpl<>( + name, + keySerde, + valSerde, + sourceNodes, + false, + streamsGraphNode, + builder + ); + } + + final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder = + OptimizableRepartitionNode.optimizableRepartitionNodeBuilder(); + + final String repartitionedSourceName = createRepartitionedSource( + builder, + repartitionedInternal.keySerde(), + repartitionedInternal.valueSerde(), + name, + repartitionedInternal.toInternalTopicProperties(), + optimizableRepartitionNodeBuilder + ); + + final OptimizableRepartitionNode optimizableRepartitionNode = optimizableRepartitionNodeBuilder.build(); + + builder.addGraphNode(this.streamsGraphNode, optimizableRepartitionNode); + + return new KStreamImpl<>( + repartitionedSourceName, + repartitionedInternal.keySerde(), + repartitionedInternal.valueSerde(), + sourceNodes, + repartitionRequired, + optimizableRepartitionNode, + builder + ); } @Override @@ -824,6 +868,7 @@ static String createRepartitionedSource(final InternalStreamsBuilder bu final Serde keySerde, final Serde valSerde, final String repartitionTopicNamePrefix, + final InternalTopicProperties internalTopicProperties, final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder) { @@ -840,18 +885,35 @@ static String createRepartitionedSource(final InternalStreamsBuilder bu ); optimizableRepartitionNodeBuilder.withKeySerde(keySerde) - .withValueSerde(valSerde) - .withSourceName(sourceName) - .withRepartitionTopic(repartitionTopic) - .withSinkName(sinkName) - .withProcessorParameters(processorParameters) - // reusing the source name for the graph node name - // adding explicit variable as it simplifies logic - .withNodeName(sourceName); + .withValueSerde(valSerde) + .withSourceName(sourceName) + .withRepartitionTopic(repartitionTopic) + .withSinkName(sinkName) + .withProcessorParameters(processorParameters) + .withInternalTopicProperties(internalTopicProperties) + // reusing the source name for the graph node name + // adding explicit variable as it simplifies logic + .withNodeName(sourceName); return sourceName; } + static String createRepartitionedSource(final InternalStreamsBuilder builder, + final Serde keySerde, + final Serde valSerde, + final String repartitionTopicNamePrefix, + final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder) { + + return createRepartitionedSource( + builder, + keySerde, + valSerde, + repartitionTopicNamePrefix, + null, + optimizableRepartitionNodeBuilder + ); + } + @Override public KStream leftJoin(final KStream other, final ValueJoiner joiner, diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java new file mode 100644 index 0000000000000..f0d05fe128bc9 --- /dev/null +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.kstream.internals; + +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.Repartitioned; +import org.apache.kafka.streams.processor.StreamPartitioner; + +public class RepartitionedInternal extends Repartitioned { + + RepartitionedInternal(Repartitioned repartitioned) { + super(repartitioned); + } + + InternalTopicProperties toInternalTopicProperties() { + return new InternalTopicProperties(numberOfPartitions()); + } + + String name() { + return name; + } + + Serde keySerde() { + return keySerde; + } + + Serde valueSerde() { + return valueSerde; + } + + StreamPartitioner streamPartitioner() { + return partitioner; + } + + Integer numberOfPartitions() { + return numberOfPartitions; + } + + public static class InternalTopicProperties { + private final int numberOfPartitions; + + InternalTopicProperties(int numberOfPartitions) { + this.numberOfPartitions = numberOfPartitions; + } + + public int getNumberOfPartitions() { + return numberOfPartitions; + } + } +} diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java index 460f640b5c35b..205f8219bb517 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; public abstract class BaseRepartitionNode extends StreamsGraphNode { @@ -29,7 +30,7 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode { protected final String sourceName; protected final String repartitionTopic; protected final ProcessorParameters processorParameters; - + protected final InternalTopicProperties internalTopicProperties; BaseRepartitionNode(final String nodeName, final String sourceName, @@ -37,7 +38,8 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode { final Serde keySerde, final Serde valueSerde, final String sinkName, - final String repartitionTopic) { + final String repartitionTopic, + final InternalTopicProperties internalTopicProperties) { super(nodeName); @@ -47,6 +49,7 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode { this.sourceName = sourceName; this.repartitionTopic = repartitionTopic; this.processorParameters = processorParameters; + this.internalTopicProperties = internalTopicProperties; } abstract Serializer getValueSerializer(); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java index 4d1b67dbc33ed..a3f79c50ea9ab 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java @@ -43,7 +43,8 @@ private GroupedTableOperationRepartitionNode(final String nodeName, keySerde, valueSerde, sinkName, - repartitionTopic + repartitionTopic, + null ); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index 4797a21399d5d..df6924412b71b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -21,18 +21,20 @@ import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; import org.apache.kafka.streams.processor.FailOnInvalidTimestamp; import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; public class OptimizableRepartitionNode extends BaseRepartitionNode { - OptimizableRepartitionNode(final String nodeName, - final String sourceName, - final ProcessorParameters processorParameters, - final Serde keySerde, - final Serde valueSerde, - final String sinkName, - final String repartitionTopic) { + private OptimizableRepartitionNode(final String nodeName, + final String sourceName, + final ProcessorParameters processorParameters, + final Serde keySerde, + final Serde valueSerde, + final String sinkName, + final String repartitionTopic, + final InternalTopicProperties internalTopicProperties) { super( nodeName, @@ -41,7 +43,8 @@ public class OptimizableRepartitionNode extends BaseRepartitionNode keySerde, valueSerde, sinkName, - repartitionTopic + repartitionTopic, + internalTopicProperties ); } @@ -65,7 +68,7 @@ Serializer getValueSerializer() { @Override Deserializer getValueDeserializer() { - return valueSerde != null ? valueSerde.deserializer() : null; + return valueSerde != null ? valueSerde.deserializer() : null; } @Override @@ -78,7 +81,11 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { final Serializer keySerializer = keySerde != null ? keySerde.serializer() : null; final Deserializer keyDeserializer = keySerde != null ? keySerde.deserializer() : null; - topologyBuilder.addInternalTopic(repartitionTopic); + if (internalTopicProperties != null) { + topologyBuilder.addInternalTopic(repartitionTopic, internalTopicProperties); + } else { + topologyBuilder.addInternalTopic(repartitionTopic); + } topologyBuilder.addProcessor( processorParameters.processorName(), @@ -120,6 +127,7 @@ public static final class OptimizableRepartitionNodeBuilder { private String sinkName; private String sourceName; private String repartitionTopic; + private InternalTopicProperties internalTopicProperties; private OptimizableRepartitionNodeBuilder() { } @@ -160,6 +168,11 @@ public OptimizableRepartitionNodeBuilder withNodeName(final String nodeNam return this; } + public OptimizableRepartitionNodeBuilder withInternalTopicProperties(final InternalTopicProperties internalTopicProperties) { + this.internalTopicProperties = internalTopicProperties; + return this; + } + public OptimizableRepartitionNode build() { return new OptimizableRepartitionNode<>( @@ -169,7 +182,8 @@ public OptimizableRepartitionNode build() { keySerde, valueSerde, sinkName, - repartitionTopic + repartitionTopic, + internalTopicProperties ); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index e5f404032659a..f65c2d59c4664 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -27,10 +27,11 @@ import org.apache.kafka.streams.processor.StreamPartitioner; import org.apache.kafka.streams.processor.TimestampExtractor; import org.apache.kafka.streams.processor.TopicNameExtractor; +import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; import org.apache.kafka.streams.state.StoreBuilder; import org.apache.kafka.streams.state.internals.SessionStoreBuilder; -import org.apache.kafka.streams.state.internals.WindowStoreBuilder; import org.apache.kafka.streams.state.internals.TimestampedWindowStoreBuilder; +import org.apache.kafka.streams.state.internals.WindowStoreBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,6 +76,8 @@ public class InternalTopologyBuilder { // all internal topics auto-created by the topology builder and used in source / sink processors private final Set internalTopicNames = new HashSet<>(); + private final Map internalTopicNamesWithProperties = new HashMap<>(); + // groups of source processors that need to be copartitioned private final List> copartitionSourceGroups = new ArrayList<>(); @@ -600,7 +603,7 @@ public final void connectProcessorAndStateStores(final String processorName, } public void connectSourceStoreAndTopic(final String sourceStoreName, - final String topic) { + final String topic) { if (storeToChangelogTopic.containsKey(sourceStoreName)) { throw new TopologyException("Source store " + sourceStoreName + " is already added."); } @@ -612,6 +615,12 @@ public final void addInternalTopic(final String topicName) { internalTopicNames.add(topicName); } + public final void addInternalTopic(final String topicName, + final InternalTopicProperties internalTopicProperties) { + addInternalTopic(topicName); + internalTopicNamesWithProperties.put(topicName, internalTopicProperties); + } + public final void copartitionSources(final Collection sourceNodes) { copartitionSourceGroups.add(Collections.unmodifiableSet(new HashSet<>(sourceNodes))); } @@ -1008,9 +1017,15 @@ public synchronized Map topicGroups() { if (internalTopicNames.contains(topic)) { // prefix the internal topic name with the application id final String internalTopic = decorateTopic(topic); - repartitionTopics.put( + + final RepartitionTopicConfig repartitionTopicConfig = new RepartitionTopicConfig( internalTopic, - new RepartitionTopicConfig(internalTopic, Collections.emptyMap())); + Collections.emptyMap() + ); + + maybeEnrichRepartitionTopicConfigWithProperties(topic, repartitionTopicConfig); + + repartitionTopics.put(internalTopic, repartitionTopicConfig); sourceTopics.add(internalTopic); } else { sourceTopics.add(topic); @@ -1056,6 +1071,14 @@ public synchronized Map topicGroups() { return Collections.unmodifiableMap(topicGroups); } + private void maybeEnrichRepartitionTopicConfigWithProperties(final String topic, + final RepartitionTopicConfig repartitionTopicConfig) { + if (internalTopicNamesWithProperties.containsKey(topic)) { + final InternalTopicProperties internalTopicProperties = internalTopicNamesWithProperties.get(topic); + repartitionTopicConfig.setNumberOfPartitions(internalTopicProperties.getNumberOfPartitions()); + } + } + private void setRegexMatchedTopicsToSourceNodes() { if (subscriptionUpdates.hasUpdates()) { for (final Map.Entry stringPatternEntry : nodeToSourcePatterns.entrySet()) { @@ -1474,7 +1497,7 @@ public void addPredecessor(final TopologyDescription.Node predecessor) { @Override public String toString() { final String topicsString = topics == null ? topicPattern.toString() : topics.toString(); - + return "Source: " + name + " (topics: " + topicsString + ")\n --> " + nodeNames(successors); } From 261aafdd644bde7e587a8575004b09162845a2e9 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 6 Aug 2019 01:59:08 +0300 Subject: [PATCH 05/68] KAFKA-8611 / Add implementation for KStream#repartition(Repartitioned, KeyValueMapper) --- .../kstream/internals/KStreamImpl.java | 57 ++++++- .../internals/RepartitionedInternal.java | 4 + .../KStreamRepartitionIntegrationTest.java | 161 ++++++++++++++++++ 3 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index ebed17ff596ac..2006d6663149b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -489,16 +489,19 @@ public KStream through(final String topic, final Produced produced) @Override public KStream repartition() { - return repartition(null); + return repartition(RepartitionedInternal.empty()); } @Override public KStream repartition(final Repartitioned repartitioned) { + Objects.requireNonNull(repartitioned, "repartitioned can't be null"); + final RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); + final String name = new NamedInternal(repartitionedInternal.name()) .orElseGenerateWithPrefix(builder, REPARTITION_NAME); - //TODO: I don't like this. ideally it should be check against source topic + //TODO: I don't like this. ideally number of partitions should be checked against source topic if (!repartitionRequired && repartitionedInternal.numberOfPartitions() == null) { return new KStreamImpl<>( name, @@ -514,6 +517,14 @@ public KStream repartition(final Repartitioned repartitioned) { final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder = OptimizableRepartitionNode.optimizableRepartitionNodeBuilder(); + if (repartitionedInternal.keySerde() == null) { + repartitionedInternal.withKeySerde(keySerde); + } + + if (repartitionedInternal.valueSerde() == null) { + repartitionedInternal.withValueSerde(valSerde); + } + final String repartitionedSourceName = createRepartitionedSource( builder, repartitionedInternal.keySerde(), @@ -539,9 +550,47 @@ public KStream repartition(final Repartitioned repartitioned) { } @Override - public KStream repartition(final KeyValueMapper mapper, + public KStream repartition(final KeyValueMapper selector, final Repartitioned repartitioned) { - return null; + RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); + + final NamedInternal namedInternal = new NamedInternal(repartitionedInternal.name() + "-SELECT-KEY"); + + final String name = namedInternal.orElseGenerateWithPrefix(builder, REPARTITION_NAME); + + final ProcessorGraphNode selectKeyNode = internalSelectKey(selector, namedInternal); + + builder.addGraphNode(this.streamsGraphNode, selectKeyNode); + + final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder = + OptimizableRepartitionNode.optimizableRepartitionNodeBuilder(); + + if (repartitionedInternal.valueSerde() == null) { + repartitionedInternal.withValueSerde(valSerde); + } + + final String repartitionedSourceName = createRepartitionedSource( + builder, + repartitionedInternal.keySerde(), + repartitionedInternal.valueSerde(), + name, + repartitionedInternal.toInternalTopicProperties(), + optimizableRepartitionNodeBuilder + ); + + final OptimizableRepartitionNode optimizableRepartitionNode = optimizableRepartitionNodeBuilder.build(); + + builder.addGraphNode(this.streamsGraphNode, optimizableRepartitionNode); + + return new KStreamImpl<>( + repartitionedSourceName, + repartitionedInternal.keySerde(), + repartitionedInternal.valueSerde(), + sourceNodes, + repartitionRequired, + optimizableRepartitionNode, + builder + ); } @Override diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java index f0d05fe128bc9..11bcff3983def 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java @@ -26,6 +26,10 @@ public class RepartitionedInternal extends Repartitioned { super(repartitioned); } + public static RepartitionedInternal empty() { + return new RepartitionedInternal<>(Repartitioned.as(null)); + } + InternalTopicProperties toInternalTopicProperties() { return new InternalTopicProperties(numberOfPartitions()); } diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java new file mode 100644 index 0000000000000..146866163de11 --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.integration; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.IntegerDeserializer; +import org.apache.kafka.common.serialization.IntegerSerializer; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.KeyValueTimestamp; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; +import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; +import org.apache.kafka.streams.kstream.Consumed; +import org.apache.kafka.streams.kstream.Repartitioned; +import org.apache.kafka.test.IntegrationTest; +import org.apache.kafka.test.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; + +@Category({IntegrationTest.class}) +public class KStreamRepartitionIntegrationTest { + private static final int NUM_BROKERS = 1; + + @ClassRule + public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS); + + private static volatile AtomicInteger testNo = new AtomicInteger(0); + private Properties streamsConfiguration; + private KafkaStreams kafkaStreams; + private String inputTopic = "kstream-repartition-test-input-topic"; + private String outputTopic = "kstream-repartition-test-output-topic"; + + @Before + public void before() throws InterruptedException { + CLUSTER.createTopic(inputTopic, 3, 1); + CLUSTER.createTopic(outputTopic, 1, 1); + + streamsConfiguration = new Properties(); + final String applicationId = "kstream-repartition-stream-test-" + testNo.incrementAndGet(); + streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId); + streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); + streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()); + streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0); + streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100); + streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); + streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass()); + } + + @After + public void whenShuttingDown() throws IOException { + if (kafkaStreams != null) { + kafkaStreams.close(); + } + IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration); + } + + @Test + public void shouldNotCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() { + + } + + @Test + public void shouldNotBlowUpThings() throws ExecutionException, InterruptedException { + final long timestamp = System.currentTimeMillis(); + + sendEvents( + timestamp, + Arrays.asList( + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") + ) + ); + + StreamsBuilder builder = new StreamsBuilder(); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition(Repartitioned.as("smart-repartition")) + .to(outputTopic); + + startStreams(builder); + + validateReceivedMessages( + Arrays.asList( + new KeyValueTimestamp<>(1, "A", timestamp), + new KeyValueTimestamp<>(2, "B", timestamp) + ) + ); + } + + private void sendEvents(long timestamp, + List> events) throws ExecutionException, InterruptedException { + IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp( + inputTopic, + events, + TestUtils.producerConfig( + CLUSTER.bootstrapServers(), + IntegerSerializer.class, + StringSerializer.class, + new Properties() + ), + timestamp + ); + } + + private void startStreams(StreamsBuilder builder) { + kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration); + kafkaStreams.start(); + } + + private void validateReceivedMessages(final List> expectedRecords) + throws InterruptedException { + final Properties consumerProperties = new Properties(); + consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); + consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "kstream-repartition-test-" + testNo); + consumerProperties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerProperties.setProperty( + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + IntegerDeserializer.class.getName() + ); + consumerProperties.setProperty( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class.getName() + ); + + IntegrationTestUtils.waitUntilFinalKeyValueTimestampRecordsReceived( + consumerProperties, + outputTopic, + expectedRecords + ); + } +} From ae73bee6703e5a5e2de5955a14cfbf51c5ec9568 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 6 Aug 2019 02:37:01 +0300 Subject: [PATCH 06/68] KAFKA-8611 / Added integration tests --- .../kstream/internals/KStreamImpl.java | 26 ++--- .../internals/RepartitionedInternal.java | 6 +- .../internals/InternalTopologyBuilder.java | 8 +- .../KStreamRepartitionIntegrationTest.java | 98 +++++++++++++++++-- 4 files changed, 111 insertions(+), 27 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 2006d6663149b..ae66eff70ef04 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -517,18 +517,14 @@ public KStream repartition(final Repartitioned repartitioned) { final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder = OptimizableRepartitionNode.optimizableRepartitionNodeBuilder(); - if (repartitionedInternal.keySerde() == null) { - repartitionedInternal.withKeySerde(keySerde); - } + final Serde keySerde = repartitionedInternal.keySerde() == null ? this.keySerde : repartitionedInternal.keySerde(); - if (repartitionedInternal.valueSerde() == null) { - repartitionedInternal.withValueSerde(valSerde); - } + final Serde valueSerde = repartitionedInternal.valueSerde() == null ? this.valSerde : repartitionedInternal.valueSerde(); final String repartitionedSourceName = createRepartitionedSource( builder, - repartitionedInternal.keySerde(), - repartitionedInternal.valueSerde(), + keySerde, + valueSerde, name, repartitionedInternal.toInternalTopicProperties(), optimizableRepartitionNodeBuilder @@ -540,8 +536,8 @@ public KStream repartition(final Repartitioned repartitioned) { return new KStreamImpl<>( repartitionedSourceName, - repartitionedInternal.keySerde(), - repartitionedInternal.valueSerde(), + keySerde, + valueSerde, sourceNodes, repartitionRequired, optimizableRepartitionNode, @@ -565,14 +561,12 @@ public KStream repartition(final KeyValueMapper optimizableRepartitionNodeBuilder = OptimizableRepartitionNode.optimizableRepartitionNodeBuilder(); - if (repartitionedInternal.valueSerde() == null) { - repartitionedInternal.withValueSerde(valSerde); - } + final Serde valueSerde = repartitionedInternal.valueSerde() == null ? this.valSerde : repartitionedInternal.valueSerde(); final String repartitionedSourceName = createRepartitionedSource( builder, repartitionedInternal.keySerde(), - repartitionedInternal.valueSerde(), + valueSerde, name, repartitionedInternal.toInternalTopicProperties(), optimizableRepartitionNodeBuilder @@ -580,12 +574,12 @@ public KStream repartition(final KeyValueMapper optimizableRepartitionNode = optimizableRepartitionNodeBuilder.build(); - builder.addGraphNode(this.streamsGraphNode, optimizableRepartitionNode); + builder.addGraphNode(selectKeyNode, optimizableRepartitionNode); return new KStreamImpl<>( repartitionedSourceName, repartitionedInternal.keySerde(), - repartitionedInternal.valueSerde(), + valueSerde, sourceNodes, repartitionRequired, optimizableRepartitionNode, diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java index 11bcff3983def..6d8ec4d28f21d 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java @@ -55,13 +55,13 @@ Integer numberOfPartitions() { } public static class InternalTopicProperties { - private final int numberOfPartitions; + private final Integer numberOfPartitions; - InternalTopicProperties(int numberOfPartitions) { + InternalTopicProperties(Integer numberOfPartitions) { this.numberOfPartitions = numberOfPartitions; } - public int getNumberOfPartitions() { + public Integer getNumberOfPartitions() { return numberOfPartitions; } } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index f65c2d59c4664..664c06292a2fb 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -22,12 +22,12 @@ import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.errors.TopologyException; +import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; import org.apache.kafka.streams.processor.ProcessorSupplier; import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.StreamPartitioner; import org.apache.kafka.streams.processor.TimestampExtractor; import org.apache.kafka.streams.processor.TopicNameExtractor; -import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; import org.apache.kafka.streams.state.StoreBuilder; import org.apache.kafka.streams.state.internals.SessionStoreBuilder; import org.apache.kafka.streams.state.internals.TimestampedWindowStoreBuilder; @@ -1075,7 +1075,11 @@ private void maybeEnrichRepartitionTopicConfigWithProperties(final String topic, final RepartitionTopicConfig repartitionTopicConfig) { if (internalTopicNamesWithProperties.containsKey(topic)) { final InternalTopicProperties internalTopicProperties = internalTopicNamesWithProperties.get(topic); - repartitionTopicConfig.setNumberOfPartitions(internalTopicProperties.getNumberOfPartitions()); + final Integer numberOfPartitions = internalTopicProperties.getNumberOfPartitions(); + + if (numberOfPartitions != null) { + repartitionTopicConfig.setNumberOfPartitions(numberOfPartitions); + } } } diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 146866163de11..abfb33ab00d88 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.streams.integration; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.IntegerDeserializer; import org.apache.kafka.common.serialization.IntegerSerializer; import org.apache.kafka.common.serialization.Serdes; @@ -40,15 +41,21 @@ import org.junit.experimental.categories.Category; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; @Category({IntegrationTest.class}) public class KStreamRepartitionIntegrationTest { private static final int NUM_BROKERS = 1; + private final Pattern repartitionTopicPattern = Pattern.compile("Sink: .*-repartition"); @ClassRule public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS); @@ -56,8 +63,8 @@ public class KStreamRepartitionIntegrationTest { private static volatile AtomicInteger testNo = new AtomicInteger(0); private Properties streamsConfiguration; private KafkaStreams kafkaStreams; - private String inputTopic = "kstream-repartition-test-input-topic"; - private String outputTopic = "kstream-repartition-test-output-topic"; + private String inputTopic = "input-topic"; + private String outputTopic = "output-topic"; @Before public void before() throws InterruptedException { @@ -85,8 +92,74 @@ public void whenShuttingDown() throws IOException { } @Test - public void shouldNotCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() { + public void shouldNotCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() throws ExecutionException, InterruptedException { + final long timestamp = System.currentTimeMillis(); + + sendEvents( + timestamp, + Arrays.asList( + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") + ) + ); + + StreamsBuilder builder = new StreamsBuilder(); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition(Repartitioned.as("smart-repartition")) + .to(outputTopic); + + startStreams(builder); + + validateReceivedMessages( + new IntegerDeserializer(), + new StringDeserializer(), + Arrays.asList( + new KeyValueTimestamp<>(1, "A", timestamp), + new KeyValueTimestamp<>(2, "B", timestamp) + ) + ); + + final String topology = builder.build().describe().toString(); + + assertEquals(0, getCountOfRepartitionTopicsFound(topology)); + } + + @Test + public void shouldCreateRepartitionTopicWhenRepartitionKeySelectorIsUsed() throws ExecutionException, InterruptedException { + final long timestamp = System.currentTimeMillis(); + + sendEvents( + timestamp, + Arrays.asList( + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") + ) + ); + + StreamsBuilder builder = new StreamsBuilder(); + + final Repartitioned repartitioned = Repartitioned.as("smart-repartition") + .withKeySerde(Serdes.String()); + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition((key, value) -> key.toString(), repartitioned) + .to(outputTopic); + + startStreams(builder); + + validateReceivedMessages( + new StringDeserializer(), + new StringDeserializer(), + Arrays.asList( + new KeyValueTimestamp<>("1", "A", timestamp), + new KeyValueTimestamp<>("2", "B", timestamp) + ) + ); + + final String topology = builder.build().describe().toString(); + + assertEquals(1, getCountOfRepartitionTopicsFound(topology)); } @Test @@ -110,6 +183,8 @@ public void shouldNotBlowUpThings() throws ExecutionException, InterruptedExcept startStreams(builder); validateReceivedMessages( + new IntegerDeserializer(), + new StringDeserializer(), Arrays.asList( new KeyValueTimestamp<>(1, "A", timestamp), new KeyValueTimestamp<>(2, "B", timestamp) @@ -117,6 +192,15 @@ public void shouldNotBlowUpThings() throws ExecutionException, InterruptedExcept ); } + private int getCountOfRepartitionTopicsFound(final String topologyString) { + final Matcher matcher = repartitionTopicPattern.matcher(topologyString); + final List repartitionTopicsFound = new ArrayList<>(); + while (matcher.find()) { + repartitionTopicsFound.add(matcher.group()); + } + return repartitionTopicsFound.size(); + } + private void sendEvents(long timestamp, List> events) throws ExecutionException, InterruptedException { IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp( @@ -137,7 +221,9 @@ private void startStreams(StreamsBuilder builder) { kafkaStreams.start(); } - private void validateReceivedMessages(final List> expectedRecords) + private void validateReceivedMessages(Deserializer keySerializer, + Deserializer valueSerializer, + final List> expectedRecords) throws InterruptedException { final Properties consumerProperties = new Properties(); consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); @@ -145,11 +231,11 @@ private void validateReceivedMessages(final List> consumerProperties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); consumerProperties.setProperty( ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - IntegerDeserializer.class.getName() + keySerializer.getClass().getName() ); consumerProperties.setProperty( ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - StringDeserializer.class.getName() + valueSerializer.getClass().getName() ); IntegrationTestUtils.waitUntilFinalKeyValueTimestampRecordsReceived( From 56887ee3d70927ed98d70f107f994effd0a95814 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 6 Aug 2019 02:47:41 +0300 Subject: [PATCH 07/68] KAFKA-8611 / Added OptimizableRepartitionNodeBuilder#withStreamsPartitioner --- .../kstream/internals/graph/BaseRepartitionNode.java | 4 ++++ .../graph/GroupedTableOperationRepartitionNode.java | 1 + .../internals/graph/OptimizableRepartitionNode.java | 11 ++++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java index 205f8219bb517..1e4242152b9c8 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; +import org.apache.kafka.streams.processor.StreamPartitioner; public abstract class BaseRepartitionNode extends StreamsGraphNode { @@ -29,6 +30,7 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode { protected final String sinkName; protected final String sourceName; protected final String repartitionTopic; + protected final StreamPartitioner partitioner; protected final ProcessorParameters processorParameters; protected final InternalTopicProperties internalTopicProperties; @@ -39,6 +41,7 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode { final Serde valueSerde, final String sinkName, final String repartitionTopic, + final StreamPartitioner partitioner, final InternalTopicProperties internalTopicProperties) { super(nodeName); @@ -47,6 +50,7 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode { this.valueSerde = valueSerde; this.sinkName = sinkName; this.sourceName = sourceName; + this.partitioner = partitioner; this.repartitionTopic = repartitionTopic; this.processorParameters = processorParameters; this.internalTopicProperties = internalTopicProperties; diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java index a3f79c50ea9ab..de16da61674ad 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java @@ -44,6 +44,7 @@ private GroupedTableOperationRepartitionNode(final String nodeName, valueSerde, sinkName, repartitionTopic, + null, null ); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index df6924412b71b..9b9011de0da46 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -23,6 +23,7 @@ import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; import org.apache.kafka.streams.processor.FailOnInvalidTimestamp; +import org.apache.kafka.streams.processor.StreamPartitioner; import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; public class OptimizableRepartitionNode extends BaseRepartitionNode { @@ -34,6 +35,7 @@ private OptimizableRepartitionNode(final String nodeName, final Serde valueSerde, final String sinkName, final String repartitionTopic, + final StreamPartitioner partitioner, final InternalTopicProperties internalTopicProperties) { super( @@ -44,6 +46,7 @@ private OptimizableRepartitionNode(final String nodeName, valueSerde, sinkName, repartitionTopic, + partitioner, internalTopicProperties ); @@ -98,7 +101,7 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { repartitionTopic, keySerializer, getValueSerializer(), - null, + partitioner, processorParameters.processorName() ); @@ -128,6 +131,7 @@ public static final class OptimizableRepartitionNodeBuilder { private String sourceName; private String repartitionTopic; private InternalTopicProperties internalTopicProperties; + private StreamPartitioner partitioner; private OptimizableRepartitionNodeBuilder() { } @@ -162,6 +166,10 @@ public OptimizableRepartitionNodeBuilder withRepartitionTopic(final String return this; } + public OptimizableRepartitionNodeBuilder withStreamsPartitioner(final StreamPartitioner partitioner) { + this.partitioner = partitioner; + return this; + } public OptimizableRepartitionNodeBuilder withNodeName(final String nodeName) { this.nodeName = nodeName; @@ -183,6 +191,7 @@ public OptimizableRepartitionNode build() { valueSerde, sinkName, repartitionTopic, + partitioner, internalTopicProperties ); From c3e4bd1dabae44f137c163256daee57dee888187 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 6 Aug 2019 02:59:51 +0300 Subject: [PATCH 08/68] KAFKA-8611 / update InternalStreamsBuilder#maybeOptimizeRepartitionOperations method with InteralTopicConfig and StreamPartitioner parameters --- .../internals/InternalStreamsBuilder.java | 26 ++++++++++++++++--- .../kstream/internals/KStreamImpl.java | 4 +++ .../graph/OptimizableRepartitionNode.java | 10 ++++++- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java index 3a90fd222d484..2ebf0a12a9ecc 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java @@ -24,6 +24,7 @@ import org.apache.kafka.streams.kstream.Grouped; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; import org.apache.kafka.streams.kstream.internals.graph.GlobalStoreNode; import org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode; import org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters; @@ -32,6 +33,7 @@ import org.apache.kafka.streams.kstream.internals.graph.StreamsGraphNode; import org.apache.kafka.streams.kstream.internals.graph.TableSourceNode; import org.apache.kafka.streams.processor.ProcessorSupplier; +import org.apache.kafka.streams.processor.StreamPartitioner; import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.StoreBuilder; @@ -313,7 +315,7 @@ private void optimizeKTableSourceTopics() { @SuppressWarnings("unchecked") private void maybeOptimizeRepartitionOperations() { maybeUpdateKeyChangingRepartitionNodeMap(); - final Iterator>> entryIterator = keyChangingOperationsToOptimizableRepartitionNodes.entrySet().iterator(); + final Iterator>> entryIterator = keyChangingOperationsToOptimizableRepartitionNodes.entrySet().iterator(); while (entryIterator.hasNext()) { final Map.Entry> entry = entryIterator.next(); @@ -327,10 +329,14 @@ private void maybeOptimizeRepartitionOperations() { final GroupedInternal groupedInternal = new GroupedInternal(getRepartitionSerdes(entry.getValue())); final String repartitionTopicName = getFirstRepartitionTopicName(entry.getValue()); + final StreamPartitioner partitioner = getFirstStreamPartitioner(entry.getValue()); + final InternalTopicProperties internalTopicProperties = getFirstInternalTopicProperties(entry.getValue()); //passing in the name of the first repartition topic, re-used to create the optimized repartition topic final StreamsGraphNode optimizedSingleRepartition = createRepartitionNode(repartitionTopicName, groupedInternal.keySerde(), - groupedInternal.valueSerde()); + groupedInternal.valueSerde(), + partitioner, + internalTopicProperties); // re-use parent buildPriority to make sure the single repartition graph node is evaluated before downstream nodes optimizedSingleRepartition.setBuildPriority(keyChangingNode.buildPriority()); @@ -405,13 +411,18 @@ private void maybeUpdateKeyChangingRepartitionNodeMap() { @SuppressWarnings("unchecked") private OptimizableRepartitionNode createRepartitionNode(final String repartitionTopicName, final Serde keySerde, - final Serde valueSerde) { + final Serde valueSerde, + StreamPartitioner partitioner, + InternalTopicProperties internalTopicProperties) { final OptimizableRepartitionNode.OptimizableRepartitionNodeBuilder repartitionNodeBuilder = OptimizableRepartitionNode.optimizableRepartitionNodeBuilder(); + KStreamImpl.createRepartitionedSource(this, keySerde, valueSerde, repartitionTopicName, + internalTopicProperties, + partitioner, repartitionNodeBuilder); // ensures setting the repartition topic to the name of the @@ -437,6 +448,15 @@ private String getFirstRepartitionTopicName(final Collection repartitionNodes) { + return repartitionNodes.iterator().next().internalTopicProperties(); + } + + @SuppressWarnings("unchecked") + private StreamPartitioner getFirstStreamPartitioner(final Collection repartitionNodes) { + return repartitionNodes.iterator().next().partitioner(); + } + @SuppressWarnings("unchecked") private GroupedInternal getRepartitionSerdes(final Collection repartitionNodes) { Serde keySerde = null; diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index ae66eff70ef04..a77fe2ee02441 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -50,6 +50,7 @@ import org.apache.kafka.streams.kstream.internals.graph.StreamsGraphNode; import org.apache.kafka.streams.processor.FailOnInvalidTimestamp; import org.apache.kafka.streams.processor.ProcessorSupplier; +import org.apache.kafka.streams.processor.StreamPartitioner; import org.apache.kafka.streams.processor.TopicNameExtractor; import org.apache.kafka.streams.processor.internals.StaticTopicNameExtractor; import org.apache.kafka.streams.state.StoreBuilder; @@ -912,6 +913,7 @@ static String createRepartitionedSource(final InternalStreamsBuilder bu final Serde valSerde, final String repartitionTopicNamePrefix, final InternalTopicProperties internalTopicProperties, + final StreamPartitioner partitioner, final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder) { @@ -934,6 +936,7 @@ static String createRepartitionedSource(final InternalStreamsBuilder bu .withSinkName(sinkName) .withProcessorParameters(processorParameters) .withInternalTopicProperties(internalTopicProperties) + .withStreamPartitioner(partitioner) // reusing the source name for the graph node name // adding explicit variable as it simplifies logic .withNodeName(sourceName); @@ -953,6 +956,7 @@ static String createRepartitionedSource(final InternalStreamsBuilder bu valSerde, repartitionTopicNamePrefix, null, + null, optimizableRepartitionNodeBuilder ); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index 9b9011de0da46..d448d27dcfcbf 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -64,6 +64,14 @@ public String repartitionTopic() { return repartitionTopic; } + public InternalTopicProperties internalTopicProperties() { + return internalTopicProperties; + } + + public StreamPartitioner partitioner() { + return partitioner; + } + @Override Serializer getValueSerializer() { return valueSerde != null ? valueSerde.serializer() : null; @@ -166,7 +174,7 @@ public OptimizableRepartitionNodeBuilder withRepartitionTopic(final String return this; } - public OptimizableRepartitionNodeBuilder withStreamsPartitioner(final StreamPartitioner partitioner) { + public OptimizableRepartitionNodeBuilder withStreamPartitioner(final StreamPartitioner partitioner) { this.partitioner = partitioner; return this; } From 6fac93f08307faa2bc908bf1f678f755a71d11d0 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 6 Aug 2019 03:02:44 +0300 Subject: [PATCH 09/68] KAFKA-8611 / update KStream#repartition implementations with StreamPartitioner --- .../org/apache/kafka/streams/kstream/Repartitioned.java | 8 ++++---- .../kafka/streams/kstream/internals/KStreamImpl.java | 2 ++ .../streams/kstream/internals/RepartitionedInternal.java | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java index d289f3423bbc2..b398ea28a1905 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java @@ -40,13 +40,13 @@ public class Repartitioned implements NamedOperation> protected final Integer numberOfPartitions; - protected final StreamPartitioner partitioner; + protected final StreamPartitioner partitioner; private Repartitioned(final String name, final Serde keySerde, final Serde valueSerde, final Integer numberOfPartitions, - final StreamPartitioner partitioner) { + final StreamPartitioner partitioner) { this.name = name; this.keySerde = keySerde; this.valueSerde = valueSerde; @@ -106,7 +106,7 @@ public static Repartitioned with(final Serde keySerde, * @see KStream#repartition(Repartitioned) * @see KStream#repartition(KeyValueMapper, Repartitioned) */ - public static Repartitioned streamPartitioner(final StreamPartitioner partitioner) { + public static Repartitioned streamPartitioner(final StreamPartitioner partitioner) { return new Repartitioned<>(null, null, null, null, partitioner); } @@ -175,7 +175,7 @@ public Repartitioned withValueSerde(final Serde valueSerde) { * {@link WindowedStreamPartitioner} will be used—otherwise {@link DefaultPartitioner} wil be used * @return a new {@link Repartitioned} instance configured with provided partitioner */ - public Repartitioned withStreamPartitioner(final StreamPartitioner partitioner) { + public Repartitioned withStreamPartitioner(final StreamPartitioner partitioner) { return new Repartitioned<>(name, keySerde, valueSerde, numberOfPartitions, partitioner); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index a77fe2ee02441..efc87f02c30d5 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -528,6 +528,7 @@ public KStream repartition(final Repartitioned repartitioned) { valueSerde, name, repartitionedInternal.toInternalTopicProperties(), + repartitionedInternal.streamPartitioner(), optimizableRepartitionNodeBuilder ); @@ -570,6 +571,7 @@ public KStream repartition(final KeyValueMapper valueSerde() { return valueSerde; } - StreamPartitioner streamPartitioner() { + StreamPartitioner streamPartitioner() { return partitioner; } From 0c369e1ef8dda2edfceac4671dc501e8ab7f1666 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 6 Aug 2019 03:06:49 +0300 Subject: [PATCH 10/68] KAFKA-8611 / Add require not null checks --- .../main/java/org/apache/kafka/streams/kstream/KStream.java | 4 ++-- .../apache/kafka/streams/kstream/internals/KStreamImpl.java | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java index f2f0c4e35a15f..4b774240c465f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java @@ -869,7 +869,7 @@ KStream through(final String topic, * The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. * Thus, an input record {@code } can be transformed into an output record {@code }. * - * @param mapper a {@link KeyValueMapper} that computes a new key for each record + * @param selector a {@link KeyValueMapper} that computes a new key for each record * @param repartitioned the {@link Repartitioned} instance used to specify {@link org.apache.kafka.common.serialization.Serdes}, * {@link StreamPartitioner} which determines how records are distributed among partitions of the topic, * part of the topic name and number of partitions for a repartition topic, if repartitioning is required. @@ -877,7 +877,7 @@ KStream through(final String topic, * @see #repartition() * @see #repartition(Repartitioned) */ - KStream repartition(final KeyValueMapper mapper, + KStream repartition(final KeyValueMapper selector, final Repartitioned repartitioned); /** diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index efc87f02c30d5..be97b28461f80 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -495,7 +495,7 @@ public KStream repartition() { @Override public KStream repartition(final Repartitioned repartitioned) { - Objects.requireNonNull(repartitioned, "repartitioned can't be null"); + Objects.requireNonNull(repartitioned, "repartitioned parameter can't be null"); final RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); @@ -550,6 +550,9 @@ public KStream repartition(final Repartitioned repartitioned) { @Override public KStream repartition(final KeyValueMapper selector, final Repartitioned repartitioned) { + Objects.requireNonNull(selector, "selector parameter can't be null"); + Objects.requireNonNull(repartitioned, "repartitioned parameter can't be null"); + RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); final NamedInternal namedInternal = new NamedInternal(repartitionedInternal.name() + "-SELECT-KEY"); From f641eb9f273cca88297437c3c26060700df65024 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 6 Aug 2019 18:03:14 +0300 Subject: [PATCH 11/68] KAFKA-8611 / Minor formatting fixes --- .../kafka/streams/kstream/internals/KStreamImpl.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index be97b28461f80..bafadc93387ee 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -502,7 +502,6 @@ public KStream repartition(final Repartitioned repartitioned) { final String name = new NamedInternal(repartitionedInternal.name()) .orElseGenerateWithPrefix(builder, REPARTITION_NAME); - //TODO: I don't like this. ideally number of partitions should be checked against source topic if (!repartitionRequired && repartitionedInternal.numberOfPartitions() == null) { return new KStreamImpl<>( name, @@ -555,16 +554,16 @@ public KStream repartition(final KeyValueMapper repartitionedInternal = new RepartitionedInternal<>(repartitioned); - final NamedInternal namedInternal = new NamedInternal(repartitionedInternal.name() + "-SELECT-KEY"); + final NamedInternal namedInternal = new NamedInternal(repartitionedInternal.name()); final String name = namedInternal.orElseGenerateWithPrefix(builder, REPARTITION_NAME); - final ProcessorGraphNode selectKeyNode = internalSelectKey(selector, namedInternal); + final ProcessorGraphNode selectKeyNode = internalSelectKey(selector, new NamedInternal(KEY_SELECT_NAME)); builder.addGraphNode(this.streamsGraphNode, selectKeyNode); - final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder = - OptimizableRepartitionNode.optimizableRepartitionNodeBuilder(); + final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder = OptimizableRepartitionNode + .optimizableRepartitionNodeBuilder(); final Serde valueSerde = repartitionedInternal.valueSerde() == null ? this.valSerde : repartitionedInternal.valueSerde(); From ef4fa8dbbc82d150fc3bf0a2cd83da6126db41cb Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 6 Aug 2019 20:35:02 +0300 Subject: [PATCH 12/68] KAFKA-8611 / Added more tests --- .../kstream/internals/KStreamImpl.java | 15 +-- .../KStreamRepartitionIntegrationTest.java | 92 +++++++++++++------ 2 files changed, 72 insertions(+), 35 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index bafadc93387ee..d71c38413f49a 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -502,11 +502,15 @@ public KStream repartition(final Repartitioned repartitioned) { final String name = new NamedInternal(repartitionedInternal.name()) .orElseGenerateWithPrefix(builder, REPARTITION_NAME); + final Serde keySerde = repartitionedInternal.keySerde() == null ? this.keySerde : repartitionedInternal.keySerde(); + + final Serde valueSerde = repartitionedInternal.valueSerde() == null ? this.valSerde : repartitionedInternal.valueSerde(); + if (!repartitionRequired && repartitionedInternal.numberOfPartitions() == null) { return new KStreamImpl<>( name, keySerde, - valSerde, + valueSerde, sourceNodes, false, streamsGraphNode, @@ -517,10 +521,6 @@ public KStream repartition(final Repartitioned repartitioned) { final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder = OptimizableRepartitionNode.optimizableRepartitionNodeBuilder(); - final Serde keySerde = repartitionedInternal.keySerde() == null ? this.keySerde : repartitionedInternal.keySerde(); - - final Serde valueSerde = repartitionedInternal.valueSerde() == null ? this.valSerde : repartitionedInternal.valueSerde(); - final String repartitionedSourceName = createRepartitionedSource( builder, keySerde, @@ -559,6 +559,7 @@ public KStream repartition(final KeyValueMapper selectKeyNode = internalSelectKey(selector, new NamedInternal(KEY_SELECT_NAME)); + selectKeyNode.keyChangingOperation(true); builder.addGraphNode(this.streamsGraphNode, selectKeyNode); @@ -569,7 +570,7 @@ public KStream repartition(final KeyValueMapper KStream repartition(final KeyValueMapper( repartitionedSourceName, - repartitionedInternal.keySerde(), + null, valueSerde, sourceNodes, repartitionRequired, diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index abfb33ab00d88..3f14f809098cf 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -16,16 +16,18 @@ */ package org.apache.kafka.streams.integration; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.IntegerDeserializer; import org.apache.kafka.common.serialization.IntegerSerializer; +import org.apache.kafka.common.serialization.LongDeserializer; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.KeyValueTimestamp; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; @@ -45,26 +47,29 @@ import java.util.Arrays; import java.util.List; import java.util.Properties; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; @Category({IntegrationTest.class}) public class KStreamRepartitionIntegrationTest { private static final int NUM_BROKERS = 1; - private final Pattern repartitionTopicPattern = Pattern.compile("Sink: .*-repartition"); + private static AtomicInteger testNo = new AtomicInteger(0); @ClassRule public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS); - private static volatile AtomicInteger testNo = new AtomicInteger(0); private Properties streamsConfiguration; private KafkaStreams kafkaStreams; - private String inputTopic = "input-topic"; - private String outputTopic = "output-topic"; + private String inputTopic = "input-topic-" + testNo.get(); + private String outputTopic = "output-topic-" + testNo.get(); + private String applicationId = "kstream-repartition-stream-test-" + testNo.get(); @Before public void before() throws InterruptedException { @@ -72,7 +77,7 @@ public void before() throws InterruptedException { CLUSTER.createTopic(outputTopic, 1, 1); streamsConfiguration = new Properties(); - final String applicationId = "kstream-repartition-stream-test-" + testNo.incrementAndGet(); + streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId); streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); @@ -89,10 +94,12 @@ public void whenShuttingDown() throws IOException { kafkaStreams.close(); } IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration); + testNo.incrementAndGet(); } @Test public void shouldNotCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() throws ExecutionException, InterruptedException { + final String repartitionName = "dummy"; final long timestamp = System.currentTimeMillis(); sendEvents( @@ -106,7 +113,7 @@ public void shouldNotCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition(Repartitioned.as("smart-repartition")) + .repartition(Repartitioned.as(repartitionName)) .to(outputTopic); startStreams(builder); @@ -115,18 +122,20 @@ public void shouldNotCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed new IntegerDeserializer(), new StringDeserializer(), Arrays.asList( - new KeyValueTimestamp<>(1, "A", timestamp), - new KeyValueTimestamp<>(2, "B", timestamp) + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") ) ); final String topology = builder.build().describe().toString(); - assertEquals(0, getCountOfRepartitionTopicsFound(topology)); + assertFalse(topicExists(toRepartitionTopicName(repartitionName))); + assertEquals(0, getCountOfRepartitionTopicsFound(topology, "Sink: .*-smart-repartition")); } @Test public void shouldCreateRepartitionTopicWhenRepartitionKeySelectorIsUsed() throws ExecutionException, InterruptedException { + final String repartitionName = "new-key"; final long timestamp = System.currentTimeMillis(); sendEvents( @@ -139,31 +148,32 @@ public void shouldCreateRepartitionTopicWhenRepartitionKeySelectorIsUsed() throw StreamsBuilder builder = new StreamsBuilder(); - final Repartitioned repartitioned = Repartitioned.as("smart-repartition") - .withKeySerde(Serdes.String()); - builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition((key, value) -> key.toString(), repartitioned) + .repartition((key, value) -> key.toString(), Repartitioned.as(repartitionName)) + .groupByKey() + .count() + .toStream() .to(outputTopic); startStreams(builder); validateReceivedMessages( new StringDeserializer(), - new StringDeserializer(), + new LongDeserializer(), Arrays.asList( - new KeyValueTimestamp<>("1", "A", timestamp), - new KeyValueTimestamp<>("2", "B", timestamp) + new KeyValue<>("1", 1L), + new KeyValue<>("2", 1L) ) ); final String topology = builder.build().describe().toString(); - assertEquals(1, getCountOfRepartitionTopicsFound(topology)); + assertTrue(topicExists(toRepartitionTopicName(repartitionName))); + assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*" + repartitionName + "-repartition.*")); } @Test - public void shouldNotBlowUpThings() throws ExecutionException, InterruptedException { + public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws ExecutionException, InterruptedException { final long timestamp = System.currentTimeMillis(); sendEvents( @@ -177,23 +187,49 @@ public void shouldNotBlowUpThings() throws ExecutionException, InterruptedExcept StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition(Repartitioned.as("smart-repartition")) + .selectKey((key, value) -> key.toString()) + .repartition() .to(outputTopic); startStreams(builder); validateReceivedMessages( - new IntegerDeserializer(), + new StringDeserializer(), new StringDeserializer(), Arrays.asList( - new KeyValueTimestamp<>(1, "A", timestamp), - new KeyValueTimestamp<>(2, "B", timestamp) + new KeyValue<>("1", "A"), + new KeyValue<>("2", "B") ) ); + + final String topology = builder.build().describe().toString(); + + assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*-repartition")); } - private int getCountOfRepartitionTopicsFound(final String topologyString) { - final Matcher matcher = repartitionTopicPattern.matcher(topologyString); + private boolean topicExists(String topic) throws InterruptedException, ExecutionException { + try (AdminClient adminClient = createAdminClient()) { + final Set topics = adminClient.listTopics() + .names() + .get(); + + return topics.contains(topic); + } + } + + private String toRepartitionTopicName(String input) { + return applicationId + "-" + input + "-repartition"; + } + + private AdminClient createAdminClient() { + Properties properties = new Properties(); + properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); + + return AdminClient.create(properties); + } + + private int getCountOfRepartitionTopicsFound(final String topologyString, final String searchPattern) { + final Matcher matcher = Pattern.compile(searchPattern).matcher(topologyString); final List repartitionTopicsFound = new ArrayList<>(); while (matcher.find()) { repartitionTopicsFound.add(matcher.group()); @@ -223,8 +259,8 @@ private void startStreams(StreamsBuilder builder) { private void validateReceivedMessages(Deserializer keySerializer, Deserializer valueSerializer, - final List> expectedRecords) - throws InterruptedException { + final List> expectedRecords) throws InterruptedException { + final Properties consumerProperties = new Properties(); consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "kstream-repartition-test-" + testNo); @@ -238,7 +274,7 @@ private void validateReceivedMessages(Deserializer keySerializer, valueSerializer.getClass().getName() ); - IntegrationTestUtils.waitUntilFinalKeyValueTimestampRecordsReceived( + IntegrationTestUtils.waitUntilFinalKeyValueRecordsReceived( consumerProperties, outputTopic, expectedRecords From 595736dc8f20e00aef9426bb2a5787b8158aaf74 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 6 Aug 2019 20:43:45 +0300 Subject: [PATCH 13/68] KAFKA-8611 / Add small doc about InternalTopologyBuilder#internalTopicNamesWithProperties; Moved InternalTopicProperties class to dedicated file --- .../internals/InternalStreamsBuilder.java | 2 +- .../kstream/internals/KStreamImpl.java | 2 +- .../internals/RepartitionedInternal.java | 13 +-------- .../internals/graph/BaseRepartitionNode.java | 2 +- .../graph/OptimizableRepartitionNode.java | 2 +- .../internals/InternalTopicProperties.java | 29 +++++++++++++++++++ .../internals/InternalTopologyBuilder.java | 2 +- 7 files changed, 35 insertions(+), 17 deletions(-) create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java index 2ebf0a12a9ecc..10138c7ee02b0 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java @@ -24,7 +24,6 @@ import org.apache.kafka.streams.kstream.Grouped; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KTable; -import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; import org.apache.kafka.streams.kstream.internals.graph.GlobalStoreNode; import org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode; import org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters; @@ -34,6 +33,7 @@ import org.apache.kafka.streams.kstream.internals.graph.TableSourceNode; import org.apache.kafka.streams.processor.ProcessorSupplier; import org.apache.kafka.streams.processor.StreamPartitioner; +import org.apache.kafka.streams.processor.internals.InternalTopicProperties; import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.StoreBuilder; diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index d71c38413f49a..c18e797c653d0 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -38,7 +38,6 @@ import org.apache.kafka.streams.kstream.ValueMapperWithKey; import org.apache.kafka.streams.kstream.ValueTransformerSupplier; import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; -import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; import org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode; import org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode.OptimizableRepartitionNodeBuilder; import org.apache.kafka.streams.kstream.internals.graph.ProcessorGraphNode; @@ -52,6 +51,7 @@ import org.apache.kafka.streams.processor.ProcessorSupplier; import org.apache.kafka.streams.processor.StreamPartitioner; import org.apache.kafka.streams.processor.TopicNameExtractor; +import org.apache.kafka.streams.processor.internals.InternalTopicProperties; import org.apache.kafka.streams.processor.internals.StaticTopicNameExtractor; import org.apache.kafka.streams.state.StoreBuilder; import org.apache.kafka.streams.state.Stores; diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java index 9de9ede9d7c20..e9eee2dca4462 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.streams.kstream.Repartitioned; import org.apache.kafka.streams.processor.StreamPartitioner; +import org.apache.kafka.streams.processor.internals.InternalTopicProperties; public class RepartitionedInternal extends Repartitioned { @@ -53,16 +54,4 @@ StreamPartitioner streamPartitioner() { Integer numberOfPartitions() { return numberOfPartitions; } - - public static class InternalTopicProperties { - private final Integer numberOfPartitions; - - InternalTopicProperties(Integer numberOfPartitions) { - this.numberOfPartitions = numberOfPartitions; - } - - public Integer getNumberOfPartitions() { - return numberOfPartitions; - } - } } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java index 1e4242152b9c8..967edd316aa0c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; import org.apache.kafka.streams.processor.StreamPartitioner; +import org.apache.kafka.streams.processor.internals.InternalTopicProperties; public abstract class BaseRepartitionNode extends StreamsGraphNode { diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index d448d27dcfcbf..b5a92b8842b0b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -21,9 +21,9 @@ import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; import org.apache.kafka.streams.processor.FailOnInvalidTimestamp; import org.apache.kafka.streams.processor.StreamPartitioner; +import org.apache.kafka.streams.processor.internals.InternalTopicProperties; import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; public class OptimizableRepartitionNode extends BaseRepartitionNode { diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java new file mode 100644 index 0000000000000..15128709b922b --- /dev/null +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.processor.internals; + +public class InternalTopicProperties { + private final Integer numberOfPartitions; + + public InternalTopicProperties(Integer numberOfPartitions) { + this.numberOfPartitions = numberOfPartitions; + } + + public Integer getNumberOfPartitions() { + return numberOfPartitions; + } +} diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index 664c06292a2fb..1a7e59fa205a5 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -22,7 +22,6 @@ import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.errors.TopologyException; -import org.apache.kafka.streams.kstream.internals.RepartitionedInternal.InternalTopicProperties; import org.apache.kafka.streams.processor.ProcessorSupplier; import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.StreamPartitioner; @@ -76,6 +75,7 @@ public class InternalTopologyBuilder { // all internal topics auto-created by the topology builder and used in source / sink processors private final Set internalTopicNames = new HashSet<>(); + // map of internal topic names with topic configuration private final Map internalTopicNamesWithProperties = new HashMap<>(); // groups of source processors that need to be copartitioned From a4a661a2d94bfa8832de0609e0568b315604afcc Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 6 Aug 2019 20:48:34 +0300 Subject: [PATCH 14/68] KAFKA-8611 / Minor refactoring --- .../kstream/internals/graph/OptimizableRepartitionNode.java | 6 +----- .../processor/internals/InternalTopologyBuilder.java | 5 ++++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index b5a92b8842b0b..8bd2a37e79721 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -92,11 +92,7 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { final Serializer keySerializer = keySerde != null ? keySerde.serializer() : null; final Deserializer keyDeserializer = keySerde != null ? keySerde.deserializer() : null; - if (internalTopicProperties != null) { - topologyBuilder.addInternalTopic(repartitionTopic, internalTopicProperties); - } else { - topologyBuilder.addInternalTopic(repartitionTopic); - } + topologyBuilder.addInternalTopic(repartitionTopic, internalTopicProperties); topologyBuilder.addProcessor( processorParameters.processorName(), diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index 1a7e59fa205a5..279e17bd65e51 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -618,7 +618,10 @@ public final void addInternalTopic(final String topicName) { public final void addInternalTopic(final String topicName, final InternalTopicProperties internalTopicProperties) { addInternalTopic(topicName); - internalTopicNamesWithProperties.put(topicName, internalTopicProperties); + + if (internalTopicProperties != null) { + internalTopicNamesWithProperties.put(topicName, internalTopicProperties); + } } public final void copartitionSources(final Collection sourceNodes) { From 2ee3c31c5b10c2c3554a70b768347371788b44f2 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 6 Aug 2019 21:18:21 +0300 Subject: [PATCH 15/68] KAFKA-8611 / pass repartitionRequired as false to downstream topology after repartition operation is performed --- .../kstream/internals/KStreamImpl.java | 5 +- .../KStreamRepartitionIntegrationTest.java | 78 +++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index c18e797c653d0..8ed0dc300dea2 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -540,7 +540,7 @@ public KStream repartition(final Repartitioned repartitioned) { keySerde, valueSerde, sourceNodes, - repartitionRequired, + false, optimizableRepartitionNode, builder ); @@ -587,7 +587,8 @@ public KStream repartition(final KeyValueMapper(1, "A"), + new KeyValue<>(2, "B") + ) + ); + + StreamsBuilder builder = new StreamsBuilder(); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition(Repartitioned.as(repartitionName).withNumberOfPartitions(1)) + .groupByKey() + .count() + .toStream() + .to(outputTopic); + + startStreams(builder); + + validateReceivedMessages( + new IntegerDeserializer(), + new LongDeserializer(), + Arrays.asList( + new KeyValue<>(1, 1L), + new KeyValue<>(2, 1L) + ) + ); + + final String topology = builder.build().describe().toString(); + + assertTrue(topicExists(toRepartitionTopicName(repartitionName))); + assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*" + repartitionName + "-repartition.*")); + } + + @Test + public void shouldCreateOnlyOneRepartitionTopicForKStreamGroupBy() throws ExecutionException, InterruptedException { + final String repartitionName = "new-partitions"; + final long timestamp = System.currentTimeMillis(); + + sendEvents( + timestamp, + Arrays.asList( + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") + ) + ); + + StreamsBuilder builder = new StreamsBuilder(); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .selectKey((key, value) -> key.toString()) + .repartition(Repartitioned.as(repartitionName).withNumberOfPartitions(1)) + .groupByKey() + .count() + .toStream() + .to(outputTopic); + + startStreams(builder); + final String topology = builder.build().describe().toString(); + + validateReceivedMessages( + new StringDeserializer(), + new LongDeserializer(), + Arrays.asList( + new KeyValue<>("1", 1L), + new KeyValue<>("2", 1L) + ) + ); + + assertTrue(topicExists(toRepartitionTopicName(repartitionName))); + assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*-repartition")); + } + @Test public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws ExecutionException, InterruptedException { final long timestamp = System.currentTimeMillis(); From 33050f288092049c1ec1286b7dbda518db4e92a0 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Wed, 7 Aug 2019 09:12:07 +0300 Subject: [PATCH 16/68] KAFKA-8611 / Fix checkstyle violations --- .../internals/InternalStreamsBuilder.java | 4 +-- .../kstream/internals/KStreamImpl.java | 2 +- .../internals/RepartitionedInternal.java | 2 +- .../graph/OptimizableRepartitionNode.java | 2 +- .../internals/InternalTopicProperties.java | 2 +- .../KStreamRepartitionIntegrationTest.java | 30 ++++++++++--------- 6 files changed, 22 insertions(+), 20 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java index 10138c7ee02b0..068a9e4ea722a 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java @@ -412,8 +412,8 @@ private void maybeUpdateKeyChangingRepartitionNodeMap() { private OptimizableRepartitionNode createRepartitionNode(final String repartitionTopicName, final Serde keySerde, final Serde valueSerde, - StreamPartitioner partitioner, - InternalTopicProperties internalTopicProperties) { + final StreamPartitioner partitioner, + final InternalTopicProperties internalTopicProperties) { final OptimizableRepartitionNode.OptimizableRepartitionNodeBuilder repartitionNodeBuilder = OptimizableRepartitionNode.optimizableRepartitionNodeBuilder(); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 8ed0dc300dea2..b84544fee09ce 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -552,7 +552,7 @@ public KStream repartition(final KeyValueMapper repartitionedInternal = new RepartitionedInternal<>(repartitioned); + final RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); final NamedInternal namedInternal = new NamedInternal(repartitionedInternal.name()); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java index e9eee2dca4462..b11dd66b06cc9 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java @@ -23,7 +23,7 @@ public class RepartitionedInternal extends Repartitioned { - RepartitionedInternal(Repartitioned repartitioned) { + RepartitionedInternal(final Repartitioned repartitioned) { super(repartitioned); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index 8bd2a37e79721..8b661fbd7bdf9 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -35,7 +35,7 @@ private OptimizableRepartitionNode(final String nodeName, final Serde valueSerde, final String sinkName, final String repartitionTopic, - final StreamPartitioner partitioner, + final StreamPartitioner partitioner, final InternalTopicProperties internalTopicProperties) { super( diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java index 15128709b922b..1523e5bef10b4 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java @@ -19,7 +19,7 @@ public class InternalTopicProperties { private final Integer numberOfPartitions; - public InternalTopicProperties(Integer numberOfPartitions) { + public InternalTopicProperties(final Integer numberOfPartitions) { this.numberOfPartitions = numberOfPartitions; } diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index bbc7dc30d1ee2..4777b3ffd2de5 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -110,7 +110,7 @@ public void shouldNotCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed ) ); - StreamsBuilder builder = new StreamsBuilder(); + final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) .repartition(Repartitioned.as(repartitionName)) @@ -146,7 +146,7 @@ public void shouldCreateRepartitionTopicWhenRepartitionKeySelectorIsUsed() throw ) ); - StreamsBuilder builder = new StreamsBuilder(); + final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) .repartition((key, value) -> key.toString(), Repartitioned.as(repartitionName)) @@ -185,7 +185,7 @@ public void shouldCreateRepartitionTopicWhenNumberOfPartitionsIsSpecified() thro ) ); - StreamsBuilder builder = new StreamsBuilder(); + final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) .repartition(Repartitioned.as(repartitionName).withNumberOfPartitions(1)) @@ -224,7 +224,7 @@ public void shouldCreateOnlyOneRepartitionTopicForKStreamGroupBy() throws Execut ) ); - StreamsBuilder builder = new StreamsBuilder(); + final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) .selectKey((key, value) -> key.toString()) @@ -235,6 +235,7 @@ public void shouldCreateOnlyOneRepartitionTopicForKStreamGroupBy() throws Execut .to(outputTopic); startStreams(builder); + final String topology = builder.build().describe().toString(); validateReceivedMessages( @@ -262,7 +263,7 @@ public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws Execut ) ); - StreamsBuilder builder = new StreamsBuilder(); + final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) .selectKey((key, value) -> key.toString()) @@ -285,7 +286,7 @@ public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws Execut assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*-repartition")); } - private boolean topicExists(String topic) throws InterruptedException, ExecutionException { + private boolean topicExists(final String topic) throws InterruptedException, ExecutionException { try (AdminClient adminClient = createAdminClient()) { final Set topics = adminClient.listTopics() .names() @@ -295,18 +296,19 @@ private boolean topicExists(String topic) throws InterruptedException, Execution } } - private String toRepartitionTopicName(String input) { + private String toRepartitionTopicName(final String input) { return applicationId + "-" + input + "-repartition"; } private AdminClient createAdminClient() { - Properties properties = new Properties(); + final Properties properties = new Properties(); properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); return AdminClient.create(properties); } - private int getCountOfRepartitionTopicsFound(final String topologyString, final String searchPattern) { + private int getCountOfRepartitionTopicsFound(final String topologyString, + final String searchPattern) { final Matcher matcher = Pattern.compile(searchPattern).matcher(topologyString); final List repartitionTopicsFound = new ArrayList<>(); while (matcher.find()) { @@ -315,8 +317,8 @@ private int getCountOfRepartitionTopicsFound(final String topologyString, final return repartitionTopicsFound.size(); } - private void sendEvents(long timestamp, - List> events) throws ExecutionException, InterruptedException { + private void sendEvents(final long timestamp, + final List> events) throws ExecutionException, InterruptedException { IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp( inputTopic, events, @@ -330,13 +332,13 @@ private void sendEvents(long timestamp, ); } - private void startStreams(StreamsBuilder builder) { + private void startStreams(final StreamsBuilder builder) { kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration); kafkaStreams.start(); } - private void validateReceivedMessages(Deserializer keySerializer, - Deserializer valueSerializer, + private void validateReceivedMessages(final Deserializer keySerializer, + final Deserializer valueSerializer, final List> expectedRecords) throws InterruptedException { final Properties consumerProperties = new Properties(); From 2dd9bbea8d795680737042b448cc8375dcb5281f Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Wed, 7 Aug 2019 12:48:45 +0300 Subject: [PATCH 17/68] KAFKA-8611 / Make RepartitionedInternal package private --- .../streams/kstream/internals/RepartitionedInternal.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java index b11dd66b06cc9..c00a722ba4c69 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java @@ -21,13 +21,13 @@ import org.apache.kafka.streams.processor.StreamPartitioner; import org.apache.kafka.streams.processor.internals.InternalTopicProperties; -public class RepartitionedInternal extends Repartitioned { +class RepartitionedInternal extends Repartitioned { RepartitionedInternal(final Repartitioned repartitioned) { super(repartitioned); } - public static RepartitionedInternal empty() { + static RepartitionedInternal empty() { return new RepartitionedInternal<>(Repartitioned.as(null)); } From 4f1a58b0145d888f6a56ca20d34e22535cd48fef Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 11 Aug 2019 13:10:22 +0300 Subject: [PATCH 18/68] KAFKA-8611 / CR fixes; introduce not optimizable repartition node; always repartition when calling KStream#reparititon operations --- .../apache/kafka/streams/kstream/KStream.java | 50 +++-- .../kafka/streams/kstream/Repartitioned.java | 56 +++--- .../internals/InternalStreamsBuilder.java | 22 +-- .../kstream/internals/KStreamImpl.java | 125 ++++++------ .../internals/RepartitionedInternal.java | 4 - .../internals/graph/BaseRepartitionNode.java | 10 +- .../GroupedTableOperationRepartitionNode.java | 7 +- .../graph/OptimizableRepartitionNode.java | 39 +--- .../internals/graph/RepartitionNode.java | 184 ++++++++++++++++++ .../internals/InternalTopicConfig.java | 11 +- .../internals/InternalTopologyBuilder.java | 45 ++--- .../internals/RepartitionTopicConfig.java | 6 + .../KStreamRepartitionIntegrationTest.java | 78 +++++++- .../InternalTopologyBuilderTest.java | 10 +- .../internals/ProcessorTopologyTest.java | 4 +- .../StreamsPartitionAssignorTest.java | 6 +- 16 files changed, 428 insertions(+), 229 deletions(-) create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/RepartitionNode.java diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java index 4b774240c465f..7a153ebf51b1e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java @@ -19,6 +19,7 @@ import org.apache.kafka.clients.producer.internals.DefaultPartitioner; import org.apache.kafka.common.annotation.InterfaceStability; import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; @@ -838,44 +839,51 @@ KStream through(final String topic, final Produced produced); /** - * Materialize this stream to a auto-generated repartition topic and creates a new {@code KStream} - * from the auto-generated topic using default serializers, deserializers, producer's {@link DefaultPartitioner}. - * Number of partitions is inherited from the source topic. + * Materialize this stream to a auto-generated repartition topic and create a new {@code KStream} + * from the auto-generated topic using default serializers, deserializers, and producer's {@link DefaultPartitioner}. + * The number of partitions is determined based on the upstream topics partition numbers. * - * @return a {@code KStream} that contains the exact same (and potentially repartitioned) records as this {@code KStream} - * @see #repartition(Repartitioned) - * @see #repartition(KeyValueMapper, Repartitioned) + * @return {@code KStream} that contains the exact same repartitioned records as this {@code KStream}. */ KStream repartition(); /** - * Materialize this stream to a auto-generated repartition topic and creates a new {@code KStream} + * Materialize this stream to a auto-generated repartition topic and create a new {@code KStream} * from the auto-generated topic using {@link Serde key serde}, {@link Serde value serde}, {@link StreamPartitioner}, - * number of partitions and topic name part as defined by {@link Repartitioned}. + * number of partitions, and topic name part as defined by {@link Repartitioned}. * - * @param repartitioned the {@link Repartitioned} instance used to specify {@link org.apache.kafka.common.serialization.Serdes}, + * @param repartitioned the {@link Repartitioned} instance used to specify {@link Serdes}, * {@link StreamPartitioner} which determines how records are distributed among partitions of the topic, - * part of the topic name and number of partitions for a repartition topic, if repartitioning is required. - * @return a {@code KStream} that contains the exact same (and potentially repartitioned) records as this {@code KStream} - * @see #repartition() - * @see #repartition(KeyValueMapper, Repartitioned) + * part of the topic name, and number of partitions for a repartition topic. + * @return a {@code KStream} that contains the exact same repartitioned records as this {@code KStream}. */ KStream repartition(final Repartitioned repartitioned); /** - * Materialize this stream to a auto-generated repartition topic and creates a new {@code KStream} + * Materialize this stream to a auto-generated repartition topic and create a new {@code KStream} + * from the auto-generated topic using default serializers, deserializers, and producer's {@link DefaultPartitioner}. + * The number of partitions is determined based on the upstream topics partition numbers. + * + * The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. + * Thus, an input record {@code } can be transformed into an output record {@code }. + * + * @param selector a {@link KeyValueMapper} that computes a new key for each record + * @return a {@code KStream} that contains records with new key (possibly of different type) and unmodified value + */ + KStream repartition(final KeyValueMapper selector); + + /** + * Materialize this stream to a auto-generated repartition topic and create a new {@code KStream} * from the auto-generated topic using {@link Serde key serde}, {@link Serde value serde}, {@link StreamPartitioner}, - * number of partitions and topic name part as defined by {@link Repartitioned}. + * number of partitions, and topic name part as defined by {@link Repartitioned}. * The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. * Thus, an input record {@code } can be transformed into an output record {@code }. * * @param selector a {@link KeyValueMapper} that computes a new key for each record - * @param repartitioned the {@link Repartitioned} instance used to specify {@link org.apache.kafka.common.serialization.Serdes}, + * @param repartitioned the {@link Repartitioned} instance used to specify {@link Serdes}, * {@link StreamPartitioner} which determines how records are distributed among partitions of the topic, - * part of the topic name and number of partitions for a repartition topic, if repartitioning is required. + * part of the topic name, and number of partitions for a repartition topic. * @return a {@code KStream} that contains records with new key (possibly of different type) and unmodified value - * @see #repartition() - * @see #repartition(Repartitioned) */ KStream repartition(final KeyValueMapper selector, final Repartitioned repartitioned); @@ -2154,7 +2162,7 @@ void process(final ProcessorSupplier processorSupplier, * records to it, and rereading all records from it, such that the resulting {@link KGroupedStream} is partitioned * correctly on its key. * - * @param grouped the {@link Grouped} instance used to specify {@link org.apache.kafka.common.serialization.Serdes} + * @param grouped the {@link Grouped} instance used to specify {@link Serdes} * and part of the name for a repartition topic if repartitioning is required. * @return a {@link KGroupedStream} that contains the grouped records of the original {@code KStream} * @see #groupBy(KeyValueMapper) @@ -2246,7 +2254,7 @@ KGroupedStream groupBy(final KeyValueMapper the key type of the result {@link KGroupedStream} * @return a {@link KGroupedStream} that contains the grouped records of the original {@code KStream} diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java index b398ea28a1905..a188677b080cd 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java @@ -24,8 +24,10 @@ /** * This class is used to provide the optional parameters for internal repartitioned topics when using: - * - {@link KStream#repartition(Repartitioned)} - * - {@link KStream#repartition(KeyValueMapper, Repartitioned)} + *
    + *
  • {@link KStream#repartition(Repartitioned)}
    • + *
  • {@link KStream#repartition(KeyValueMapper, Repartitioned)}
    • + *
* * @param key type * @param value type @@ -33,13 +35,9 @@ public class Repartitioned implements NamedOperation> { protected final String name; - protected final Serde keySerde; - protected final Serde valueSerde; - protected final Integer numberOfPartitions; - protected final StreamPartitioner partitioner; private Repartitioned(final String name, @@ -65,12 +63,12 @@ protected Repartitioned(final Repartitioned repartitioned) { } /** - * Create a {@link Repartitioned} instance with the provided name used as part of the repartition topic if required. + * Create a {@code Repartitioned} instance with the provided name used as part of the repartition topic. * - * @param name the name used as a processor named and part of the repartition topic name if required. + * @param name the name used as a processor named and part of the repartition topic name. * @param key type * @param value type - * @return A new {@link Repartitioned} instance configured with processor name and repartition topic name + * @return A new {@code Repartitioned} instance configured with processor name and repartition topic name * @see KStream#repartition(Repartitioned) * @see KStream#repartition(KeyValueMapper, Repartitioned) */ @@ -79,13 +77,13 @@ public static Repartitioned as(final String name) { } /** - * Create a {@link Repartitioned} instance with provided key serde and value serde. + * Create a {@code Repartitioned} instance with provided key serde and value serde. * * @param keySerde Serde to use for serializing the key * @param valueSerde Serde to use for serializing the value * @param key type * @param value type - * @return A new {@link Repartitioned} instance configured with key serde and value serde + * @return A new {@code Repartitioned} instance configured with key serde and value serde * @see KStream#repartition(Repartitioned) * @see KStream#repartition(KeyValueMapper, Repartitioned) */ @@ -95,14 +93,14 @@ public static Repartitioned with(final Serde keySerde, } /** - * Create a {@link Repartitioned} instance with provided partitioner. + * Create a {@code Repartitioned} instance with provided partitioner. * * @param partitioner the function used to determine how records are distributed among partitions of the topic, * if not specified and the key serde provides a {@link WindowedSerializer} for the key * {@link WindowedStreamPartitioner} will be used—otherwise {@link DefaultPartitioner} will be used * @param key type * @param value type - * @return A new {@link Repartitioned} instance configured with partitioner + * @return A new {@code Repartitioned} instance configured with partitioner * @see KStream#repartition(Repartitioned) * @see KStream#repartition(KeyValueMapper, Repartitioned) */ @@ -111,12 +109,12 @@ public static Repartitioned streamPartitioner(final StreamPartition } /** - * Create a {@link Repartitioned} instance with provided number of partitions for repartition topic if required. + * Create a {@code Repartitioned} instance with provided number of partitions for repartition topic. * - * @param numberOfPartitions number of partitions used when creating repartition topic if required + * @param numberOfPartitions number of partitions used when creating repartition topic * @param key type * @param value type - * @return A new {@link Repartitioned} instance configured number of partitions + * @return A new {@code Repartitioned} instance configured number of partitions * @see KStream#repartition(Repartitioned) * @see KStream#repartition(KeyValueMapper, Repartitioned) */ @@ -125,11 +123,10 @@ public static Repartitioned numberOfPartitions(final int numberOfPa } /** - * Create a new instance of {@link Repartitioned} with the provided name used as part of repartition topic and processor name. - * Note that Kafka Streams creates repartition topic only if required. + * Create a new instance of {@code Repartitioned} with the provided name used as part of repartition topic and processor name. * - * @param name the name used for the processor name and as part of the repartition topic name if required - * @return a new {@link Repartitioned} instance configured with the name + * @param name the name used for the processor name and as part of the repartition topic + * @return a new {@code Repartitioned} instance configured with the name */ @Override public Repartitioned withName(final String name) { @@ -137,43 +134,42 @@ public Repartitioned withName(final String name) { } /** - * Create a new instance of {@link Repartitioned} with the provided number of partitions for repartition topic. - * Note that Kafka Streams creates repartition topic only if required. + * Create a new instance of {@code Repartitioned} with the provided number of partitions for repartition topic. * - * @param numberOfPartitions the name used for the processor name and as part of the repartition topic name if required - * @return a new {@link Repartitioned} instance configured with the number of partitions + * @param numberOfPartitions the name used for the processor name and as part of the repartition topic name + * @return a new {@code Repartitioned} instance configured with the number of partitions */ public Repartitioned withNumberOfPartitions(final int numberOfPartitions) { return new Repartitioned<>(name, keySerde, valueSerde, numberOfPartitions, partitioner); } /** - * Create a new instance of {@link Repartitioned} with the provided key serde. + * Create a new instance of {@code Repartitioned} with the provided key serde. * * @param keySerde Serde to use for serializing the key - * @return a new {@link Repartitioned} instance configured with the key serde + * @return a new {@code Repartitioned} instance configured with the key serde */ public Repartitioned withKeySerde(final Serde keySerde) { return new Repartitioned<>(name, keySerde, valueSerde, numberOfPartitions, partitioner); } /** - * Create a new instance of {@link Repartitioned} with the provided value serde. + * Create a new instance of {@code Repartitioned} with the provided value serde. * * @param valueSerde Serde to use for serializing the value - * @return a new {@link Repartitioned} instance configured with the value serde + * @return a new {@code Repartitioned} instance configured with the value serde */ public Repartitioned withValueSerde(final Serde valueSerde) { return new Repartitioned<>(name, keySerde, valueSerde, numberOfPartitions, partitioner); } /** - * Create a new instance of {@link Repartitioned} with the provided partitioner. + * Create a new instance of {@code Repartitioned} with the provided partitioner. * * @param partitioner the function used to determine how records are distributed among partitions of the topic, * if not specified and the key serde provides a {@link WindowedSerializer} for the key * {@link WindowedStreamPartitioner} will be used—otherwise {@link DefaultPartitioner} wil be used - * @return a new {@link Repartitioned} instance configured with provided partitioner + * @return a new {@code Repartitioned} instance configured with provided partitioner */ public Repartitioned withStreamPartitioner(final StreamPartitioner partitioner) { return new Repartitioned<>(name, keySerde, valueSerde, numberOfPartitions, partitioner); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java index 068a9e4ea722a..25978d9510499 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java @@ -329,14 +329,11 @@ private void maybeOptimizeRepartitionOperations() { final GroupedInternal groupedInternal = new GroupedInternal(getRepartitionSerdes(entry.getValue())); final String repartitionTopicName = getFirstRepartitionTopicName(entry.getValue()); - final StreamPartitioner partitioner = getFirstStreamPartitioner(entry.getValue()); - final InternalTopicProperties internalTopicProperties = getFirstInternalTopicProperties(entry.getValue()); + //passing in the name of the first repartition topic, re-used to create the optimized repartition topic final StreamsGraphNode optimizedSingleRepartition = createRepartitionNode(repartitionTopicName, groupedInternal.keySerde(), - groupedInternal.valueSerde(), - partitioner, - internalTopicProperties); + groupedInternal.valueSerde()); // re-use parent buildPriority to make sure the single repartition graph node is evaluated before downstream nodes optimizedSingleRepartition.setBuildPriority(keyChangingNode.buildPriority()); @@ -411,9 +408,7 @@ private void maybeUpdateKeyChangingRepartitionNodeMap() { @SuppressWarnings("unchecked") private OptimizableRepartitionNode createRepartitionNode(final String repartitionTopicName, final Serde keySerde, - final Serde valueSerde, - final StreamPartitioner partitioner, - final InternalTopicProperties internalTopicProperties) { + final Serde valueSerde) { final OptimizableRepartitionNode.OptimizableRepartitionNodeBuilder repartitionNodeBuilder = OptimizableRepartitionNode.optimizableRepartitionNodeBuilder(); @@ -421,8 +416,6 @@ private OptimizableRepartitionNode createRepartitionNode(final String repartitio keySerde, valueSerde, repartitionTopicName, - internalTopicProperties, - partitioner, repartitionNodeBuilder); // ensures setting the repartition topic to the name of the @@ -448,15 +441,6 @@ private String getFirstRepartitionTopicName(final Collection repartitionNodes) { - return repartitionNodes.iterator().next().internalTopicProperties(); - } - - @SuppressWarnings("unchecked") - private StreamPartitioner getFirstStreamPartitioner(final Collection repartitionNodes) { - return repartitionNodes.iterator().next().partitioner(); - } - @SuppressWarnings("unchecked") private GroupedInternal getRepartitionSerdes(final Collection repartitionNodes) { Serde keySerde = null; diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index b84544fee09ce..9469889a6699d 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -42,6 +42,8 @@ import org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode.OptimizableRepartitionNodeBuilder; import org.apache.kafka.streams.kstream.internals.graph.ProcessorGraphNode; import org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters; +import org.apache.kafka.streams.kstream.internals.graph.RepartitionNode; +import org.apache.kafka.streams.kstream.internals.graph.RepartitionNode.RepartitionNodeBuilder; import org.apache.kafka.streams.kstream.internals.graph.StatefulProcessorNode; import org.apache.kafka.streams.kstream.internals.graph.StreamSinkNode; import org.apache.kafka.streams.kstream.internals.graph.StreamStreamJoinNode; @@ -490,7 +492,7 @@ public KStream through(final String topic, final Produced produced) @Override public KStream repartition() { - return repartition(RepartitionedInternal.empty()); + return repartition(Repartitioned.as(null)); } @Override @@ -506,34 +508,21 @@ public KStream repartition(final Repartitioned repartitioned) { final Serde valueSerde = repartitionedInternal.valueSerde() == null ? this.valSerde : repartitionedInternal.valueSerde(); - if (!repartitionRequired && repartitionedInternal.numberOfPartitions() == null) { - return new KStreamImpl<>( - name, - keySerde, - valueSerde, - sourceNodes, - false, - streamsGraphNode, - builder - ); - } + final RepartitionNodeBuilder repartitionNodeBuilder = RepartitionNode.repartitionNodeBuilder(); - final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder = - OptimizableRepartitionNode.optimizableRepartitionNodeBuilder(); - - final String repartitionedSourceName = createRepartitionedSource( + final String repartitionedSourceName = createNotOptimizableRepartitionedSource( builder, keySerde, valueSerde, name, repartitionedInternal.toInternalTopicProperties(), repartitionedInternal.streamPartitioner(), - optimizableRepartitionNodeBuilder + repartitionNodeBuilder ); - final OptimizableRepartitionNode optimizableRepartitionNode = optimizableRepartitionNodeBuilder.build(); + final RepartitionNode repartitionNode = repartitionNodeBuilder.build(); - builder.addGraphNode(this.streamsGraphNode, optimizableRepartitionNode); + builder.addGraphNode(this.streamsGraphNode, repartitionNode); return new KStreamImpl<>( repartitionedSourceName, @@ -541,11 +530,16 @@ public KStream repartition(final Repartitioned repartitioned) { valueSerde, sourceNodes, false, - optimizableRepartitionNode, + repartitionNode, builder ); } + @Override + public KStream repartition(KeyValueMapper selector) { + return repartition(selector, Repartitioned.as(null)); + } + @Override public KStream repartition(final KeyValueMapper selector, final Repartitioned repartitioned) { @@ -556,40 +550,39 @@ public KStream repartition(final KeyValueMapper selectKeyNode = internalSelectKey(selector, namedInternal); - final ProcessorGraphNode selectKeyNode = internalSelectKey(selector, new NamedInternal(KEY_SELECT_NAME)); selectKeyNode.keyChangingOperation(true); builder.addGraphNode(this.streamsGraphNode, selectKeyNode); - final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder = OptimizableRepartitionNode - .optimizableRepartitionNodeBuilder(); + final RepartitionNodeBuilder repartitionNodeBuilder = RepartitionNode.repartitionNodeBuilder(); + final Serde keySerde = repartitionedInternal.keySerde(); final Serde valueSerde = repartitionedInternal.valueSerde() == null ? this.valSerde : repartitionedInternal.valueSerde(); - final String repartitionedSourceName = createRepartitionedSource( + final String repartitionedSourceName = createNotOptimizableRepartitionedSource( builder, - null, + keySerde, valueSerde, - name, + namedInternal.orElseGenerateWithPrefix(builder, REPARTITION_NAME), repartitionedInternal.toInternalTopicProperties(), repartitionedInternal.streamPartitioner(), - optimizableRepartitionNodeBuilder + repartitionNodeBuilder ); - final OptimizableRepartitionNode optimizableRepartitionNode = optimizableRepartitionNodeBuilder.build(); + final RepartitionNode repartitionNode = repartitionNodeBuilder.build(); - builder.addGraphNode(selectKeyNode, optimizableRepartitionNode); + builder.addGraphNode(selectKeyNode, repartitionNode); return new KStreamImpl<>( repartitionedSourceName, - null, + keySerde, valueSerde, sourceNodes, // explicitly set repartition required as false false, - optimizableRepartitionNode, + repartitionNode, builder ); } @@ -914,15 +907,13 @@ private KStreamImpl repartitionForJoin(final String repartitionName, return new KStreamImpl<>(repartitionedSourceName, repartitionKeySerde, repartitionValueSerde, Collections.singleton(repartitionedSourceName), false, optimizableRepartitionNode, builder); } - static String createRepartitionedSource(final InternalStreamsBuilder builder, - final Serde keySerde, - final Serde valSerde, - final String repartitionTopicNamePrefix, - final InternalTopicProperties internalTopicProperties, - final StreamPartitioner partitioner, - final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder) { - - + private static String createNotOptimizableRepartitionedSource(final InternalStreamsBuilder builder, + final Serde keySerde, + final Serde valSerde, + final String repartitionTopicNamePrefix, + final InternalTopicProperties internalTopicProperties, + final StreamPartitioner partitioner, + final RepartitionNodeBuilder repartitionNodeBuilder) { final String repartitionTopic = repartitionTopicNamePrefix + REPARTITION_TOPIC_SUFFIX; final String sinkName = builder.newProcessorName(SINK_NAME); final String nullKeyFilterProcessorName = builder.newProcessorName(FILTER_NAME); @@ -935,17 +926,17 @@ static String createRepartitionedSource(final InternalStreamsBuilder bu nullKeyFilterProcessorName ); - optimizableRepartitionNodeBuilder.withKeySerde(keySerde) - .withValueSerde(valSerde) - .withSourceName(sourceName) - .withRepartitionTopic(repartitionTopic) - .withSinkName(sinkName) - .withProcessorParameters(processorParameters) - .withInternalTopicProperties(internalTopicProperties) - .withStreamPartitioner(partitioner) - // reusing the source name for the graph node name - // adding explicit variable as it simplifies logic - .withNodeName(sourceName); + repartitionNodeBuilder.withKeySerde(keySerde) + .withValueSerde(valSerde) + .withSourceName(sourceName) + .withRepartitionTopic(repartitionTopic) + .withSinkName(sinkName) + .withProcessorParameters(processorParameters) + .withInternalTopicProperties(internalTopicProperties) + .withStreamPartitioner(partitioner) + // reusing the source name for the graph node name + // adding explicit variable as it simplifies logic + .withNodeName(sourceName); return sourceName; } @@ -956,15 +947,29 @@ static String createRepartitionedSource(final InternalStreamsBuilder bu final String repartitionTopicNamePrefix, final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder) { - return createRepartitionedSource( - builder, - keySerde, - valSerde, - repartitionTopicNamePrefix, - null, - null, - optimizableRepartitionNodeBuilder + final String repartitionTopic = repartitionTopicNamePrefix + REPARTITION_TOPIC_SUFFIX; + final String sinkName = builder.newProcessorName(SINK_NAME); + final String nullKeyFilterProcessorName = builder.newProcessorName(FILTER_NAME); + final String sourceName = builder.newProcessorName(SOURCE_NAME); + + final Predicate notNullKeyPredicate = (k, v) -> k != null; + + final ProcessorParameters processorParameters = new ProcessorParameters<>( + new KStreamFilter<>(notNullKeyPredicate, false), + nullKeyFilterProcessorName ); + + optimizableRepartitionNodeBuilder.withKeySerde(keySerde) + .withValueSerde(valSerde) + .withSourceName(sourceName) + .withRepartitionTopic(repartitionTopic) + .withSinkName(sinkName) + .withProcessorParameters(processorParameters) + // reusing the source name for the graph node name + // adding explicit variable as it simplifies logic + .withNodeName(sourceName); + + return sourceName; } @Override diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java index c00a722ba4c69..b95370f19083c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/RepartitionedInternal.java @@ -27,10 +27,6 @@ class RepartitionedInternal extends Repartitioned { super(repartitioned); } - static RepartitionedInternal empty() { - return new RepartitionedInternal<>(Repartitioned.as(null)); - } - InternalTopicProperties toInternalTopicProperties() { return new InternalTopicProperties(numberOfPartitions()); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java index 967edd316aa0c..d65402d38703f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java @@ -20,8 +20,6 @@ import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.streams.processor.StreamPartitioner; -import org.apache.kafka.streams.processor.internals.InternalTopicProperties; public abstract class BaseRepartitionNode extends StreamsGraphNode { @@ -30,9 +28,7 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode { protected final String sinkName; protected final String sourceName; protected final String repartitionTopic; - protected final StreamPartitioner partitioner; protected final ProcessorParameters processorParameters; - protected final InternalTopicProperties internalTopicProperties; BaseRepartitionNode(final String nodeName, final String sourceName, @@ -40,9 +36,7 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode { final Serde keySerde, final Serde valueSerde, final String sinkName, - final String repartitionTopic, - final StreamPartitioner partitioner, - final InternalTopicProperties internalTopicProperties) { + final String repartitionTopic) { super(nodeName); @@ -50,10 +44,8 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode { this.valueSerde = valueSerde; this.sinkName = sinkName; this.sourceName = sourceName; - this.partitioner = partitioner; this.repartitionTopic = repartitionTopic; this.processorParameters = processorParameters; - this.internalTopicProperties = internalTopicProperties; } abstract Serializer getValueSerializer(); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java index de16da61674ad..034c5a33eb7dd 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java @@ -43,9 +43,7 @@ private GroupedTableOperationRepartitionNode(final String nodeName, keySerde, valueSerde, sinkName, - repartitionTopic, - null, - null + repartitionTopic ); } @@ -81,8 +79,7 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { final Serializer keySerializer = keySerde != null ? keySerde.serializer() : null; final Deserializer keyDeserializer = keySerde != null ? keySerde.deserializer() : null; - - topologyBuilder.addInternalTopic(repartitionTopic); + topologyBuilder.addInternalTopic(repartitionTopic, null); topologyBuilder.addSink( sinkName, diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index 8b661fbd7bdf9..94b9cd8bead02 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -17,13 +17,10 @@ package org.apache.kafka.streams.kstream.internals.graph; - import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.streams.processor.FailOnInvalidTimestamp; -import org.apache.kafka.streams.processor.StreamPartitioner; -import org.apache.kafka.streams.processor.internals.InternalTopicProperties; import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; public class OptimizableRepartitionNode extends BaseRepartitionNode { @@ -34,9 +31,7 @@ private OptimizableRepartitionNode(final String nodeName, final Serde keySerde, final Serde valueSerde, final String sinkName, - final String repartitionTopic, - final StreamPartitioner partitioner, - final InternalTopicProperties internalTopicProperties) { + final String repartitionTopic) { super( nodeName, @@ -45,9 +40,7 @@ private OptimizableRepartitionNode(final String nodeName, keySerde, valueSerde, sinkName, - repartitionTopic, - partitioner, - internalTopicProperties + repartitionTopic ); } @@ -64,14 +57,6 @@ public String repartitionTopic() { return repartitionTopic; } - public InternalTopicProperties internalTopicProperties() { - return internalTopicProperties; - } - - public StreamPartitioner partitioner() { - return partitioner; - } - @Override Serializer getValueSerializer() { return valueSerde != null ? valueSerde.serializer() : null; @@ -92,7 +77,7 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { final Serializer keySerializer = keySerde != null ? keySerde.serializer() : null; final Deserializer keyDeserializer = keySerde != null ? keySerde.deserializer() : null; - topologyBuilder.addInternalTopic(repartitionTopic, internalTopicProperties); + topologyBuilder.addInternalTopic(repartitionTopic, null); topologyBuilder.addProcessor( processorParameters.processorName(), @@ -105,7 +90,7 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { repartitionTopic, keySerializer, getValueSerializer(), - partitioner, + null, processorParameters.processorName() ); @@ -134,8 +119,6 @@ public static final class OptimizableRepartitionNodeBuilder { private String sinkName; private String sourceName; private String repartitionTopic; - private InternalTopicProperties internalTopicProperties; - private StreamPartitioner partitioner; private OptimizableRepartitionNodeBuilder() { } @@ -170,21 +153,11 @@ public OptimizableRepartitionNodeBuilder withRepartitionTopic(final String return this; } - public OptimizableRepartitionNodeBuilder withStreamPartitioner(final StreamPartitioner partitioner) { - this.partitioner = partitioner; - return this; - } - public OptimizableRepartitionNodeBuilder withNodeName(final String nodeName) { this.nodeName = nodeName; return this; } - public OptimizableRepartitionNodeBuilder withInternalTopicProperties(final InternalTopicProperties internalTopicProperties) { - this.internalTopicProperties = internalTopicProperties; - return this; - } - public OptimizableRepartitionNode build() { return new OptimizableRepartitionNode<>( @@ -194,9 +167,7 @@ public OptimizableRepartitionNode build() { keySerde, valueSerde, sinkName, - repartitionTopic, - partitioner, - internalTopicProperties + repartitionTopic ); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/RepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/RepartitionNode.java new file mode 100644 index 0000000000000..a78de10b3d0fd --- /dev/null +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/RepartitionNode.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.kstream.internals.graph; + +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.streams.processor.FailOnInvalidTimestamp; +import org.apache.kafka.streams.processor.StreamPartitioner; +import org.apache.kafka.streams.processor.internals.InternalTopicProperties; +import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; + +/** + * Repartition node that is not subject of optimization algorithm + */ +public class RepartitionNode extends BaseRepartitionNode { + private final StreamPartitioner partitioner; + private final InternalTopicProperties internalTopicProperties; + + private RepartitionNode(String nodeName, + String sourceName, + ProcessorParameters processorParameters, + Serde keySerde, + Serde valueSerde, + String sinkName, + String repartitionTopic, + StreamPartitioner partitioner, + InternalTopicProperties internalTopicProperties) { + super( + nodeName, + sourceName, + processorParameters, + keySerde, + valueSerde, + sinkName, + repartitionTopic + ); + + this.partitioner = partitioner; + this.internalTopicProperties = internalTopicProperties; + } + + @Override + Serializer getValueSerializer() { + return valueSerde != null ? valueSerde.serializer() : null; + } + + @Override + Deserializer getValueDeserializer() { + return valueSerde != null ? valueSerde.deserializer() : null; + } + + @Override + public void writeToTopology(InternalTopologyBuilder topologyBuilder) { + final Serializer keySerializer = getKeySerializer(); + final Deserializer keyDeserializer = getKeyDeserializer(); + + topologyBuilder.addInternalTopic(repartitionTopic, internalTopicProperties); + + topologyBuilder.addProcessor( + processorParameters.processorName(), + processorParameters.processorSupplier(), + parentNodeNames() + ); + + topologyBuilder.addSink( + sinkName, + repartitionTopic, + keySerializer, + getValueSerializer(), + partitioner, + processorParameters.processorName() + ); + + topologyBuilder.addSource( + null, + sourceName, + new FailOnInvalidTimestamp(), + keyDeserializer, + getValueDeserializer(), + repartitionTopic + ); + } + + private Serializer getKeySerializer() { + return keySerde != null ? keySerde.serializer() : null; + } + + private Deserializer getKeyDeserializer() { + return keySerde != null ? keySerde.deserializer() : null; + } + + public static RepartitionNodeBuilder repartitionNodeBuilder() { + return new RepartitionNodeBuilder<>(); + } + + public static final class RepartitionNodeBuilder { + + private String nodeName; + private ProcessorParameters processorParameters; + private Serde keySerde; + private Serde valueSerde; + private String sinkName; + private String sourceName; + private String repartitionTopic; + private InternalTopicProperties internalTopicProperties; + private StreamPartitioner partitioner; + + public RepartitionNodeBuilder withProcessorParameters(final ProcessorParameters processorParameters) { + this.processorParameters = processorParameters; + return this; + } + + public RepartitionNodeBuilder withKeySerde(final Serde keySerde) { + this.keySerde = keySerde; + return this; + } + + public RepartitionNodeBuilder withValueSerde(final Serde valueSerde) { + this.valueSerde = valueSerde; + return this; + } + + public RepartitionNodeBuilder withSinkName(final String sinkName) { + this.sinkName = sinkName; + return this; + } + + public RepartitionNodeBuilder withSourceName(final String sourceName) { + this.sourceName = sourceName; + return this; + } + + public RepartitionNodeBuilder withRepartitionTopic(final String repartitionTopic) { + this.repartitionTopic = repartitionTopic; + return this; + } + + public RepartitionNodeBuilder withStreamPartitioner(final StreamPartitioner partitioner) { + this.partitioner = partitioner; + return this; + } + + public RepartitionNodeBuilder withNodeName(final String nodeName) { + this.nodeName = nodeName; + return this; + } + + public RepartitionNodeBuilder withInternalTopicProperties(final InternalTopicProperties internalTopicProperties) { + this.internalTopicProperties = internalTopicProperties; + return this; + } + + public RepartitionNode build() { + + return new RepartitionNode<>( + nodeName, + sourceName, + processorParameters, + keySerde, + valueSerde, + sinkName, + repartitionTopic, + partitioner, + internalTopicProperties + ); + + } + } +} diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java index 9005311c7af31..e61008e3fe7ca 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java @@ -30,14 +30,21 @@ public abstract class InternalTopicConfig { final String name; final Map topicConfigs; - private int numberOfPartitions = StreamsPartitionAssignor.UNKNOWN; + private int numberOfPartitions; - InternalTopicConfig(final String name, final Map topicConfigs) { + InternalTopicConfig(final String name, + final int numberOfPartitions, + final Map topicConfigs) { Objects.requireNonNull(name, "name can't be null"); Topic.validate(name); this.name = name; this.topicConfigs = topicConfigs; + this.numberOfPartitions = numberOfPartitions; + } + + InternalTopicConfig(final String name, final Map topicConfigs) { + this(name, StreamsPartitionAssignor.UNKNOWN, topicConfigs); } /** diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index 279e17bd65e51..f97a129289133 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -610,14 +610,10 @@ public void connectSourceStoreAndTopic(final String sourceStoreName, storeToChangelogTopic.put(sourceStoreName, topic); } - public final void addInternalTopic(final String topicName) { - Objects.requireNonNull(topicName, "topicName can't be null"); - internalTopicNames.add(topicName); - } - public final void addInternalTopic(final String topicName, final InternalTopicProperties internalTopicProperties) { - addInternalTopic(topicName); + Objects.requireNonNull(topicName, "topicName can't be null"); + internalTopicNames.add(topicName); if (internalTopicProperties != null) { internalTopicNamesWithProperties.put(topicName, internalTopicProperties); @@ -1018,18 +1014,10 @@ public synchronized Map topicGroups() { continue; } if (internalTopicNames.contains(topic)) { - // prefix the internal topic name with the application id - final String internalTopic = decorateTopic(topic); - - final RepartitionTopicConfig repartitionTopicConfig = new RepartitionTopicConfig( - internalTopic, - Collections.emptyMap() - ); + final RepartitionTopicConfig repartitionTopicConfig = buildRepartitionTopicConfig(topic); - maybeEnrichRepartitionTopicConfigWithProperties(topic, repartitionTopicConfig); - - repartitionTopics.put(internalTopic, repartitionTopicConfig); - sourceTopics.add(internalTopic); + repartitionTopics.put(repartitionTopicConfig.name(), repartitionTopicConfig); + sourceTopics.add(repartitionTopicConfig.name()); } else { sourceTopics.add(topic); } @@ -1074,16 +1062,23 @@ public synchronized Map topicGroups() { return Collections.unmodifiableMap(topicGroups); } - private void maybeEnrichRepartitionTopicConfigWithProperties(final String topic, - final RepartitionTopicConfig repartitionTopicConfig) { - if (internalTopicNamesWithProperties.containsKey(topic)) { - final InternalTopicProperties internalTopicProperties = internalTopicNamesWithProperties.get(topic); - final Integer numberOfPartitions = internalTopicProperties.getNumberOfPartitions(); + private RepartitionTopicConfig buildRepartitionTopicConfig(String topic) { + // prefix the internal topic name with the application id + String internalTopic = decorateTopic(topic); - if (numberOfPartitions != null) { - repartitionTopicConfig.setNumberOfPartitions(numberOfPartitions); - } + final InternalTopicProperties internalTopicProperties = internalTopicNamesWithProperties.get(topic); + + if (internalTopicProperties == null) { + return new RepartitionTopicConfig(internalTopic, Collections.emptyMap()); } + + final Integer numberOfPartitions = internalTopicProperties.getNumberOfPartitions(); + + if (numberOfPartitions == null) { + return new RepartitionTopicConfig(internalTopic, Collections.emptyMap()); + } + + return new RepartitionTopicConfig(internalTopic, numberOfPartitions, Collections.emptyMap()); } private void setRegexMatchedTopicsToSourceNodes() { diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java index 7161a3fc88030..b71d3545a310e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java @@ -42,6 +42,12 @@ public class RepartitionTopicConfig extends InternalTopicConfig { super(name, topicConfigs); } + RepartitionTopicConfig(final String name, + final int numberOfPartitions, + final Map topicConfigs) { + super(name, numberOfPartitions, topicConfigs); + } + /** * Get the configured properties for this topic. If retentionMs is set then * we add additionalRetentionMs to work out the desired retention when cleanup.policy=compact,delete diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 4777b3ffd2de5..d7e9a30489499 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -33,6 +33,8 @@ import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; import org.apache.kafka.streams.kstream.Consumed; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.Repartitioned; import org.apache.kafka.test.IntegrationTest; import org.apache.kafka.test.TestUtils; @@ -54,7 +56,6 @@ import java.util.regex.Pattern; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @Category({IntegrationTest.class}) @@ -84,8 +85,9 @@ public void before() throws InterruptedException { streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()); streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0); streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100); - streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); - streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass()); + streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass()); + streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); + streamsConfiguration.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.OPTIMIZE); } @After @@ -98,7 +100,55 @@ public void whenShuttingDown() throws IOException { } @Test - public void shouldNotCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() throws ExecutionException, InterruptedException { + public void shouldNotOptimizeRepartitionOperationTopology() throws ExecutionException, InterruptedException { + final long timestamp = System.currentTimeMillis(); + + sendEvents( + timestamp, + Arrays.asList( + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") + ) + ); + + final StreamsBuilder builder = new StreamsBuilder(); + + final KStream stream = builder.stream(inputTopic); + + final KStream countStream1 = stream + .repartition((key, value) -> key) + .groupByKey() + .count() + .toStream(); + + final KStream countStream2 = stream + .repartition((key, value) -> key) + .groupByKey() + .count() + .toStream(); + + countStream1.merge(countStream2).to(outputTopic, Produced.with(Serdes.Integer(), Serdes.Long())); + + startStreams(builder); + + validateReceivedMessages( + new IntegerDeserializer(), + new LongDeserializer(), + Arrays.asList( + new KeyValue<>(1, 1L), + new KeyValue<>(1, 1L), + new KeyValue<>(2, 1L), + new KeyValue<>(2, 1L) + ) + ); + + final String topology = builder.build().describe().toString(); + + assertEquals(2, getCountOfRepartitionTopicsFound(topology, "Sink: .*-repartition.*")); + } + + @Test + public void shouldCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() throws ExecutionException, InterruptedException { final String repartitionName = "dummy"; final long timestamp = System.currentTimeMillis(); @@ -129,8 +179,8 @@ public void shouldNotCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed final String topology = builder.build().describe().toString(); - assertFalse(topicExists(toRepartitionTopicName(repartitionName))); - assertEquals(0, getCountOfRepartitionTopicsFound(topology, "Sink: .*-smart-repartition")); + assertTrue(topicExists(toRepartitionTopicName(repartitionName))); + assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*dummy-repartition.*")); } @Test @@ -148,8 +198,11 @@ public void shouldCreateRepartitionTopicWhenRepartitionKeySelectorIsUsed() throw final StreamsBuilder builder = new StreamsBuilder(); + final Repartitioned repartitioned = Repartitioned.as(repartitionName) + .withKeySerde(Serdes.String()); + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition((key, value) -> key.toString(), Repartitioned.as(repartitionName)) + .repartition((key, value) -> key.toString(), repartitioned) .groupByKey() .count() .toStream() @@ -226,9 +279,14 @@ public void shouldCreateOnlyOneRepartitionTopicForKStreamGroupBy() throws Execut final StreamsBuilder builder = new StreamsBuilder(); + final Repartitioned repartitioned = Repartitioned.as(repartitionName) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .withNumberOfPartitions(1); + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) .selectKey((key, value) -> key.toString()) - .repartition(Repartitioned.as(repartitionName).withNumberOfPartitions(1)) + .repartition(repartitioned) .groupByKey() .count() .toStream() @@ -267,7 +325,7 @@ public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws Execut builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) .selectKey((key, value) -> key.toString()) - .repartition() + .repartition(Repartitioned.with(Serdes.String(), Serdes.String())) .to(outputTopic); startStreams(builder); @@ -333,7 +391,7 @@ private void sendEvents(final long timestamp, } private void startStreams(final StreamsBuilder builder) { - kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration); + kafkaStreams = new KafkaStreams(builder.build(streamsConfiguration), streamsConfiguration); kafkaStreams.start(); } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java index b86211fbcd238..e80a5c681afa1 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java @@ -236,7 +236,7 @@ public void testSourceTopics() { builder.addSource(null, "source-1", null, null, null, "topic-1"); builder.addSource(null, "source-2", null, null, null, "topic-2"); builder.addSource(null, "source-3", null, null, null, "topic-3"); - builder.addInternalTopic("topic-3"); + builder.addInternalTopic("topic-3", null); final Pattern expectedPattern = Pattern.compile("X-topic-3|topic-1|topic-2"); @@ -336,7 +336,7 @@ public void testAddStateStore() { @Test public void testTopicGroups() { builder.setApplicationId("X"); - builder.addInternalTopic("topic-1x"); + builder.addInternalTopic("topic-1x", null); builder.addSource(null, "source-1", null, null, null, "topic-1", "topic-1x"); builder.addSource(null, "source-2", null, null, null, "topic-2"); builder.addSource(null, "source-3", null, null, null, "topic-3"); @@ -524,7 +524,7 @@ public void shouldNotAllowNullStateStoreNameWhenConnectingProcessorAndStateStore @Test(expected = NullPointerException.class) public void shouldNotAddNullInternalTopic() { - builder.addInternalTopic(null); + builder.addInternalTopic(null, null); } @Test(expected = NullPointerException.class) @@ -568,7 +568,7 @@ public void shouldAssociateStateStoreNameWhenStateStoreSupplierIsExternal() { @Test public void shouldCorrectlyMapStateStoreToInternalTopics() { builder.setApplicationId("appId"); - builder.addInternalTopic("internal-topic"); + builder.addInternalTopic("internal-topic", null); builder.addSource(null, "source", null, null, null, "internal-topic"); builder.addProcessor("processor", new MockProcessorSupplier(), "source"); builder.addStateStore(storeBuilder, "processor"); @@ -633,7 +633,7 @@ public void shouldAddInternalTopicConfigForNonWindowStores() { @Test public void shouldAddInternalTopicConfigForRepartitionTopics() { builder.setApplicationId("appId"); - builder.addInternalTopic("foo"); + builder.addInternalTopic("foo", null); builder.addSource(null, "source", null, null, null, "foo"); final InternalTopologyBuilder.TopicsInfo topicsInfo = builder.topicGroups().values().iterator().next(); final InternalTopicConfig topicConfig = topicsInfo.repartitionSourceTopics.get("appId-foo"); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java index 2669039ed61d7..a8150c840dfbb 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java @@ -557,7 +557,7 @@ private Topology createInternalRepartitioningTopology() { // use wrapper to get the internal topology builder to add internal topic final InternalTopologyBuilder internalTopologyBuilder = TopologyWrapper.getInternalTopologyBuilder(topology); - internalTopologyBuilder.addInternalTopic(THROUGH_TOPIC_1); + internalTopologyBuilder.addInternalTopic(THROUGH_TOPIC_1, null); return topology; } @@ -571,7 +571,7 @@ private Topology createInternalRepartitioningWithValueTimestampTopology() { // use wrapper to get the internal topology builder to add internal topic final InternalTopologyBuilder internalTopologyBuilder = TopologyWrapper.getInternalTopologyBuilder(topology); - internalTopologyBuilder.addInternalTopic(THROUGH_TOPIC_1); + internalTopologyBuilder.addInternalTopic(THROUGH_TOPIC_1, null); return topology; } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java index 616deaf39040c..cc41e65623ef2 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java @@ -691,7 +691,7 @@ public void testOnAssignment() { @Test public void testAssignWithInternalTopics() { builder.setApplicationId(applicationId); - builder.addInternalTopic("topicX"); + builder.addInternalTopic("topicX", null); builder.addSource(null, "source1", null, null, null, "topic1"); builder.addProcessor("processor1", new MockProcessorSupplier(), "source1"); builder.addSink("sink1", "topicX", null, null, null, "processor1"); @@ -721,12 +721,12 @@ public void testAssignWithInternalTopics() { public void testAssignWithInternalTopicThatsSourceIsAnotherInternalTopic() { final String applicationId = "test"; builder.setApplicationId(applicationId); - builder.addInternalTopic("topicX"); + builder.addInternalTopic("topicX", null); builder.addSource(null, "source1", null, null, null, "topic1"); builder.addProcessor("processor1", new MockProcessorSupplier(), "source1"); builder.addSink("sink1", "topicX", null, null, null, "processor1"); builder.addSource(null, "source2", null, null, null, "topicX"); - builder.addInternalTopic("topicZ"); + builder.addInternalTopic("topicZ", null); builder.addProcessor("processor2", new MockProcessorSupplier(), "source2"); builder.addSink("sink2", "topicZ", null, null, null, "processor2"); builder.addSource(null, "source3", null, null, null, "topicZ"); From e9a78187e2797b54c65858cf9858c454aecd1257 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 11 Aug 2019 13:15:15 +0300 Subject: [PATCH 19/68] KAFKA-8611 / Remove unused imports from InternalStreamsBuilder --- .../streams/kstream/internals/InternalStreamsBuilder.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java index 25978d9510499..d760ffd5707d6 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java @@ -32,8 +32,6 @@ import org.apache.kafka.streams.kstream.internals.graph.StreamsGraphNode; import org.apache.kafka.streams.kstream.internals.graph.TableSourceNode; import org.apache.kafka.streams.processor.ProcessorSupplier; -import org.apache.kafka.streams.processor.StreamPartitioner; -import org.apache.kafka.streams.processor.internals.InternalTopicProperties; import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.StoreBuilder; @@ -329,7 +327,6 @@ private void maybeOptimizeRepartitionOperations() { final GroupedInternal groupedInternal = new GroupedInternal(getRepartitionSerdes(entry.getValue())); final String repartitionTopicName = getFirstRepartitionTopicName(entry.getValue()); - //passing in the name of the first repartition topic, re-used to create the optimized repartition topic final StreamsGraphNode optimizedSingleRepartition = createRepartitionNode(repartitionTopicName, groupedInternal.keySerde(), @@ -411,7 +408,6 @@ private OptimizableRepartitionNode createRepartitionNode(final String repartitio final Serde valueSerde) { final OptimizableRepartitionNode.OptimizableRepartitionNodeBuilder repartitionNodeBuilder = OptimizableRepartitionNode.optimizableRepartitionNodeBuilder(); - KStreamImpl.createRepartitionedSource(this, keySerde, valueSerde, From ff90d5f04f913656d69af966a0b7a058a2e63b41 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 11 Aug 2019 13:34:52 +0300 Subject: [PATCH 20/68] KAFKA-8611 / Rename RepartitionNode -> UnoptimizableRepartitionNode --- .../kstream/internals/KStreamImpl.java | 68 +++++++++---------- ...java => UnoptimizableRepartitionNode.java} | 48 ++++++------- 2 files changed, 58 insertions(+), 58 deletions(-) rename streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/{RepartitionNode.java => UnoptimizableRepartitionNode.java} (69%) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 9469889a6699d..921123bfafb5a 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -42,8 +42,8 @@ import org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode.OptimizableRepartitionNodeBuilder; import org.apache.kafka.streams.kstream.internals.graph.ProcessorGraphNode; import org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters; -import org.apache.kafka.streams.kstream.internals.graph.RepartitionNode; -import org.apache.kafka.streams.kstream.internals.graph.RepartitionNode.RepartitionNodeBuilder; +import org.apache.kafka.streams.kstream.internals.graph.UnoptimizableRepartitionNode; +import org.apache.kafka.streams.kstream.internals.graph.UnoptimizableRepartitionNode.UnoptimizableRepartitionNodeBuilder; import org.apache.kafka.streams.kstream.internals.graph.StatefulProcessorNode; import org.apache.kafka.streams.kstream.internals.graph.StreamSinkNode; import org.apache.kafka.streams.kstream.internals.graph.StreamStreamJoinNode; @@ -508,29 +508,29 @@ public KStream repartition(final Repartitioned repartitioned) { final Serde valueSerde = repartitionedInternal.valueSerde() == null ? this.valSerde : repartitionedInternal.valueSerde(); - final RepartitionNodeBuilder repartitionNodeBuilder = RepartitionNode.repartitionNodeBuilder(); + final UnoptimizableRepartitionNodeBuilder unoptimizableRepartitionNodeBuilder = UnoptimizableRepartitionNode.repartitionNodeBuilder(); - final String repartitionedSourceName = createNotOptimizableRepartitionedSource( + final String repartitionSourceName = createUnoptimizableRepartitionSource( builder, keySerde, valueSerde, name, repartitionedInternal.toInternalTopicProperties(), repartitionedInternal.streamPartitioner(), - repartitionNodeBuilder + unoptimizableRepartitionNodeBuilder ); - final RepartitionNode repartitionNode = repartitionNodeBuilder.build(); + final UnoptimizableRepartitionNode unoptimizableRepartitionNode = unoptimizableRepartitionNodeBuilder.build(); - builder.addGraphNode(this.streamsGraphNode, repartitionNode); + builder.addGraphNode(this.streamsGraphNode, unoptimizableRepartitionNode); return new KStreamImpl<>( - repartitionedSourceName, + repartitionSourceName, keySerde, valueSerde, sourceNodes, false, - repartitionNode, + unoptimizableRepartitionNode, builder ); } @@ -556,33 +556,33 @@ public KStream repartition(final KeyValueMapper repartitionNodeBuilder = RepartitionNode.repartitionNodeBuilder(); + final UnoptimizableRepartitionNodeBuilder unoptimizableRepartitionNodeBuilder = UnoptimizableRepartitionNode.repartitionNodeBuilder(); final Serde keySerde = repartitionedInternal.keySerde(); final Serde valueSerde = repartitionedInternal.valueSerde() == null ? this.valSerde : repartitionedInternal.valueSerde(); - final String repartitionedSourceName = createNotOptimizableRepartitionedSource( + final String repartitionSourceName = createUnoptimizableRepartitionSource( builder, keySerde, valueSerde, namedInternal.orElseGenerateWithPrefix(builder, REPARTITION_NAME), repartitionedInternal.toInternalTopicProperties(), repartitionedInternal.streamPartitioner(), - repartitionNodeBuilder + unoptimizableRepartitionNodeBuilder ); - final RepartitionNode repartitionNode = repartitionNodeBuilder.build(); + final UnoptimizableRepartitionNode unoptimizableRepartitionNode = unoptimizableRepartitionNodeBuilder.build(); - builder.addGraphNode(selectKeyNode, repartitionNode); + builder.addGraphNode(selectKeyNode, unoptimizableRepartitionNode); return new KStreamImpl<>( - repartitionedSourceName, + repartitionSourceName, keySerde, valueSerde, sourceNodes, // explicitly set repartition required as false false, - repartitionNode, + unoptimizableRepartitionNode, builder ); } @@ -907,13 +907,13 @@ private KStreamImpl repartitionForJoin(final String repartitionName, return new KStreamImpl<>(repartitionedSourceName, repartitionKeySerde, repartitionValueSerde, Collections.singleton(repartitionedSourceName), false, optimizableRepartitionNode, builder); } - private static String createNotOptimizableRepartitionedSource(final InternalStreamsBuilder builder, - final Serde keySerde, - final Serde valSerde, - final String repartitionTopicNamePrefix, - final InternalTopicProperties internalTopicProperties, - final StreamPartitioner partitioner, - final RepartitionNodeBuilder repartitionNodeBuilder) { + private static String createUnoptimizableRepartitionSource(final InternalStreamsBuilder builder, + final Serde keySerde, + final Serde valSerde, + final String repartitionTopicNamePrefix, + final InternalTopicProperties internalTopicProperties, + final StreamPartitioner partitioner, + final UnoptimizableRepartitionNodeBuilder unoptimizableRepartitionNodeBuilder) { final String repartitionTopic = repartitionTopicNamePrefix + REPARTITION_TOPIC_SUFFIX; final String sinkName = builder.newProcessorName(SINK_NAME); final String nullKeyFilterProcessorName = builder.newProcessorName(FILTER_NAME); @@ -926,17 +926,17 @@ private static String createNotOptimizableRepartitionedSource(final Int nullKeyFilterProcessorName ); - repartitionNodeBuilder.withKeySerde(keySerde) - .withValueSerde(valSerde) - .withSourceName(sourceName) - .withRepartitionTopic(repartitionTopic) - .withSinkName(sinkName) - .withProcessorParameters(processorParameters) - .withInternalTopicProperties(internalTopicProperties) - .withStreamPartitioner(partitioner) - // reusing the source name for the graph node name - // adding explicit variable as it simplifies logic - .withNodeName(sourceName); + unoptimizableRepartitionNodeBuilder.withKeySerde(keySerde) + .withValueSerde(valSerde) + .withSourceName(sourceName) + .withRepartitionTopic(repartitionTopic) + .withSinkName(sinkName) + .withProcessorParameters(processorParameters) + .withInternalTopicProperties(internalTopicProperties) + .withStreamPartitioner(partitioner) + // reusing the source name for the graph node name + // adding explicit variable as it simplifies logic + .withNodeName(sourceName); return sourceName; } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/RepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java similarity index 69% rename from streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/RepartitionNode.java rename to streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java index a78de10b3d0fd..157c4ff235d19 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/RepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java @@ -27,19 +27,19 @@ /** * Repartition node that is not subject of optimization algorithm */ -public class RepartitionNode extends BaseRepartitionNode { +public class UnoptimizableRepartitionNode extends BaseRepartitionNode { private final StreamPartitioner partitioner; private final InternalTopicProperties internalTopicProperties; - private RepartitionNode(String nodeName, - String sourceName, - ProcessorParameters processorParameters, - Serde keySerde, - Serde valueSerde, - String sinkName, - String repartitionTopic, - StreamPartitioner partitioner, - InternalTopicProperties internalTopicProperties) { + private UnoptimizableRepartitionNode(String nodeName, + String sourceName, + ProcessorParameters processorParameters, + Serde keySerde, + Serde valueSerde, + String sinkName, + String repartitionTopic, + StreamPartitioner partitioner, + InternalTopicProperties internalTopicProperties) { super( nodeName, sourceName, @@ -104,11 +104,11 @@ private Deserializer getKeyDeserializer() { return keySerde != null ? keySerde.deserializer() : null; } - public static RepartitionNodeBuilder repartitionNodeBuilder() { - return new RepartitionNodeBuilder<>(); + public static UnoptimizableRepartitionNodeBuilder repartitionNodeBuilder() { + return new UnoptimizableRepartitionNodeBuilder<>(); } - public static final class RepartitionNodeBuilder { + public static final class UnoptimizableRepartitionNodeBuilder { private String nodeName; private ProcessorParameters processorParameters; @@ -120,54 +120,54 @@ public static final class RepartitionNodeBuilder { private InternalTopicProperties internalTopicProperties; private StreamPartitioner partitioner; - public RepartitionNodeBuilder withProcessorParameters(final ProcessorParameters processorParameters) { + public UnoptimizableRepartitionNodeBuilder withProcessorParameters(final ProcessorParameters processorParameters) { this.processorParameters = processorParameters; return this; } - public RepartitionNodeBuilder withKeySerde(final Serde keySerde) { + public UnoptimizableRepartitionNodeBuilder withKeySerde(final Serde keySerde) { this.keySerde = keySerde; return this; } - public RepartitionNodeBuilder withValueSerde(final Serde valueSerde) { + public UnoptimizableRepartitionNodeBuilder withValueSerde(final Serde valueSerde) { this.valueSerde = valueSerde; return this; } - public RepartitionNodeBuilder withSinkName(final String sinkName) { + public UnoptimizableRepartitionNodeBuilder withSinkName(final String sinkName) { this.sinkName = sinkName; return this; } - public RepartitionNodeBuilder withSourceName(final String sourceName) { + public UnoptimizableRepartitionNodeBuilder withSourceName(final String sourceName) { this.sourceName = sourceName; return this; } - public RepartitionNodeBuilder withRepartitionTopic(final String repartitionTopic) { + public UnoptimizableRepartitionNodeBuilder withRepartitionTopic(final String repartitionTopic) { this.repartitionTopic = repartitionTopic; return this; } - public RepartitionNodeBuilder withStreamPartitioner(final StreamPartitioner partitioner) { + public UnoptimizableRepartitionNodeBuilder withStreamPartitioner(final StreamPartitioner partitioner) { this.partitioner = partitioner; return this; } - public RepartitionNodeBuilder withNodeName(final String nodeName) { + public UnoptimizableRepartitionNodeBuilder withNodeName(final String nodeName) { this.nodeName = nodeName; return this; } - public RepartitionNodeBuilder withInternalTopicProperties(final InternalTopicProperties internalTopicProperties) { + public UnoptimizableRepartitionNodeBuilder withInternalTopicProperties(final InternalTopicProperties internalTopicProperties) { this.internalTopicProperties = internalTopicProperties; return this; } - public RepartitionNode build() { + public UnoptimizableRepartitionNode build() { - return new RepartitionNode<>( + return new UnoptimizableRepartitionNode<>( nodeName, sourceName, processorParameters, From ebd77b55a763d9ad1e697ca00348e5767b565b16 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 11 Aug 2019 13:57:14 +0300 Subject: [PATCH 21/68] KAFKA-8611 / Added test case for verifying partition number of repartition topic --- .../KStreamRepartitionIntegrationTest.java | 63 +++++++++++++++++-- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index d7e9a30489499..355ea8e506508 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -18,6 +18,7 @@ import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.IntegerDeserializer; @@ -47,6 +48,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Properties; import java.util.Set; @@ -220,8 +222,9 @@ public void shouldCreateRepartitionTopicWhenRepartitionKeySelectorIsUsed() throw ); final String topology = builder.build().describe().toString(); + final String repartitionTopicName = toRepartitionTopicName(repartitionName); - assertTrue(topicExists(toRepartitionTopicName(repartitionName))); + assertTrue(topicExists(repartitionTopicName)); assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*" + repartitionName + "-repartition.*")); } @@ -241,7 +244,7 @@ public void shouldCreateRepartitionTopicWhenNumberOfPartitionsIsSpecified() thro final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition(Repartitioned.as(repartitionName).withNumberOfPartitions(1)) + .repartition(Repartitioned.as(repartitionName).withNumberOfPartitions(2)) .groupByKey() .count() .toStream() @@ -258,10 +261,49 @@ public void shouldCreateRepartitionTopicWhenNumberOfPartitionsIsSpecified() thro ) ); - final String topology = builder.build().describe().toString(); + final String repartitionTopicName = toRepartitionTopicName(repartitionName); - assertTrue(topicExists(toRepartitionTopicName(repartitionName))); - assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*" + repartitionName + "-repartition.*")); + assertTrue(topicExists(repartitionTopicName)); + assertEquals(2, getNumberOfPartitionsForTopic(repartitionTopicName)); + } + + @Test + public void shouldInheritRepartitionTopicPartitionNumberFromUpstreamTopicWhenNumberOfPartitionsIsNotSpecified() throws InterruptedException, ExecutionException { + final String repartitionName = "new-topic"; + final long timestamp = System.currentTimeMillis(); + + sendEvents( + timestamp, + Arrays.asList( + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") + ) + ); + + final StreamsBuilder builder = new StreamsBuilder(); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition(Repartitioned.as(repartitionName)) + .groupByKey() + .count() + .toStream() + .to(outputTopic); + + startStreams(builder); + + validateReceivedMessages( + new IntegerDeserializer(), + new LongDeserializer(), + Arrays.asList( + new KeyValue<>(1, 1L), + new KeyValue<>(2, 1L) + ) + ); + + final String repartitionTopicName = toRepartitionTopicName(repartitionName); + + assertTrue(topicExists(repartitionTopicName)); + assertEquals(3, getNumberOfPartitionsForTopic(repartitionTopicName)); } @Test @@ -344,6 +386,17 @@ public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws Execut assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*-repartition")); } + private int getNumberOfPartitionsForTopic(final String topic) throws ExecutionException, InterruptedException { + try (AdminClient adminClient = createAdminClient()) { + final TopicDescription topicDescription = adminClient.describeTopics(Collections.singleton(topic)) + .values() + .get(topic) + .get(); + + return topicDescription.partitions().size(); + } + } + private boolean topicExists(final String topic) throws InterruptedException, ExecutionException { try (AdminClient adminClient = createAdminClient()) { final Set topics = adminClient.listTopics() From 317bd94a6532236d283b3ddf9d55bf36e96044cc Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 11 Aug 2019 14:17:48 +0300 Subject: [PATCH 22/68] KAFKA-8611 / Small refactoring of integration test --- .../KStreamRepartitionIntegrationTest.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 355ea8e506508..bd238db5fa480 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -146,7 +146,7 @@ public void shouldNotOptimizeRepartitionOperationTopology() throws ExecutionExce final String topology = builder.build().describe().toString(); - assertEquals(2, getCountOfRepartitionTopicsFound(topology, "Sink: .*-repartition.*")); + assertEquals(2, countOccurencesInTopology(topology, "Sink: .*-repartition.*")); } @Test @@ -182,12 +182,12 @@ public void shouldCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() final String topology = builder.build().describe().toString(); assertTrue(topicExists(toRepartitionTopicName(repartitionName))); - assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*dummy-repartition.*")); + assertEquals(1, countOccurencesInTopology(topology, "Sink: .*dummy-repartition.*")); } @Test - public void shouldCreateRepartitionTopicWhenRepartitionKeySelectorIsUsed() throws ExecutionException, InterruptedException { - final String repartitionName = "new-key"; + public void shouldPerformKeySelectOperationWhenRepartitionOperationIsUsedWithKeySelector() throws ExecutionException, InterruptedException { + final String repartitionedName = "new-key"; final long timestamp = System.currentTimeMillis(); sendEvents( @@ -200,7 +200,7 @@ public void shouldCreateRepartitionTopicWhenRepartitionKeySelectorIsUsed() throw final StreamsBuilder builder = new StreamsBuilder(); - final Repartitioned repartitioned = Repartitioned.as(repartitionName) + final Repartitioned repartitioned = Repartitioned.as(repartitionedName) .withKeySerde(Serdes.String()); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) @@ -222,14 +222,15 @@ public void shouldCreateRepartitionTopicWhenRepartitionKeySelectorIsUsed() throw ); final String topology = builder.build().describe().toString(); - final String repartitionTopicName = toRepartitionTopicName(repartitionName); + final String repartitionTopicName = toRepartitionTopicName(repartitionedName); assertTrue(topicExists(repartitionTopicName)); - assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*" + repartitionName + "-repartition.*")); + assertEquals(1, countOccurencesInTopology(topology, "Sink: .*" + repartitionedName + "-repartition.*")); + assertEquals(1, countOccurencesInTopology(topology, "<-- " + repartitionedName)); } @Test - public void shouldCreateRepartitionTopicWhenNumberOfPartitionsIsSpecified() throws ExecutionException, InterruptedException { + public void shouldCreateRepartitionTopicWithSpecifiedNumberOfPartitions() throws ExecutionException, InterruptedException { final String repartitionName = "new-partitions"; final long timestamp = System.currentTimeMillis(); @@ -307,7 +308,7 @@ public void shouldInheritRepartitionTopicPartitionNumberFromUpstreamTopicWhenNum } @Test - public void shouldCreateOnlyOneRepartitionTopicForKStreamGroupBy() throws ExecutionException, InterruptedException { + public void shouldCreateOnlyOneRepartitionTopicWhenRepartitionIsFollowedByGroupByKey() throws ExecutionException, InterruptedException { final String repartitionName = "new-partitions"; final long timestamp = System.currentTimeMillis(); @@ -348,7 +349,7 @@ public void shouldCreateOnlyOneRepartitionTopicForKStreamGroupBy() throws Execut ); assertTrue(topicExists(toRepartitionTopicName(repartitionName))); - assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*-repartition")); + assertEquals(1, countOccurencesInTopology(topology, "Sink: .*-repartition")); } @Test @@ -383,7 +384,7 @@ public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws Execut final String topology = builder.build().describe().toString(); - assertEquals(1, getCountOfRepartitionTopicsFound(topology, "Sink: .*-repartition")); + assertEquals(1, countOccurencesInTopology(topology, "Sink: .*-repartition")); } private int getNumberOfPartitionsForTopic(final String topic) throws ExecutionException, InterruptedException { @@ -418,8 +419,8 @@ private AdminClient createAdminClient() { return AdminClient.create(properties); } - private int getCountOfRepartitionTopicsFound(final String topologyString, - final String searchPattern) { + private int countOccurencesInTopology(final String topologyString, + final String searchPattern) { final Matcher matcher = Pattern.compile(searchPattern).matcher(topologyString); final List repartitionTopicsFound = new ArrayList<>(); while (matcher.find()) { From cffd3ec5f43bb7fa018f0e40582e2702d2c984d7 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 11 Aug 2019 14:57:46 +0300 Subject: [PATCH 23/68] KAFKA-8611 / Optimize imports --- .../apache/kafka/streams/kstream/internals/KStreamImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 921123bfafb5a..92d833e72d720 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -42,13 +42,13 @@ import org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode.OptimizableRepartitionNodeBuilder; import org.apache.kafka.streams.kstream.internals.graph.ProcessorGraphNode; import org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters; -import org.apache.kafka.streams.kstream.internals.graph.UnoptimizableRepartitionNode; -import org.apache.kafka.streams.kstream.internals.graph.UnoptimizableRepartitionNode.UnoptimizableRepartitionNodeBuilder; import org.apache.kafka.streams.kstream.internals.graph.StatefulProcessorNode; import org.apache.kafka.streams.kstream.internals.graph.StreamSinkNode; import org.apache.kafka.streams.kstream.internals.graph.StreamStreamJoinNode; import org.apache.kafka.streams.kstream.internals.graph.StreamTableJoinNode; import org.apache.kafka.streams.kstream.internals.graph.StreamsGraphNode; +import org.apache.kafka.streams.kstream.internals.graph.UnoptimizableRepartitionNode; +import org.apache.kafka.streams.kstream.internals.graph.UnoptimizableRepartitionNode.UnoptimizableRepartitionNodeBuilder; import org.apache.kafka.streams.processor.FailOnInvalidTimestamp; import org.apache.kafka.streams.processor.ProcessorSupplier; import org.apache.kafka.streams.processor.StreamPartitioner; From e24d773b6028d96dbe53160d92e5e3f5cf625138 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 11 Aug 2019 19:32:08 +0300 Subject: [PATCH 24/68] KAFKA-8611 / Fix checkstyle; add unit tests for InternalTopologyBuilder --- .../kstream/internals/KStreamImpl.java | 2 +- .../graph/UnoptimizableRepartitionNode.java | 20 +++---- .../internals/InternalTopicConfig.java | 2 +- .../internals/InternalTopologyBuilder.java | 6 +- .../internals/RepartitionTopicConfig.java | 3 +- .../InternalTopologyBuilderTest.java | 56 +++++++++++++++++++ 6 files changed, 73 insertions(+), 16 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 92d833e72d720..2f3eb7f884fbf 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -536,7 +536,7 @@ public KStream repartition(final Repartitioned repartitioned) { } @Override - public KStream repartition(KeyValueMapper selector) { + public KStream repartition(final KeyValueMapper selector) { return repartition(selector, Repartitioned.as(null)); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java index 157c4ff235d19..47d1633e723ff 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java @@ -31,15 +31,15 @@ public class UnoptimizableRepartitionNode extends BaseRepartitionNode partitioner; private final InternalTopicProperties internalTopicProperties; - private UnoptimizableRepartitionNode(String nodeName, - String sourceName, - ProcessorParameters processorParameters, - Serde keySerde, - Serde valueSerde, - String sinkName, - String repartitionTopic, - StreamPartitioner partitioner, - InternalTopicProperties internalTopicProperties) { + private UnoptimizableRepartitionNode(final String nodeName, + final String sourceName, + final ProcessorParameters processorParameters, + final Serde keySerde, + final Serde valueSerde, + final String sinkName, + final String repartitionTopic, + final StreamPartitioner partitioner, + final InternalTopicProperties internalTopicProperties) { super( nodeName, sourceName, @@ -65,7 +65,7 @@ Deserializer getValueDeserializer() { } @Override - public void writeToTopology(InternalTopologyBuilder topologyBuilder) { + public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { final Serializer keySerializer = getKeySerializer(); final Deserializer keyDeserializer = getKeyDeserializer(); diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java index e61008e3fe7ca..3b3b1f7eb3594 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java @@ -30,7 +30,7 @@ public abstract class InternalTopicConfig { final String name; final Map topicConfigs; - private int numberOfPartitions; + int numberOfPartitions; InternalTopicConfig(final String name, final int numberOfPartitions, diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index f97a129289133..99340409418fc 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -1062,14 +1062,14 @@ public synchronized Map topicGroups() { return Collections.unmodifiableMap(topicGroups); } - private RepartitionTopicConfig buildRepartitionTopicConfig(String topic) { + private RepartitionTopicConfig buildRepartitionTopicConfig(final String topic) { // prefix the internal topic name with the application id - String internalTopic = decorateTopic(topic); + final String internalTopic = decorateTopic(topic); final InternalTopicProperties internalTopicProperties = internalTopicNamesWithProperties.get(topic); if (internalTopicProperties == null) { - return new RepartitionTopicConfig(internalTopic, Collections.emptyMap()); + return new RepartitionTopicConfig(internalTopic, Collections.emptyMap()); } final Integer numberOfPartitions = internalTopicProperties.getNumberOfPartitions(); diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java index b71d3545a310e..7286273d0c332 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java @@ -76,7 +76,8 @@ public boolean equals(final Object o) { } final RepartitionTopicConfig that = (RepartitionTopicConfig) o; return Objects.equals(name, that.name) && - Objects.equals(topicConfigs, that.topicConfigs); + Objects.equals(topicConfigs, that.topicConfigs) && + Objects.equals(numberOfPartitions, that.numberOfPartitions); } @Override diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java index e80a5c681afa1..5eead4860c745 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java @@ -846,4 +846,60 @@ public void sourceShouldNotBeEqualForDifferentPattern() { assertThat(base, not(equalTo(differentPattern))); assertThat(base, not(equalTo(overlappingPattern))); } + + @Test + public void shouldHaveCorrectInternalTopicConfigWhenInternalTopicPropertiesArePresent() { + final int numberOfPartitions = 10; + builder.setApplicationId("Z"); + builder.addInternalTopic("topic-1z", new InternalTopicProperties(numberOfPartitions)); + builder.addSource(null, "source-1", null, null, null, "topic-1z"); + + final Map topicGroups = builder.topicGroups(); + + Map repartitionSourceTopics = topicGroups.get(0).repartitionSourceTopics; + + assertEquals( + repartitionSourceTopics.get("Z-topic-1z"), + new RepartitionTopicConfig( + "Z-topic-1z", + numberOfPartitions, + Collections.emptyMap() + ) + ); + } + + @Test + public void shouldHandleWhenTopicPropertiesNumberOfPartitionsIsNull() { + builder.setApplicationId("T"); + builder.addInternalTopic("topic-1t", new InternalTopicProperties(null)); + builder.addSource(null, "source-1", null, null, null, "topic-1t"); + + final Map topicGroups = builder.topicGroups(); + + Map repartitionSourceTopics = topicGroups.get(0).repartitionSourceTopics; + + assertEquals( + repartitionSourceTopics.get("T-topic-1t"), + new RepartitionTopicConfig( + "T-topic-1t", + Collections.emptyMap() + ) + ); + } + + @Test + public void shouldHaveCorrectInternalTopicConfigWhenInternalTopicPropertiesAreNotPresent() { + builder.setApplicationId("Y"); + builder.addInternalTopic("topic-1y", null); + builder.addSource(null, "source-1", null, null, null, "topic-1y"); + + final Map topicGroups = builder.topicGroups(); + + Map repartitionSourceTopics = topicGroups.get(0).repartitionSourceTopics; + + assertEquals( + repartitionSourceTopics.get("Y-topic-1y"), + new RepartitionTopicConfig("Y-topic-1y", Collections.emptyMap()) + ); + } } From 0918c51fb793d74068d0b34206c253bf744fd19e Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 11 Aug 2019 19:43:30 +0300 Subject: [PATCH 25/68] KAFKA-8611 / Fix checkstyle test --- .../processor/internals/InternalTopologyBuilderTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java index 5eead4860c745..fa87c01a124bf 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java @@ -856,7 +856,7 @@ public void shouldHaveCorrectInternalTopicConfigWhenInternalTopicPropertiesArePr final Map topicGroups = builder.topicGroups(); - Map repartitionSourceTopics = topicGroups.get(0).repartitionSourceTopics; + final Map repartitionSourceTopics = topicGroups.get(0).repartitionSourceTopics; assertEquals( repartitionSourceTopics.get("Z-topic-1z"), @@ -876,7 +876,7 @@ public void shouldHandleWhenTopicPropertiesNumberOfPartitionsIsNull() { final Map topicGroups = builder.topicGroups(); - Map repartitionSourceTopics = topicGroups.get(0).repartitionSourceTopics; + final Map repartitionSourceTopics = topicGroups.get(0).repartitionSourceTopics; assertEquals( repartitionSourceTopics.get("T-topic-1t"), @@ -895,7 +895,7 @@ public void shouldHaveCorrectInternalTopicConfigWhenInternalTopicPropertiesAreNo final Map topicGroups = builder.topicGroups(); - Map repartitionSourceTopics = topicGroups.get(0).repartitionSourceTopics; + final Map repartitionSourceTopics = topicGroups.get(0).repartitionSourceTopics; assertEquals( repartitionSourceTopics.get("Y-topic-1y"), From a752f3fd37112c5caf51f8f2a3126a0e5895a716 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sat, 17 Aug 2019 13:39:08 +0300 Subject: [PATCH 26/68] KAFKA-8611 / Add integration test for rebalancing --- .../KStreamRepartitionIntegrationTest.java | 105 +++++++++++++++--- 1 file changed, 89 insertions(+), 16 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index bd238db5fa480..1aaa009e2835d 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -50,6 +50,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -69,17 +70,18 @@ public class KStreamRepartitionIntegrationTest { public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS); private Properties streamsConfiguration; - private KafkaStreams kafkaStreams; + private List kafkaStreamsInstances; private String inputTopic = "input-topic-" + testNo.get(); private String outputTopic = "output-topic-" + testNo.get(); private String applicationId = "kstream-repartition-stream-test-" + testNo.get(); @Before public void before() throws InterruptedException { - CLUSTER.createTopic(inputTopic, 3, 1); + CLUSTER.createTopic(inputTopic, 4, 1); CLUSTER.createTopic(outputTopic, 1, 1); streamsConfiguration = new Properties(); + kafkaStreamsInstances = new ArrayList<>(); streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId); streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); @@ -94,9 +96,11 @@ public void before() throws InterruptedException { @After public void whenShuttingDown() throws IOException { - if (kafkaStreams != null) { - kafkaStreams.close(); - } + kafkaStreamsInstances + .stream() + .filter(Objects::nonNull) + .forEach(KafkaStreams::close); + IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration); testNo.incrementAndGet(); } @@ -146,7 +150,7 @@ public void shouldNotOptimizeRepartitionOperationTopology() throws ExecutionExce final String topology = builder.build().describe().toString(); - assertEquals(2, countOccurencesInTopology(topology, "Sink: .*-repartition.*")); + assertEquals(2, countOccurrencesInTopology(topology, "Sink: .*-repartition.*")); } @Test @@ -182,7 +186,7 @@ public void shouldCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() final String topology = builder.build().describe().toString(); assertTrue(topicExists(toRepartitionTopicName(repartitionName))); - assertEquals(1, countOccurencesInTopology(topology, "Sink: .*dummy-repartition.*")); + assertEquals(1, countOccurrencesInTopology(topology, "Sink: .*dummy-repartition.*")); } @Test @@ -225,8 +229,8 @@ public void shouldPerformKeySelectOperationWhenRepartitionOperationIsUsedWithKey final String repartitionTopicName = toRepartitionTopicName(repartitionedName); assertTrue(topicExists(repartitionTopicName)); - assertEquals(1, countOccurencesInTopology(topology, "Sink: .*" + repartitionedName + "-repartition.*")); - assertEquals(1, countOccurencesInTopology(topology, "<-- " + repartitionedName)); + assertEquals(1, countOccurrencesInTopology(topology, "Sink: .*" + repartitionedName + "-repartition.*")); + assertEquals(1, countOccurrencesInTopology(topology, "<-- " + repartitionedName)); } @Test @@ -304,7 +308,7 @@ public void shouldInheritRepartitionTopicPartitionNumberFromUpstreamTopicWhenNum final String repartitionTopicName = toRepartitionTopicName(repartitionName); assertTrue(topicExists(repartitionTopicName)); - assertEquals(3, getNumberOfPartitionsForTopic(repartitionTopicName)); + assertEquals(4, getNumberOfPartitionsForTopic(repartitionTopicName)); } @Test @@ -349,7 +353,7 @@ public void shouldCreateOnlyOneRepartitionTopicWhenRepartitionIsFollowedByGroupB ); assertTrue(topicExists(toRepartitionTopicName(repartitionName))); - assertEquals(1, countOccurencesInTopology(topology, "Sink: .*-repartition")); + assertEquals(1, countOccurrencesInTopology(topology, "Sink: .*-repartition")); } @Test @@ -384,7 +388,72 @@ public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws Execut final String topology = builder.build().describe().toString(); - assertEquals(1, countOccurencesInTopology(topology, "Sink: .*-repartition")); + assertEquals(1, countOccurrencesInTopology(topology, "Sink: .*-repartition")); + } + + @Test + public void shouldGoThroughRebalancingCorrectly() throws ExecutionException, InterruptedException { + final String repartitionName = "rebalancing-test"; + final long timestamp = System.currentTimeMillis(); + + sendEvents( + timestamp, + Arrays.asList( + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") + ) + ); + + final StreamsBuilder builder = new StreamsBuilder(); + + final Repartitioned repartitioned = Repartitioned.as(repartitionName) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .withNumberOfPartitions(2); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .selectKey((key, value) -> key.toString()) + .repartition(repartitioned) + .groupByKey() + .count() + .toStream() + .to(outputTopic); + + startStreams(builder); + final KafkaStreams kafkaStreamsToClose = startStreams(builder); + + validateReceivedMessages( + new StringDeserializer(), + new LongDeserializer(), + Arrays.asList( + new KeyValue<>("1", 1L), + new KeyValue<>("2", 1L) + ) + ); + + kafkaStreamsToClose.close(); + + sendEvents( + timestamp, + Arrays.asList( + new KeyValue<>(1, "C"), + new KeyValue<>(2, "D") + ) + ); + + validateReceivedMessages( + new StringDeserializer(), + new LongDeserializer(), + Arrays.asList( + new KeyValue<>("1", 2L), + new KeyValue<>("2", 2L) + ) + ); + + final String repartitionTopicName = toRepartitionTopicName(repartitionName); + + assertTrue(topicExists(repartitionTopicName)); + assertEquals(2, getNumberOfPartitionsForTopic(repartitionTopicName)); } private int getNumberOfPartitionsForTopic(final String topic) throws ExecutionException, InterruptedException { @@ -419,13 +488,15 @@ private AdminClient createAdminClient() { return AdminClient.create(properties); } - private int countOccurencesInTopology(final String topologyString, - final String searchPattern) { + private int countOccurrencesInTopology(final String topologyString, + final String searchPattern) { final Matcher matcher = Pattern.compile(searchPattern).matcher(topologyString); final List repartitionTopicsFound = new ArrayList<>(); + while (matcher.find()) { repartitionTopicsFound.add(matcher.group()); } + return repartitionTopicsFound.size(); } @@ -444,9 +515,11 @@ private void sendEvents(final long timestamp, ); } - private void startStreams(final StreamsBuilder builder) { - kafkaStreams = new KafkaStreams(builder.build(streamsConfiguration), streamsConfiguration); + private KafkaStreams startStreams(final StreamsBuilder builder) { + KafkaStreams kafkaStreams = new KafkaStreams(builder.build(streamsConfiguration), streamsConfiguration); kafkaStreams.start(); + kafkaStreamsInstances.add(kafkaStreams); + return kafkaStreams; } private void validateReceivedMessages(final Deserializer keySerializer, From 6187f50b332b68b134cc3d522cac12864fd57bcc Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sat, 17 Aug 2019 23:21:06 +0300 Subject: [PATCH 27/68] KAFKA-8611 / fix checkstyle test --- .../streams/integration/KStreamRepartitionIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 1aaa009e2835d..1cc18e0b80f11 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -516,7 +516,7 @@ private void sendEvents(final long timestamp, } private KafkaStreams startStreams(final StreamsBuilder builder) { - KafkaStreams kafkaStreams = new KafkaStreams(builder.build(streamsConfiguration), streamsConfiguration); + final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(streamsConfiguration), streamsConfiguration); kafkaStreams.start(); kafkaStreamsInstances.add(kafkaStreams); return kafkaStreams; From b02983596e2db2031a784efabacdd3ea6d45a11b Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 25 Aug 2019 11:39:11 +0300 Subject: [PATCH 28/68] CR: KStream javadoc update Co-Authored-By: John Roesler --- .../org/apache/kafka/streams/kstream/KStream.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java index 7a153ebf51b1e..461e0f788829e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java @@ -839,8 +839,8 @@ KStream through(final String topic, final Produced produced); /** - * Materialize this stream to a auto-generated repartition topic and create a new {@code KStream} - * from the auto-generated topic using default serializers, deserializers, and producer's {@link DefaultPartitioner}. + * Materialize this stream to an auto-generated repartition topic and create a new {@code KStream} + * from the auto-generated topic using default serializers, deserializers, and producers {@link DefaultPartitioner}. * The number of partitions is determined based on the upstream topics partition numbers. * * @return {@code KStream} that contains the exact same repartitioned records as this {@code KStream}. @@ -848,7 +848,7 @@ KStream through(final String topic, KStream repartition(); /** - * Materialize this stream to a auto-generated repartition topic and create a new {@code KStream} + * Materialize this stream to an auto-generated repartition topic and create a new {@code KStream} * from the auto-generated topic using {@link Serde key serde}, {@link Serde value serde}, {@link StreamPartitioner}, * number of partitions, and topic name part as defined by {@link Repartitioned}. * @@ -860,8 +860,8 @@ KStream through(final String topic, KStream repartition(final Repartitioned repartitioned); /** - * Materialize this stream to a auto-generated repartition topic and create a new {@code KStream} - * from the auto-generated topic using default serializers, deserializers, and producer's {@link DefaultPartitioner}. + * Materialize this stream to an auto-generated repartition topic and create a new {@code KStream} + * from the auto-generated topic using default serializers, deserializers, and producers {@link DefaultPartitioner}. * The number of partitions is determined based on the upstream topics partition numbers. * * The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. @@ -873,7 +873,7 @@ KStream through(final String topic, KStream repartition(final KeyValueMapper selector); /** - * Materialize this stream to a auto-generated repartition topic and create a new {@code KStream} + * Materialize this stream to an auto-generated repartition topic and create a new {@code KStream} * from the auto-generated topic using {@link Serde key serde}, {@link Serde value serde}, {@link StreamPartitioner}, * number of partitions, and topic name part as defined by {@link Repartitioned}. * The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. From 2f7b69094a592b6db006489207b6ac216d6e7ab8 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 25 Aug 2019 11:48:36 +0300 Subject: [PATCH 29/68] KAFKA-8611 / CR: add numberOfPartitions field to toString and hashCode methods --- .../streams/processor/internals/RepartitionTopicConfig.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java index 7286273d0c332..007c61dea642e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java @@ -82,7 +82,7 @@ public boolean equals(final Object o) { @Override public int hashCode() { - return Objects.hash(name, topicConfigs); + return Objects.hash(name, topicConfigs, numberOfPartitions); } @Override @@ -90,6 +90,7 @@ public String toString() { return "RepartitionTopicConfig(" + "name=" + name + ", topicConfigs=" + topicConfigs + + ", numberOfPartitions=" + numberOfPartitions + ")"; } } From 185e26b43497da8bcf5717bae8122211ac5a7f3b Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 25 Aug 2019 11:59:47 +0300 Subject: [PATCH 30/68] KAFKA-8611 / CR: do not use NamedInternal for KStream#repartition operation --- .../apache/kafka/streams/kstream/internals/KStreamImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 2f3eb7f884fbf..21c972aee438b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -501,8 +501,7 @@ public KStream repartition(final Repartitioned repartitioned) { final RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); - final String name = new NamedInternal(repartitionedInternal.name()) - .orElseGenerateWithPrefix(builder, REPARTITION_NAME); + final String name = repartitionedInternal.name() != null ? repartitionedInternal.name() : builder.newProcessorName(REPARTITION_NAME); final Serde keySerde = repartitionedInternal.keySerde() == null ? this.keySerde : repartitionedInternal.keySerde(); From 55acc802791aee74c8d192c9416df77a50023d9b Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 25 Aug 2019 12:35:59 +0300 Subject: [PATCH 31/68] KAFKA-8611 / CR: ditch InternalTopologyBuilder#internalTopicNames set --- .../GroupedTableOperationRepartitionNode.java | 3 +- .../graph/OptimizableRepartitionNode.java | 3 +- .../internals/InternalTopicProperties.java | 4 ++ .../internals/InternalTopologyBuilder.java | 39 ++++++++----------- .../InternalTopologyBuilderTest.java | 19 +++++---- .../internals/ProcessorTopologyTest.java | 4 +- .../StreamsPartitionAssignorTest.java | 6 +-- 7 files changed, 41 insertions(+), 37 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java index 034c5a33eb7dd..c235cabaa3d1b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java @@ -23,6 +23,7 @@ import org.apache.kafka.streams.kstream.internals.ChangedDeserializer; import org.apache.kafka.streams.kstream.internals.ChangedSerializer; import org.apache.kafka.streams.processor.FailOnInvalidTimestamp; +import org.apache.kafka.streams.processor.internals.InternalTopicProperties; import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; public class GroupedTableOperationRepartitionNode extends BaseRepartitionNode { @@ -79,7 +80,7 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { final Serializer keySerializer = keySerde != null ? keySerde.serializer() : null; final Deserializer keyDeserializer = keySerde != null ? keySerde.deserializer() : null; - topologyBuilder.addInternalTopic(repartitionTopic, null); + topologyBuilder.addInternalTopic(repartitionTopic, InternalTopicProperties.empty()); topologyBuilder.addSink( sinkName, diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index 94b9cd8bead02..e4f85bb6d3633 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.streams.processor.FailOnInvalidTimestamp; +import org.apache.kafka.streams.processor.internals.InternalTopicProperties; import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; public class OptimizableRepartitionNode extends BaseRepartitionNode { @@ -77,7 +78,7 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { final Serializer keySerializer = keySerde != null ? keySerde.serializer() : null; final Deserializer keyDeserializer = keySerde != null ? keySerde.deserializer() : null; - topologyBuilder.addInternalTopic(repartitionTopic, null); + topologyBuilder.addInternalTopic(repartitionTopic, InternalTopicProperties.empty()); topologyBuilder.addProcessor( processorParameters.processorName(), diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java index 1523e5bef10b4..2c4c6a8b4d13d 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java @@ -26,4 +26,8 @@ public InternalTopicProperties(final Integer numberOfPartitions) { public Integer getNumberOfPartitions() { return numberOfPartitions; } + + public static InternalTopicProperties empty() { + return new InternalTopicProperties(null); + } } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index 99340409418fc..71d8d0cbb0613 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -72,10 +72,7 @@ public class InternalTopologyBuilder { // all topics subscribed from source processors (without application-id prefix for internal topics) private final Set sourceTopicNames = new HashSet<>(); - // all internal topics auto-created by the topology builder and used in source / sink processors - private final Set internalTopicNames = new HashSet<>(); - - // map of internal topic names with topic configuration + // all internal topics with their corresponding properties auto-created by the topology builder and used in source / sink processors private final Map internalTopicNamesWithProperties = new HashMap<>(); // groups of source processors that need to be copartitioned @@ -312,7 +309,7 @@ private SinkNodeFactory(final String name, public ProcessorNode build() { if (topicExtractor instanceof StaticTopicNameExtractor) { final String topic = ((StaticTopicNameExtractor) topicExtractor).topicName; - if (internalTopicNames.contains(topic)) { + if (internalTopicNamesWithProperties.containsKey(topic)) { // prefix the internal topic name with the application id return new SinkNode<>(name, new StaticTopicNameExtractor<>(decorateTopic(topic)), keySerializer, valSerializer, partitioner); } else { @@ -613,11 +610,9 @@ public void connectSourceStoreAndTopic(final String sourceStoreName, public final void addInternalTopic(final String topicName, final InternalTopicProperties internalTopicProperties) { Objects.requireNonNull(topicName, "topicName can't be null"); - internalTopicNames.add(topicName); - - if (internalTopicProperties != null) { - internalTopicNamesWithProperties.put(topicName, internalTopicProperties); - } + Objects.requireNonNull(internalTopicProperties, "internalTopicProperties can't be null"); + + internalTopicNamesWithProperties.put(topicName, internalTopicProperties); } public final void copartitionSources(final Collection sourceNodes) { @@ -905,7 +900,7 @@ private void buildSinkNode(final Map processorMap, if (sinkNodeFactory.topicExtractor instanceof StaticTopicNameExtractor) { final String topic = ((StaticTopicNameExtractor) sinkNodeFactory.topicExtractor).topicName; - if (internalTopicNames.contains(topic)) { + if (internalTopicNamesWithProperties.containsKey(topic)) { // prefix the internal topic name with the application id final String decoratedTopic = decorateTopic(topic); topicSinkMap.put(decoratedTopic, node); @@ -928,7 +923,7 @@ private void buildSourceNode(final Map topicSourceMap, sourceNodeFactory.topics; for (final String topic : topics) { - if (internalTopicNames.contains(topic)) { + if (internalTopicNamesWithProperties.containsKey(topic)) { // prefix the internal topic name with the application id final String decoratedTopic = decorateTopic(topic); topicSourceMap.put(decoratedTopic, node); @@ -1013,8 +1008,11 @@ public synchronized Map topicGroups() { if (globalTopics.contains(topic)) { continue; } - if (internalTopicNames.contains(topic)) { - final RepartitionTopicConfig repartitionTopicConfig = buildRepartitionTopicConfig(topic); + if (internalTopicNamesWithProperties.containsKey(topic)) { + final RepartitionTopicConfig repartitionTopicConfig = buildRepartitionTopicConfig( + topic, + internalTopicNamesWithProperties.get(topic) + ); repartitionTopics.put(repartitionTopicConfig.name(), repartitionTopicConfig); sourceTopics.add(repartitionTopicConfig.name()); @@ -1027,7 +1025,7 @@ public synchronized Map topicGroups() { // if the node is a sink node, add to the sink topics final String topic = nodeToSinkTopic.get(node); if (topic != null) { - if (internalTopicNames.contains(topic)) { + if (internalTopicNamesWithProperties.containsKey(topic)) { // prefix the change log topic name with the application id sinkTopics.add(decorateTopic(topic)); } else { @@ -1062,16 +1060,11 @@ public synchronized Map topicGroups() { return Collections.unmodifiableMap(topicGroups); } - private RepartitionTopicConfig buildRepartitionTopicConfig(final String topic) { + private RepartitionTopicConfig buildRepartitionTopicConfig(final String topic, + final InternalTopicProperties internalTopicProperties) { // prefix the internal topic name with the application id final String internalTopic = decorateTopic(topic); - final InternalTopicProperties internalTopicProperties = internalTopicNamesWithProperties.get(topic); - - if (internalTopicProperties == null) { - return new RepartitionTopicConfig(internalTopic, Collections.emptyMap()); - } - final Integer numberOfPartitions = internalTopicProperties.getNumberOfPartitions(); if (numberOfPartitions == null) { @@ -1191,7 +1184,7 @@ public synchronized Collection> copartitionGroups() { private List maybeDecorateInternalSourceTopics(final Collection sourceTopics) { final List decoratedTopics = new ArrayList<>(); for (final String topic : sourceTopics) { - if (internalTopicNames.contains(topic)) { + if (internalTopicNamesWithProperties.containsKey(topic)) { decoratedTopics.add(decorateTopic(topic)); } else { decoratedTopics.add(topic); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java index fa87c01a124bf..34b5feee8c90b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java @@ -236,7 +236,7 @@ public void testSourceTopics() { builder.addSource(null, "source-1", null, null, null, "topic-1"); builder.addSource(null, "source-2", null, null, null, "topic-2"); builder.addSource(null, "source-3", null, null, null, "topic-3"); - builder.addInternalTopic("topic-3", null); + builder.addInternalTopic("topic-3", InternalTopicProperties.empty()); final Pattern expectedPattern = Pattern.compile("X-topic-3|topic-1|topic-2"); @@ -336,7 +336,7 @@ public void testAddStateStore() { @Test public void testTopicGroups() { builder.setApplicationId("X"); - builder.addInternalTopic("topic-1x", null); + builder.addInternalTopic("topic-1x", InternalTopicProperties.empty()); builder.addSource(null, "source-1", null, null, null, "topic-1", "topic-1x"); builder.addSource(null, "source-2", null, null, null, "topic-2"); builder.addSource(null, "source-3", null, null, null, "topic-3"); @@ -524,7 +524,12 @@ public void shouldNotAllowNullStateStoreNameWhenConnectingProcessorAndStateStore @Test(expected = NullPointerException.class) public void shouldNotAddNullInternalTopic() { - builder.addInternalTopic(null, null); + builder.addInternalTopic(null, InternalTopicProperties.empty()); + } + + @Test(expected = NullPointerException.class) + public void shouldNotAddNullInternalTopicProperties() { + builder.addInternalTopic("topic", null); } @Test(expected = NullPointerException.class) @@ -568,7 +573,7 @@ public void shouldAssociateStateStoreNameWhenStateStoreSupplierIsExternal() { @Test public void shouldCorrectlyMapStateStoreToInternalTopics() { builder.setApplicationId("appId"); - builder.addInternalTopic("internal-topic", null); + builder.addInternalTopic("internal-topic", InternalTopicProperties.empty()); builder.addSource(null, "source", null, null, null, "internal-topic"); builder.addProcessor("processor", new MockProcessorSupplier(), "source"); builder.addStateStore(storeBuilder, "processor"); @@ -633,7 +638,7 @@ public void shouldAddInternalTopicConfigForNonWindowStores() { @Test public void shouldAddInternalTopicConfigForRepartitionTopics() { builder.setApplicationId("appId"); - builder.addInternalTopic("foo", null); + builder.addInternalTopic("foo", InternalTopicProperties.empty()); builder.addSource(null, "source", null, null, null, "foo"); final InternalTopologyBuilder.TopicsInfo topicsInfo = builder.topicGroups().values().iterator().next(); final InternalTopicConfig topicConfig = topicsInfo.repartitionSourceTopics.get("appId-foo"); @@ -871,7 +876,7 @@ public void shouldHaveCorrectInternalTopicConfigWhenInternalTopicPropertiesArePr @Test public void shouldHandleWhenTopicPropertiesNumberOfPartitionsIsNull() { builder.setApplicationId("T"); - builder.addInternalTopic("topic-1t", new InternalTopicProperties(null)); + builder.addInternalTopic("topic-1t", InternalTopicProperties.empty()); builder.addSource(null, "source-1", null, null, null, "topic-1t"); final Map topicGroups = builder.topicGroups(); @@ -890,7 +895,7 @@ public void shouldHandleWhenTopicPropertiesNumberOfPartitionsIsNull() { @Test public void shouldHaveCorrectInternalTopicConfigWhenInternalTopicPropertiesAreNotPresent() { builder.setApplicationId("Y"); - builder.addInternalTopic("topic-1y", null); + builder.addInternalTopic("topic-1y", InternalTopicProperties.empty()); builder.addSource(null, "source-1", null, null, null, "topic-1y"); final Map topicGroups = builder.topicGroups(); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java index a8150c840dfbb..f8dec82cf988b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorTopologyTest.java @@ -557,7 +557,7 @@ private Topology createInternalRepartitioningTopology() { // use wrapper to get the internal topology builder to add internal topic final InternalTopologyBuilder internalTopologyBuilder = TopologyWrapper.getInternalTopologyBuilder(topology); - internalTopologyBuilder.addInternalTopic(THROUGH_TOPIC_1, null); + internalTopologyBuilder.addInternalTopic(THROUGH_TOPIC_1, InternalTopicProperties.empty()); return topology; } @@ -571,7 +571,7 @@ private Topology createInternalRepartitioningWithValueTimestampTopology() { // use wrapper to get the internal topology builder to add internal topic final InternalTopologyBuilder internalTopologyBuilder = TopologyWrapper.getInternalTopologyBuilder(topology); - internalTopologyBuilder.addInternalTopic(THROUGH_TOPIC_1, null); + internalTopologyBuilder.addInternalTopic(THROUGH_TOPIC_1, InternalTopicProperties.empty()); return topology; } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java index cc41e65623ef2..6e8e33df9d3da 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java @@ -691,7 +691,7 @@ public void testOnAssignment() { @Test public void testAssignWithInternalTopics() { builder.setApplicationId(applicationId); - builder.addInternalTopic("topicX", null); + builder.addInternalTopic("topicX", InternalTopicProperties.empty()); builder.addSource(null, "source1", null, null, null, "topic1"); builder.addProcessor("processor1", new MockProcessorSupplier(), "source1"); builder.addSink("sink1", "topicX", null, null, null, "processor1"); @@ -721,12 +721,12 @@ public void testAssignWithInternalTopics() { public void testAssignWithInternalTopicThatsSourceIsAnotherInternalTopic() { final String applicationId = "test"; builder.setApplicationId(applicationId); - builder.addInternalTopic("topicX", null); + builder.addInternalTopic("topicX", InternalTopicProperties.empty()); builder.addSource(null, "source1", null, null, null, "topic1"); builder.addProcessor("processor1", new MockProcessorSupplier(), "source1"); builder.addSink("sink1", "topicX", null, null, null, "processor1"); builder.addSource(null, "source2", null, null, null, "topicX"); - builder.addInternalTopic("topicZ", null); + builder.addInternalTopic("topicZ", InternalTopicProperties.empty()); builder.addProcessor("processor2", new MockProcessorSupplier(), "source2"); builder.addSink("sink2", "topicZ", null, null, null, "processor2"); builder.addSource(null, "source3", null, null, null, "topicZ"); From cb7b2df659f43bdf9806382c418aadbd62ecf28d Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 25 Aug 2019 12:50:34 +0300 Subject: [PATCH 32/68] KAFKA-8611 / CR: KStreamRepartitionIntegrationTest update --- .../KStreamRepartitionIntegrationTest.java | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 1cc18e0b80f11..73abdfcf73ab2 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -53,27 +53,32 @@ import java.util.Objects; import java.util.Properties; import java.util.Set; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.kafka.streams.KafkaStreams.State.REBALANCING; +import static org.apache.kafka.streams.KafkaStreams.State.RUNNING; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @Category({IntegrationTest.class}) public class KStreamRepartitionIntegrationTest { private static final int NUM_BROKERS = 1; - private static AtomicInteger testNo = new AtomicInteger(0); + private static final AtomicInteger TEST_NUM = new AtomicInteger(0); @ClassRule public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS); + private final String inputTopic = "input-topic-" + TEST_NUM.get(); + private final String outputTopic = "output-topic-" + TEST_NUM.get(); + private final String applicationId = "kstream-repartition-stream-test-" + TEST_NUM.get(); + private Properties streamsConfiguration; private List kafkaStreamsInstances; - private String inputTopic = "input-topic-" + testNo.get(); - private String outputTopic = "output-topic-" + testNo.get(); - private String applicationId = "kstream-repartition-stream-test-" + testNo.get(); @Before public void before() throws InterruptedException { @@ -102,7 +107,7 @@ public void whenShuttingDown() throws IOException { .forEach(KafkaStreams::close); IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration); - testNo.incrementAndGet(); + TEST_NUM.incrementAndGet(); } @Test @@ -288,7 +293,7 @@ public void shouldInheritRepartitionTopicPartitionNumberFromUpstreamTopicWhenNum final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition(Repartitioned.as(repartitionName)) + .repartition(Repartitioned.as(repartitionName)) .groupByKey() .count() .toStream() @@ -457,7 +462,7 @@ public void shouldGoThroughRebalancingCorrectly() throws ExecutionException, Int } private int getNumberOfPartitionsForTopic(final String topic) throws ExecutionException, InterruptedException { - try (AdminClient adminClient = createAdminClient()) { + try (final AdminClient adminClient = createAdminClient()) { final TopicDescription topicDescription = adminClient.describeTopics(Collections.singleton(topic)) .values() .get(topic) @@ -468,7 +473,7 @@ private int getNumberOfPartitionsForTopic(final String topic) throws ExecutionEx } private boolean topicExists(final String topic) throws InterruptedException, ExecutionException { - try (AdminClient adminClient = createAdminClient()) { + try (final AdminClient adminClient = createAdminClient()) { final Set topics = adminClient.listTopics() .names() .get(); @@ -516,10 +521,24 @@ private void sendEvents(final long timestamp, } private KafkaStreams startStreams(final StreamsBuilder builder) { + final CountDownLatch latch = new CountDownLatch(1); final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(streamsConfiguration), streamsConfiguration); + + kafkaStreams.setStateListener((newState, oldState) -> { + if (REBALANCING == oldState && RUNNING == newState) { + latch.countDown(); + } + }); + kafkaStreams.start(); - kafkaStreamsInstances.add(kafkaStreams); - return kafkaStreams; + + try { + latch.await(IntegrationTestUtils.DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS); + kafkaStreamsInstances.add(kafkaStreams); + return kafkaStreams; + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } } private void validateReceivedMessages(final Deserializer keySerializer, @@ -528,7 +547,7 @@ private void validateReceivedMessages(final Deserializer keySerializer final Properties consumerProperties = new Properties(); consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); - consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "kstream-repartition-test-" + testNo); + consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "kstream-repartition-test-" + TEST_NUM.get()); consumerProperties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); consumerProperties.setProperty( ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, From 37b393af9624e4f1278c607ba5e47315ac2b4f21 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 25 Aug 2019 13:30:29 +0300 Subject: [PATCH 33/68] KAFKA-8611 / CR: add unit test for topology optimization --- .../KStreamRepartitionIntegrationTest.java | 51 ------------------- .../internals/graph/StreamsGraphTest.java | 35 +++++++++++++ 2 files changed, 35 insertions(+), 51 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 73abdfcf73ab2..6fffd7809e35f 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -34,8 +34,6 @@ import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; import org.apache.kafka.streams.kstream.Consumed; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.Repartitioned; import org.apache.kafka.test.IntegrationTest; import org.apache.kafka.test.TestUtils; @@ -96,7 +94,6 @@ public void before() throws InterruptedException { streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100); streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass()); streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); - streamsConfiguration.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.OPTIMIZE); } @After @@ -110,54 +107,6 @@ public void whenShuttingDown() throws IOException { TEST_NUM.incrementAndGet(); } - @Test - public void shouldNotOptimizeRepartitionOperationTopology() throws ExecutionException, InterruptedException { - final long timestamp = System.currentTimeMillis(); - - sendEvents( - timestamp, - Arrays.asList( - new KeyValue<>(1, "A"), - new KeyValue<>(2, "B") - ) - ); - - final StreamsBuilder builder = new StreamsBuilder(); - - final KStream stream = builder.stream(inputTopic); - - final KStream countStream1 = stream - .repartition((key, value) -> key) - .groupByKey() - .count() - .toStream(); - - final KStream countStream2 = stream - .repartition((key, value) -> key) - .groupByKey() - .count() - .toStream(); - - countStream1.merge(countStream2).to(outputTopic, Produced.with(Serdes.Integer(), Serdes.Long())); - - startStreams(builder); - - validateReceivedMessages( - new IntegerDeserializer(), - new LongDeserializer(), - Arrays.asList( - new KeyValue<>(1, 1L), - new KeyValue<>(1, 1L), - new KeyValue<>(2, 1L), - new KeyValue<>(2, 1L) - ) - ); - - final String topology = builder.build().describe().toString(); - - assertEquals(2, countOccurrencesInTopology(topology, "Sink: .*-repartition.*")); - } - @Test public void shouldCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() throws ExecutionException, InterruptedException { final String repartitionName = "dummy"; diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/graph/StreamsGraphTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/graph/StreamsGraphTest.java index 0fecaa2963eb1..7f88ff5b3e772 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/graph/StreamsGraphTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/graph/StreamsGraphTest.java @@ -124,6 +124,16 @@ public void shouldNotOptimizeWhenAThroughOperationIsDone() { } + @Test + public void shouldNotOptimizeWhenRepartitionOperationIsDone() { + final Topology attemptedOptimize = getTopologyWithRepartitionOperation(StreamsConfig.OPTIMIZE); + final Topology noOptimziation = getTopologyWithRepartitionOperation(StreamsConfig.NO_OPTIMIZATION); + + assertEquals(attemptedOptimize.describe().toString(), noOptimziation.describe().toString()); + assertEquals(2, getCountOfRepartitionTopicsFound(attemptedOptimize.describe().toString())); + assertEquals(2, getCountOfRepartitionTopicsFound(noOptimziation.describe().toString())); + } + private Topology getTopologyWithChangingValuesAfterChangingKey(final String optimizeConfig) { final StreamsBuilder builder = new StreamsBuilder(); @@ -156,6 +166,31 @@ private Topology getTopologyWithThroughOperation(final String optimizeConfig) { } + private Topology getTopologyWithRepartitionOperation(final String optimizeConfig) { + final StreamsBuilder builder = new StreamsBuilder(); + final Properties properties = new Properties(); + properties.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, optimizeConfig); + + final KStream inputStream = builder.stream("input").selectKey((k, v) -> k + v); + + inputStream + .repartition() + .groupByKey() + .count() + .toStream() + .to("output"); + + inputStream + .repartition() + .groupByKey() + .windowedBy(TimeWindows.of(ofMillis(5000))) + .count() + .toStream() + .to("windowed-output"); + + return builder.build(properties); + } + private int getCountOfRepartitionTopicsFound(final String topologyString) { final Matcher matcher = repartitionTopicPattern.matcher(topologyString); final List repartitionTopicsFound = new ArrayList<>(); From 463a237b7b477ea43d2fefcbe2ca4c8d2dc26952 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 25 Aug 2019 18:27:24 +0300 Subject: [PATCH 34/68] KAFKA-8611 / CR: make methods static in KStreamRepartitionIntegrationTest --- .../integration/KStreamRepartitionIntegrationTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 6fffd7809e35f..348cddd356473 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -435,15 +435,15 @@ private String toRepartitionTopicName(final String input) { return applicationId + "-" + input + "-repartition"; } - private AdminClient createAdminClient() { + private static AdminClient createAdminClient() { final Properties properties = new Properties(); properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); return AdminClient.create(properties); } - private int countOccurrencesInTopology(final String topologyString, - final String searchPattern) { + private static int countOccurrencesInTopology(final String topologyString, + final String searchPattern) { final Matcher matcher = Pattern.compile(searchPattern).matcher(topologyString); final List repartitionTopicsFound = new ArrayList<>(); From e6d6598e2c2313250b4112f4dd014a78e441c983 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 25 Aug 2019 20:49:58 +0300 Subject: [PATCH 35/68] KAFKA-8611 / CR: add partitioner and select key integration tests --- .../KStreamRepartitionIntegrationTest.java | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 348cddd356473..98f5d9114d60b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -107,6 +107,81 @@ public void whenShuttingDown() throws IOException { TEST_NUM.incrementAndGet(); } + @Test + public void shouldUseStreamPartitionerForRepartitionOperation() throws ExecutionException, InterruptedException { + final int partition = 1; + final String repartitionName = "partitioner-test"; + final long timestamp = System.currentTimeMillis(); + final AtomicInteger partitionerInvocation = new AtomicInteger(0); + + final List> expectedRecords = Arrays.asList( + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") + ); + + sendEvents(timestamp, expectedRecords); + + final StreamsBuilder builder = new StreamsBuilder(); + + final Repartitioned repartitioned = Repartitioned + .as(repartitionName) + .withStreamPartitioner((topic, key, value, numPartitions) -> { + partitionerInvocation.incrementAndGet(); + return partition; + }); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition(repartitioned) + .to(outputTopic); + + startStreams(builder); + + final String topic = toRepartitionTopicName(repartitionName); + + validateReceivedMessages( + new IntegerDeserializer(), + new StringDeserializer(), + expectedRecords + ); + + assertTrue(topicExists(topic)); + assertEquals(expectedRecords.size(), partitionerInvocation.get()); + } + + @Test + public void shouldPerformSelectKeyWithRepartitionOperation() throws ExecutionException, InterruptedException { + final long timestamp = System.currentTimeMillis(); + + sendEvents( + timestamp, + Arrays.asList( + new KeyValue<>(1, "10"), + new KeyValue<>(2, "20") + ) + ); + + final StreamsBuilder builder = new StreamsBuilder(); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition((key, value) -> Integer.valueOf(value)) + .to(outputTopic); + + startStreams(builder); + + validateReceivedMessages( + new IntegerDeserializer(), + new StringDeserializer(), + Arrays.asList( + new KeyValue<>(10, "10"), + new KeyValue<>(20, "20") + ) + ); + + final String topology = builder.build().describe().toString(); + + assertEquals(1, countOccurrencesInTopology(topology, "Sink: .*-repartition.*")); + } + @Test public void shouldCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() throws ExecutionException, InterruptedException { final String repartitionName = "dummy"; From 65702152882db15234d73486e4133405337fab08 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 25 Aug 2019 22:23:53 +0300 Subject: [PATCH 36/68] KAFKA-8611 / CR: add unit tests for stream partitioner --- .../KStreamRepartitionIntegrationTest.java | 13 +-- .../internals/KStreamRepartitionTest.java | 80 +++++++++++++++++++ 2 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 98f5d9114d60b..07429cd45ebed 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -88,7 +88,6 @@ public void before() throws InterruptedException { streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId); streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); - streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()); streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0); streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100); @@ -544,7 +543,7 @@ private void sendEvents(final long timestamp, ); } - private KafkaStreams startStreams(final StreamsBuilder builder) { + private KafkaStreams startStreams(final StreamsBuilder builder) throws InterruptedException { final CountDownLatch latch = new CountDownLatch(1); final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(streamsConfiguration), streamsConfiguration); @@ -556,13 +555,9 @@ private KafkaStreams startStreams(final StreamsBuilder builder) { kafkaStreams.start(); - try { - latch.await(IntegrationTestUtils.DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS); - kafkaStreamsInstances.add(kafkaStreams); - return kafkaStreams; - } catch (final InterruptedException e) { - throw new RuntimeException(e); - } + latch.await(IntegrationTestUtils.DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS); + kafkaStreamsInstances.add(kafkaStreams); + return kafkaStreams; } private void validateReceivedMessages(final Deserializer keySerializer, diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java new file mode 100644 index 0000000000000..128dcb7f4f213 --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java @@ -0,0 +1,80 @@ +package org.apache.kafka.streams.kstream.internals; + +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KeyValueTimestamp; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.TopologyTestDriver; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Repartitioned; +import org.apache.kafka.streams.processor.StreamPartitioner; +import org.apache.kafka.streams.test.ConsumerRecordFactory; +import org.apache.kafka.test.MockProcessor; +import org.apache.kafka.test.MockProcessorSupplier; +import org.apache.kafka.test.StreamsTestUtils; +import org.easymock.EasyMock; +import org.easymock.EasyMockRunner; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.Properties; + +import static org.easymock.EasyMock.anyInt; +import static org.easymock.EasyMock.anyString; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +@RunWith(EasyMockRunner.class) +public class KStreamRepartitionTest { + private final String inputTopic = "input-topic"; + + private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String()); + private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>( + Serdes.Integer().serializer(), + Serdes.String().serializer(), + 0L + ); + + private StreamsBuilder builder; + + @Before + public void setUp() { + builder = new StreamsBuilder(); + } + + @Test + public void shouldInvokePartitionerWhenSet() { + final int[] expectedKeys = new int[]{0, 1}; + final StreamPartitioner streamPartitionerMock = EasyMock.mock(StreamPartitioner.class); + + expect(streamPartitionerMock.partition(anyString(), eq(0), eq("X0"), anyInt())).andReturn(1).times(1); + expect(streamPartitionerMock.partition(anyString(), eq(1), eq("X1"), anyInt())).andReturn(1).times(1); + replay(streamPartitionerMock); + + final Repartitioned repartitioned = Repartitioned + .streamPartitioner(streamPartitionerMock) + .withName("repartition"); + + final KStream repartitionedStream = builder.stream(inputTopic) + .repartition(repartitioned); + + final MockProcessorSupplier supplier = new MockProcessorSupplier<>(); + + repartitionedStream.process(supplier); + + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { + for (int i = 0; i < 2; i++) { + driver.pipeInput(recordFactory.create(inputTopic, expectedKeys[i], "X" + expectedKeys[i], i + 10)); + } + final MockProcessor proc = supplier.theCapturedProcessor(); + proc.checkAndClearProcessResult( + new KeyValueTimestamp<>(0, "X0", 10), + new KeyValueTimestamp<>(1, "X1", 11) + ); + } + + verify(streamPartitionerMock); + } +} From 934f831e6c7a4c7a7b3e2dd0047260803fe0ac39 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 25 Aug 2019 22:44:00 +0300 Subject: [PATCH 37/68] KAFKA-8611 / checkstyle fix --- .../internals/KStreamRepartitionTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java index 128dcb7f4f213..a8273207b2cd2 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.kafka.streams.kstream.internals; import org.apache.kafka.common.serialization.Serdes; From dfaccd2202ba2388c440773d251637ec06239036 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Mon, 9 Sep 2019 20:37:12 +0300 Subject: [PATCH 38/68] KAFKA-8611 / fix checkstyle --- .../kafka/streams/processor/internals/InternalTopicConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java index cd8d7bc0b6bba..b2b70a76def14 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java @@ -30,7 +30,7 @@ public abstract class InternalTopicConfig { final String name; final Map topicConfigs; - Optional numberOfPartitions ; + Optional numberOfPartitions; InternalTopicConfig(final String name, final Integer numberOfPartitions, From 1af73c9678688c488946f7de53291160bb85b2de Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 29 Sep 2019 00:59:47 +0300 Subject: [PATCH 39/68] KAFKA-8611 / CR fixes: Update KStream#repartition javadocs. Refactored KStream#repartition implementations. BaseRepartitionNode related refactoring. Added integration tests for join operations --- .../apache/kafka/streams/kstream/KStream.java | 47 ++++++++ .../kstream/internals/KStreamImpl.java | 82 ++++++------- .../internals/graph/BaseRepartitionNode.java | 16 ++- .../GroupedTableOperationRepartitionNode.java | 19 ++- .../graph/OptimizableRepartitionNode.java | 23 +--- .../graph/UnoptimizableRepartitionNode.java | 31 +---- .../internals/InternalTopicConfig.java | 6 +- .../internals/InternalTopicProperties.java | 6 +- .../internals/InternalTopologyBuilder.java | 24 ++-- .../internals/RepartitionTopicConfig.java | 3 +- .../KStreamRepartitionIntegrationTest.java | 113 +++++++++++++++++- .../InternalTopologyBuilderTest.java | 3 +- 12 files changed, 242 insertions(+), 131 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java index 77697c8cb2be7..fa4b970ec8b4d 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java @@ -814,6 +814,10 @@ KStream flatMapValues(final ValueMapperWithKey through(final String topic); @@ -832,6 +836,10 @@ KStream flatMapValues(final ValueMapperWithKey through(final String topic, final Produced produced); @@ -841,7 +849,16 @@ KStream through(final String topic, * from the auto-generated topic using default serializers, deserializers, and producers {@link DefaultPartitioner}. * The number of partitions is determined based on the upstream topics partition numbers. * + * This operation is similar to {@link #through}, however, Kafka Streams manages the used topic automatically. + * Created topic is considered as internal topic and is meant to be used only by current Kafka Streams instance. + * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be purged by Kafka Streams explicitly. + * The topic will be named as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in + * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, + * "<name>" is an internally generated name, and "-repartition" is a fixed suffix. + * * @return {@code KStream} that contains the exact same repartitioned records as this {@code KStream}. + * @see #through(String) + * @see #through(String, Produced) */ KStream repartition(); @@ -850,10 +867,20 @@ KStream through(final String topic, * from the auto-generated topic using {@link Serde key serde}, {@link Serde value serde}, {@link StreamPartitioner}, * number of partitions, and topic name part as defined by {@link Repartitioned}. * + * This operation is similar to {@link #through}, however, Kafka Streams manages the used topic automatically. + * Created topic is considered as internal topic and is meant to be used only by current Kafka Streams instance. + * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be purged by Kafka Streams explicitly. + * The topic will be named as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in + * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, + * "<name>" is either provided via {@link Repartitioned#as(String)} or an internally + * generated name, and "-repartition" is a fixed suffix. + * * @param repartitioned the {@link Repartitioned} instance used to specify {@link Serdes}, * {@link StreamPartitioner} which determines how records are distributed among partitions of the topic, * part of the topic name, and number of partitions for a repartition topic. * @return a {@code KStream} that contains the exact same repartitioned records as this {@code KStream}. + * @see #through(String) + * @see #through(String, Produced) */ KStream repartition(final Repartitioned repartitioned); @@ -862,11 +889,20 @@ KStream through(final String topic, * from the auto-generated topic using default serializers, deserializers, and producers {@link DefaultPartitioner}. * The number of partitions is determined based on the upstream topics partition numbers. * + * This operation is similar to {@link #through}, however, Kafka Streams manages the used topic automatically. + * Created topic is considered as internal topic and is meant to be used only by current Kafka Streams instance. + * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be purged by Kafka Streams explicitly. + * The topic will be named as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in + * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, + * "<name>" is an internally generated name, and "-repartition" is a fixed suffix. + * * The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. * Thus, an input record {@code } can be transformed into an output record {@code }. * * @param selector a {@link KeyValueMapper} that computes a new key for each record * @return a {@code KStream} that contains records with new key (possibly of different type) and unmodified value + * @see #through(String) + * @see #through(String, Produced) */ KStream repartition(final KeyValueMapper selector); @@ -874,6 +910,15 @@ KStream through(final String topic, * Materialize this stream to an auto-generated repartition topic and create a new {@code KStream} * from the auto-generated topic using {@link Serde key serde}, {@link Serde value serde}, {@link StreamPartitioner}, * number of partitions, and topic name part as defined by {@link Repartitioned}. + * + * This operation is similar to {@link #through}, however, Kafka Streams manages the used topic automatically. + * Created topic is considered as internal topic and is meant to be used only by current Kafka Streams instance. + * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be purged by Kafka Streams explicitly. + * The topic will be named as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in + * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, + * "<name>" is either provided via {@link Repartitioned#as(String)} or an internally + * generated name, and "-repartition" is a fixed suffix. + * * The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. * Thus, an input record {@code } can be transformed into an output record {@code }. * @@ -882,6 +927,8 @@ KStream through(final String topic, * {@link StreamPartitioner} which determines how records are distributed among partitions of the topic, * part of the topic name, and number of partitions for a repartition topic. * @return a {@code KStream} that contains records with new key (possibly of different type) and unmodified value + * @see #through(String) + * @see #through(String, Produced) */ KStream repartition(final KeyValueMapper selector, final Repartitioned repartitioned); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 21c972aee438b..ee0fc894c2384 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -492,51 +492,21 @@ public KStream through(final String topic, final Produced produced) @Override public KStream repartition() { - return repartition(Repartitioned.as(null)); + return doRepartition(Repartitioned.as(null), null); } @Override public KStream repartition(final Repartitioned repartitioned) { Objects.requireNonNull(repartitioned, "repartitioned parameter can't be null"); - final RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); - - final String name = repartitionedInternal.name() != null ? repartitionedInternal.name() : builder.newProcessorName(REPARTITION_NAME); - - final Serde keySerde = repartitionedInternal.keySerde() == null ? this.keySerde : repartitionedInternal.keySerde(); - - final Serde valueSerde = repartitionedInternal.valueSerde() == null ? this.valSerde : repartitionedInternal.valueSerde(); - - final UnoptimizableRepartitionNodeBuilder unoptimizableRepartitionNodeBuilder = UnoptimizableRepartitionNode.repartitionNodeBuilder(); - - final String repartitionSourceName = createUnoptimizableRepartitionSource( - builder, - keySerde, - valueSerde, - name, - repartitionedInternal.toInternalTopicProperties(), - repartitionedInternal.streamPartitioner(), - unoptimizableRepartitionNodeBuilder - ); - - final UnoptimizableRepartitionNode unoptimizableRepartitionNode = unoptimizableRepartitionNodeBuilder.build(); - - builder.addGraphNode(this.streamsGraphNode, unoptimizableRepartitionNode); - - return new KStreamImpl<>( - repartitionSourceName, - keySerde, - valueSerde, - sourceNodes, - false, - unoptimizableRepartitionNode, - builder - ); + return doRepartition(repartitioned, null); } @Override public KStream repartition(final KeyValueMapper selector) { - return repartition(selector, Repartitioned.as(null)); + Objects.requireNonNull(selector, "selector parameter can't be null"); + + return doRepartition(Repartitioned.as(null), selector); } @Override @@ -545,26 +515,29 @@ public KStream repartition(final KeyValueMapper repartitionedInternal = new RepartitionedInternal<>(repartitioned); - - final NamedInternal namedInternal = new NamedInternal(repartitionedInternal.name()); + return doRepartition(repartitioned, selector); + } - final ProcessorGraphNode selectKeyNode = internalSelectKey(selector, namedInternal); + @SuppressWarnings("unchecked") + private KStream doRepartition(final Repartitioned repartitioned, + final KeyValueMapper selector) { + final RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); - selectKeyNode.keyChangingOperation(true); + final String name = repartitionedInternal.name() != null ? repartitionedInternal.name() : builder + .newProcessorName(REPARTITION_NAME); - builder.addGraphNode(this.streamsGraphNode, selectKeyNode); + final Serde keySerde = repartitionedInternal.keySerde() == null ? (Serde) this.keySerde : repartitionedInternal.keySerde(); - final UnoptimizableRepartitionNodeBuilder unoptimizableRepartitionNodeBuilder = UnoptimizableRepartitionNode.repartitionNodeBuilder(); + final Serde valueSerde = repartitionedInternal.valueSerde() == null ? valSerde : repartitionedInternal.valueSerde(); - final Serde keySerde = repartitionedInternal.keySerde(); - final Serde valueSerde = repartitionedInternal.valueSerde() == null ? this.valSerde : repartitionedInternal.valueSerde(); + final UnoptimizableRepartitionNodeBuilder unoptimizableRepartitionNodeBuilder = UnoptimizableRepartitionNode + .repartitionNodeBuilder(); final String repartitionSourceName = createUnoptimizableRepartitionSource( builder, keySerde, valueSerde, - namedInternal.orElseGenerateWithPrefix(builder, REPARTITION_NAME), + name, repartitionedInternal.toInternalTopicProperties(), repartitionedInternal.streamPartitioner(), unoptimizableRepartitionNodeBuilder @@ -572,13 +545,28 @@ public KStream repartition(final KeyValueMapper unoptimizableRepartitionNode = unoptimizableRepartitionNodeBuilder.build(); - builder.addGraphNode(selectKeyNode, unoptimizableRepartitionNode); + if (selector == null) { + builder.addGraphNode(streamsGraphNode, unoptimizableRepartitionNode); + } else { + final NamedInternal namedInternal = new NamedInternal(repartitionedInternal.name()); + + final ProcessorGraphNode selectKeyNode = internalSelectKey(selector, namedInternal); + + selectKeyNode.keyChangingOperation(true); + + builder.addGraphNode(streamsGraphNode, selectKeyNode); + + builder.addGraphNode(selectKeyNode, unoptimizableRepartitionNode); + } + + final Set sourceNodes = new HashSet<>(); + sourceNodes.add(unoptimizableRepartitionNode.nodeName()); return new KStreamImpl<>( repartitionSourceName, keySerde, valueSerde, - sourceNodes, + Collections.unmodifiableSet(sourceNodes), // explicitly set repartition required as false false, unoptimizableRepartitionNode, diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java index d65402d38703f..77f61870980f9 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java @@ -48,9 +48,21 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode { this.processorParameters = processorParameters; } - abstract Serializer getValueSerializer(); + Serializer valueSerializer() { + return valueSerde != null ? valueSerde.serializer() : null; + } + + Deserializer valueDeserializer() { + return valueSerde != null ? valueSerde.deserializer() : null; + } - abstract Deserializer getValueDeserializer(); + Serializer keySerializer() { + return keySerde != null ? keySerde.serializer() : null; + } + + Deserializer keyDeserializer() { + return keySerde != null ? keySerde.deserializer() : null; + } @Override public String toString() { diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java index c235cabaa3d1b..901b88634ad00 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java @@ -49,8 +49,8 @@ private GroupedTableOperationRepartitionNode(final String nodeName, } @Override - Serializer getValueSerializer() { - final Serializer valueSerializer = valueSerde == null ? null : valueSerde.serializer(); + Serializer valueSerializer() { + final Serializer valueSerializer = super.valueSerializer(); return unsafeCastChangedToValueSerializer(valueSerializer); } @@ -60,8 +60,8 @@ private Serializer unsafeCastChangedToValueSerializer(final Serializer val } @Override - Deserializer getValueDeserializer() { - final Deserializer valueDeserializer = valueSerde == null ? null : valueSerde.deserializer(); + Deserializer valueDeserializer() { + final Deserializer valueDeserializer = super.valueDeserializer(); return unsafeCastChangedToValueDeserializer(valueDeserializer); } @@ -77,16 +77,13 @@ public String toString() { @Override public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { - final Serializer keySerializer = keySerde != null ? keySerde.serializer() : null; - final Deserializer keyDeserializer = keySerde != null ? keySerde.deserializer() : null; - topologyBuilder.addInternalTopic(repartitionTopic, InternalTopicProperties.empty()); topologyBuilder.addSink( sinkName, repartitionTopic, - keySerializer, - getValueSerializer(), + keySerializer(), + valueSerializer(), null, parentNodeNames() ); @@ -95,8 +92,8 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { null, sourceName, new FailOnInvalidTimestamp(), - keyDeserializer, - getValueDeserializer(), + keyDeserializer(), + valueDeserializer(), repartitionTopic ); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index e4f85bb6d3633..952bd769b4ad5 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -17,9 +17,7 @@ package org.apache.kafka.streams.kstream.internals.graph; -import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.streams.processor.FailOnInvalidTimestamp; import org.apache.kafka.streams.processor.internals.InternalTopicProperties; import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; @@ -58,16 +56,6 @@ public String repartitionTopic() { return repartitionTopic; } - @Override - Serializer getValueSerializer() { - return valueSerde != null ? valueSerde.serializer() : null; - } - - @Override - Deserializer getValueDeserializer() { - return valueSerde != null ? valueSerde.deserializer() : null; - } - @Override public String toString() { return "OptimizableRepartitionNode{ " + super.toString() + " }"; @@ -75,9 +63,6 @@ public String toString() { @Override public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { - final Serializer keySerializer = keySerde != null ? keySerde.serializer() : null; - final Deserializer keyDeserializer = keySerde != null ? keySerde.deserializer() : null; - topologyBuilder.addInternalTopic(repartitionTopic, InternalTopicProperties.empty()); topologyBuilder.addProcessor( @@ -89,8 +74,8 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { topologyBuilder.addSink( sinkName, repartitionTopic, - keySerializer, - getValueSerializer(), + keySerializer(), + valueSerializer(), null, processorParameters.processorName() ); @@ -99,8 +84,8 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { null, sourceName, new FailOnInvalidTimestamp(), - keyDeserializer, - getValueDeserializer(), + keyDeserializer(), + valueDeserializer(), repartitionTopic ); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java index 47d1633e723ff..4bb8038812cac 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java @@ -16,9 +16,7 @@ */ package org.apache.kafka.streams.kstream.internals.graph; -import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.streams.processor.FailOnInvalidTimestamp; import org.apache.kafka.streams.processor.StreamPartitioner; import org.apache.kafka.streams.processor.internals.InternalTopicProperties; @@ -54,21 +52,8 @@ private UnoptimizableRepartitionNode(final String nodeName, this.internalTopicProperties = internalTopicProperties; } - @Override - Serializer getValueSerializer() { - return valueSerde != null ? valueSerde.serializer() : null; - } - - @Override - Deserializer getValueDeserializer() { - return valueSerde != null ? valueSerde.deserializer() : null; - } - @Override public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { - final Serializer keySerializer = getKeySerializer(); - final Deserializer keyDeserializer = getKeyDeserializer(); - topologyBuilder.addInternalTopic(repartitionTopic, internalTopicProperties); topologyBuilder.addProcessor( @@ -80,8 +65,8 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { topologyBuilder.addSink( sinkName, repartitionTopic, - keySerializer, - getValueSerializer(), + keySerializer(), + valueSerializer(), partitioner, processorParameters.processorName() ); @@ -90,20 +75,12 @@ public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { null, sourceName, new FailOnInvalidTimestamp(), - keyDeserializer, - getValueDeserializer(), + keyDeserializer(), + valueDeserializer(), repartitionTopic ); } - private Serializer getKeySerializer() { - return keySerde != null ? keySerde.serializer() : null; - } - - private Deserializer getKeyDeserializer() { - return keySerde != null ? keySerde.deserializer() : null; - } - public static UnoptimizableRepartitionNodeBuilder repartitionNodeBuilder() { return new UnoptimizableRepartitionNodeBuilder<>(); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java index b2b70a76def14..1e20cbb5b9f33 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java @@ -33,18 +33,18 @@ public abstract class InternalTopicConfig { Optional numberOfPartitions; InternalTopicConfig(final String name, - final Integer numberOfPartitions, + final Optional numberOfPartitions, final Map topicConfigs) { Objects.requireNonNull(name, "name can't be null"); Topic.validate(name); this.name = name; this.topicConfigs = topicConfigs; - this.numberOfPartitions = Optional.ofNullable(numberOfPartitions); + this.numberOfPartitions = numberOfPartitions; } InternalTopicConfig(final String name, final Map topicConfigs) { - this(name, null, topicConfigs); + this(name, Optional.empty(), topicConfigs); } /** diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java index 2c4c6a8b4d13d..508e11bc62821 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicProperties.java @@ -16,6 +16,8 @@ */ package org.apache.kafka.streams.processor.internals; +import java.util.Optional; + public class InternalTopicProperties { private final Integer numberOfPartitions; @@ -23,8 +25,8 @@ public InternalTopicProperties(final Integer numberOfPartitions) { this.numberOfPartitions = numberOfPartitions; } - public Integer getNumberOfPartitions() { - return numberOfPartitions; + public Optional getNumberOfPartitions() { + return Optional.ofNullable(numberOfPartitions); } public static InternalTopicProperties empty() { diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index 1fe6714bed53a..c9dfda5fa21bc 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -1009,9 +1009,13 @@ public synchronized Map topicGroups() { continue; } if (internalTopicNamesWithProperties.containsKey(topic)) { - final RepartitionTopicConfig repartitionTopicConfig = buildRepartitionTopicConfig( - topic, - internalTopicNamesWithProperties.get(topic) + // prefix the internal topic name with the application id + final String internalTopic = decorateTopic(topic); + + final RepartitionTopicConfig repartitionTopicConfig = new RepartitionTopicConfig( + internalTopic, + internalTopicNamesWithProperties.get(topic).getNumberOfPartitions(), + Collections.emptyMap() ); repartitionTopics.put(repartitionTopicConfig.name(), repartitionTopicConfig); @@ -1060,20 +1064,6 @@ public synchronized Map topicGroups() { return Collections.unmodifiableMap(topicGroups); } - private RepartitionTopicConfig buildRepartitionTopicConfig(final String topic, - final InternalTopicProperties internalTopicProperties) { - // prefix the internal topic name with the application id - final String internalTopic = decorateTopic(topic); - - final Integer numberOfPartitions = internalTopicProperties.getNumberOfPartitions(); - - if (numberOfPartitions == null) { - return new RepartitionTopicConfig(internalTopic, Collections.emptyMap()); - } - - return new RepartitionTopicConfig(internalTopic, numberOfPartitions, Collections.emptyMap()); - } - private void setRegexMatchedTopicsToSourceNodes() { if (subscriptionUpdates.hasUpdates()) { for (final Map.Entry stringPatternEntry : nodeToSourcePatterns.entrySet()) { diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java index 007c61dea642e..14495891d65b6 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.Optional; /** * RepartitionTopicConfig captures the properties required for configuring @@ -43,7 +44,7 @@ public class RepartitionTopicConfig extends InternalTopicConfig { } RepartitionTopicConfig(final String name, - final int numberOfPartitions, + final Optional numberOfPartitions, final Map topicConfigs) { super(name, numberOfPartitions, topicConfigs); } diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 07429cd45ebed..ef557679f9df0 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -34,6 +34,9 @@ import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; import org.apache.kafka.streams.kstream.Consumed; +import org.apache.kafka.streams.kstream.JoinWindows; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.Repartitioned; import org.apache.kafka.test.IntegrationTest; import org.apache.kafka.test.TestUtils; @@ -44,6 +47,7 @@ import org.junit.experimental.categories.Category; import java.io.IOException; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -106,6 +110,107 @@ public void whenShuttingDown() throws IOException { TEST_NUM.incrementAndGet(); } + @Test + public void shouldChooseMaxNumberOfPartitionedBetweenTwoRepartitionOperationsWhenJoining() throws ExecutionException, InterruptedException { + final String topicB = "topic-b-" + TEST_NUM.get(); + final String topicBRepartitionedName = "topic-b-repartitioned"; + final String inputTopicRepartitionedName = "input-topic-repartition"; + + final long timestamp = System.currentTimeMillis(); + + CLUSTER.createTopic(topicB, 1, 1); + + final List> expectedRecords = Arrays.asList( + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") + ); + + sendEvents(timestamp, expectedRecords); + sendEvents(topicB, timestamp, expectedRecords); + + final StreamsBuilder builder = new StreamsBuilder(); + + final Repartitioned inputTopicRepartitioned = Repartitioned + .as(inputTopicRepartitionedName) + .withNumberOfPartitions(4); + + final Repartitioned topicBRepartitioned = Repartitioned + .as(topicBRepartitionedName) + .withNumberOfPartitions(2); + + final KStream topicBStream = builder + .stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition(topicBRepartitioned); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition(inputTopicRepartitioned) + .join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))) + .to(outputTopic); + + startStreams(builder); + + assertEquals(4, getNumberOfPartitionsForTopic(toRepartitionTopicName(topicBRepartitionedName))); + assertEquals(4, getNumberOfPartitionsForTopic(toRepartitionTopicName(inputTopicRepartitionedName))); + + validateReceivedMessages( + new IntegerDeserializer(), + new StringDeserializer(), + expectedRecords + ); + } + + @Test + public void shouldChooseMaxPartitionNumberFromSourceTopicsForJoinOperation() throws ExecutionException, InterruptedException { + final String topicBMapperName = "topic-b-mapper"; + final String topicB = "topic-b-" + TEST_NUM.get(); + final int topicBNumberOfPartitions = 6; + final String inputTopicRepartitionName = "join-repartition-test"; + + final long timestamp = System.currentTimeMillis(); + + CLUSTER.createTopic(topicB, topicBNumberOfPartitions, 1); + + final List> expectedRecords = Arrays.asList( + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") + ); + + sendEvents(inputTopic, timestamp, expectedRecords); + sendEvents(topicB, timestamp, expectedRecords); + + final StreamsBuilder builder = new StreamsBuilder(); + + final Repartitioned inputTopicRepartitioned = Repartitioned + .as(inputTopicRepartitionName) + .withNumberOfPartitions(2); + + final KStream topicBStream = builder + .stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())) + .map(KeyValue::new, Named.as(topicBMapperName)); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition(inputTopicRepartitioned) + .join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))) + .to(outputTopic); + + startStreams(builder); + + final String repartitionTopicName = toRepartitionTopicName(inputTopicRepartitionName); + final String mapperRepartitionTopicName = toRepartitionTopicName(topicBMapperName); + + final int mapperNumOfPartitions = getNumberOfPartitionsForTopic(mapperRepartitionTopicName); + final int repartitionTopicNumOfPartitions = getNumberOfPartitionsForTopic(repartitionTopicName); + + validateReceivedMessages( + new IntegerDeserializer(), + new StringDeserializer(), + expectedRecords + ); + + assertEquals(topicBNumberOfPartitions, mapperNumOfPartitions); + assertEquals(topicBNumberOfPartitions, repartitionTopicNumOfPartitions); + } + @Test public void shouldUseStreamPartitionerForRepartitionOperation() throws ExecutionException, InterruptedException { final int partition = 1; @@ -530,8 +635,14 @@ private static int countOccurrencesInTopology(final String topologyString, private void sendEvents(final long timestamp, final List> events) throws ExecutionException, InterruptedException { + sendEvents(inputTopic, timestamp, events); + } + + private void sendEvents(final String topic, + final long timestamp, + final List> events) throws ExecutionException, InterruptedException { IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp( - inputTopic, + topic, events, TestUtils.producerConfig( CLUSTER.bootstrapServers(), diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java index 0d21f469e1135..d102244ea3791 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java @@ -42,6 +42,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.regex.Pattern; @@ -887,7 +888,7 @@ public void shouldHaveCorrectInternalTopicConfigWhenInternalTopicPropertiesArePr repartitionSourceTopics.get("Z-topic-1z"), new RepartitionTopicConfig( "Z-topic-1z", - numberOfPartitions, + Optional.of(numberOfPartitions), Collections.emptyMap() ) ); From 2ccfa1ed853dc423872ddb0e22b6ac8953f2a253 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 29 Sep 2019 01:48:48 +0300 Subject: [PATCH 40/68] KAFKA-8611 / Make sure co-partitioning won't break during optimization --- .../internals/InternalStreamsBuilder.java | 5 ++ .../internals/InternalTopologyBuilder.java | 12 ++++- .../KStreamRepartitionIntegrationTest.java | 54 +++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java index d760ffd5707d6..6acfd61ae4058 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java @@ -368,6 +368,11 @@ private void maybeOptimizeRepartitionOperations() { } repartitionNodeToBeReplaced.clearChildren(); + // if replaced repartition node is part of any copartition group, + // we need to update it with the new node name so that co-partitioning won't break. + internalTopologyBuilder.maybeUpdateCopartitionSourceGroups(repartitionNodeToBeReplaced.nodeName(), + optimizedSingleRepartition.nodeName()); + LOG.debug("Updated node {} children {}", optimizedSingleRepartition, optimizedSingleRepartition.children()); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index c9dfda5fa21bc..01eac984e784b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -616,7 +616,17 @@ public final void addInternalTopic(final String topicName, } public final void copartitionSources(final Collection sourceNodes) { - copartitionSourceGroups.add(Collections.unmodifiableSet(new HashSet<>(sourceNodes))); + copartitionSourceGroups.add(new HashSet<>(sourceNodes)); + } + + public synchronized void maybeUpdateCopartitionSourceGroups(final String replacedNodeName, + final String optimizedNodeName) { + for (Set copartitionSourceGroup : copartitionSourceGroups) { + if (copartitionSourceGroup.contains(replacedNodeName)) { + copartitionSourceGroup.remove(replacedNodeName); + copartitionSourceGroup.add(optimizedNodeName); + } + } } private void validateGlobalStoreArguments(final String sourceName, diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index ef557679f9df0..d8277a0e314cf 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -211,6 +211,60 @@ public void shouldChooseMaxPartitionNumberFromSourceTopicsForJoinOperation() thr assertEquals(topicBNumberOfPartitions, repartitionTopicNumOfPartitions); } + @Test + public void shouldChooseMaxPartitionNumberFromSourceTopicsForJoinOperationWhenTopologyOptimizationIsSpecified() throws ExecutionException, InterruptedException { + streamsConfiguration.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.OPTIMIZE); + + final String topicBMapperName = "topic-b-mapper"; + final String topicB = "topic-b-" + TEST_NUM.get(); + final int topicBNumberOfPartitions = 6; + final String inputTopicRepartitionName = "join-repartition-test"; + + final long timestamp = System.currentTimeMillis(); + + CLUSTER.createTopic(topicB, topicBNumberOfPartitions, 1); + + final List> expectedRecords = Arrays.asList( + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") + ); + + sendEvents(inputTopic, timestamp, expectedRecords); + sendEvents(topicB, timestamp, expectedRecords); + + final StreamsBuilder builder = new StreamsBuilder(); + + final Repartitioned inputTopicRepartitioned = Repartitioned + .as(inputTopicRepartitionName) + .withNumberOfPartitions(2); + + final KStream topicBStream = builder + .stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())) + .map(KeyValue::new, Named.as(topicBMapperName)); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition(inputTopicRepartitioned) + .join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))) + .to(outputTopic); + + startStreams(builder); + + final String repartitionTopicName = toRepartitionTopicName(inputTopicRepartitionName); + final String mapperRepartitionTopicName = toRepartitionTopicName(topicBMapperName); + + final int mapperNumOfPartitions = getNumberOfPartitionsForTopic(mapperRepartitionTopicName); + final int repartitionTopicNumOfPartitions = getNumberOfPartitionsForTopic(repartitionTopicName); + + validateReceivedMessages( + new IntegerDeserializer(), + new StringDeserializer(), + expectedRecords + ); + + assertEquals(topicBNumberOfPartitions, mapperNumOfPartitions); + assertEquals(topicBNumberOfPartitions, repartitionTopicNumOfPartitions); + } + @Test public void shouldUseStreamPartitionerForRepartitionOperation() throws ExecutionException, InterruptedException { final int partition = 1; From 47e338b950cac4c060d6823c0a86e5b8118f1bfc Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 29 Sep 2019 01:51:44 +0300 Subject: [PATCH 41/68] KAFKA-8611 / Add logs --- .../streams/processor/internals/InternalTopologyBuilder.java | 1 + 1 file changed, 1 insertion(+) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index 01eac984e784b..7f6ac23decac4 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -625,6 +625,7 @@ public synchronized void maybeUpdateCopartitionSourceGroups(final String replace if (copartitionSourceGroup.contains(replacedNodeName)) { copartitionSourceGroup.remove(replacedNodeName); copartitionSourceGroup.add(optimizedNodeName); + log.debug("replaced node '{}' with '{}' in copartition source group", replacedNodeName, optimizedNodeName); } } } From 1bf7a158161aab56ab5710c11314a001fe9a52e4 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 29 Sep 2019 02:28:58 +0300 Subject: [PATCH 42/68] KAFKA-8611 / Add unit test for copartition groups --- .../internals/InternalTopologyBuilderTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java index d102244ea3791..2ec36af3a0c78 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java @@ -35,6 +35,7 @@ import org.junit.Test; import java.lang.reflect.Field; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -928,4 +929,19 @@ public void shouldHaveCorrectInternalTopicConfigWhenInternalTopicPropertiesAreNo new RepartitionTopicConfig("Y-topic-1y", Collections.emptyMap()) ); } + + @Test + public void shouldUpdateCopartitionGroups() { + builder.copartitionSources(mkSet("SOURCE-NODE-1", "SOURCE-NODE-2", "SOURCE-NODE-3")); + + builder.addSource(null, "SOURCE-NODE-1", null, null, null, "topic-1"); + builder.addSource(null, "OPTIMIZED-SOURCE-NODE-2", null, null, null, "topic-2"); + builder.addSource(null, "SOURCE-NODE-3", null, null, null, "topic-3"); + + builder.maybeUpdateCopartitionSourceGroups("SOURCE-NODE-2", "OPTIMIZED-SOURCE-NODE-2"); + + final Collection> copartitionGroups = builder.copartitionGroups(); + + assertEquals(copartitionGroups, Collections.singletonList(mkSet("topic-1", "topic-2", "topic-3"))); + } } From 8a9d70c8d558319c5a2666e50980dd4faae6fc9d Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 29 Sep 2019 22:31:41 +0300 Subject: [PATCH 43/68] KAFKA-8611 / fix checkstyle --- .../streams/processor/internals/InternalTopologyBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index 7f6ac23decac4..6c64b0ce01204 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -621,7 +621,7 @@ public final void copartitionSources(final Collection sourceNodes) { public synchronized void maybeUpdateCopartitionSourceGroups(final String replacedNodeName, final String optimizedNodeName) { - for (Set copartitionSourceGroup : copartitionSourceGroups) { + for (final Set copartitionSourceGroup : copartitionSourceGroups) { if (copartitionSourceGroup.contains(replacedNodeName)) { copartitionSourceGroup.remove(replacedNodeName); copartitionSourceGroup.add(optimizedNodeName); From b205af839b5d8d268d54cbd105fab8a40d4e810c Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 29 Sep 2019 22:48:00 +0300 Subject: [PATCH 44/68] KAFKA-8611 / fix checkstyle test --- .../streams/processor/internals/InternalTopologyBuilderTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java index 2ec36af3a0c78..1985cde8e3d1b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java @@ -35,7 +35,6 @@ import org.junit.Test; import java.lang.reflect.Field; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; From 0fb703f3e3e20366ad6d6d8e01a2a35a6900db66 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Fri, 4 Oct 2019 10:38:19 +0300 Subject: [PATCH 45/68] KAFKA-8611 / sync with trunk --- .../apache/kafka/streams/kstream/internals/KTableImpl.java | 5 +++-- .../graph/KTableKTableForeignKeyJoinResolutionNode.java | 3 ++- .../kstream/internals/graph/OptimizableRepartitionNode.java | 3 ++- .../internals/graph/UnoptimizableRepartitionNode.java | 6 ++---- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java index 05e04e8c2912d..f86cfdb7c5005 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java @@ -59,6 +59,7 @@ import org.apache.kafka.streams.kstream.internals.suppress.NamedSuppressed; import org.apache.kafka.streams.kstream.internals.suppress.SuppressedInternal; import org.apache.kafka.streams.processor.ProcessorSupplier; +import org.apache.kafka.streams.processor.internals.InternalTopicProperties; import org.apache.kafka.streams.processor.internals.StaticTopicNameExtractor; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.StoreBuilder; @@ -901,7 +902,7 @@ private KTable doJoinOnForeignKey(final KTable forei final NamedInternal renamed = new NamedInternal(joinName); final String subscriptionTopicName = renamed.suffixWithOrElseGet("-subscription-registration", builder, SUBSCRIPTION_REGISTRATION) + TOPIC_SUFFIX; - builder.internalTopologyBuilder.addInternalTopic(subscriptionTopicName); + builder.internalTopologyBuilder.addInternalTopic(subscriptionTopicName, InternalTopicProperties.empty()); final CombinedKeySchema combinedKeySchema = new CombinedKeySchema<>(subscriptionTopicName, foreignKeySerde, keySerde); final ProcessorGraphNode> subscriptionNode = new ProcessorGraphNode<>( @@ -987,7 +988,7 @@ private KTable doJoinOnForeignKey(final KTable forei final String finalRepartitionTopicName = renamed.suffixWithOrElseGet("-subscription-response", builder, SUBSCRIPTION_RESPONSE) + TOPIC_SUFFIX; - builder.internalTopologyBuilder.addInternalTopic(finalRepartitionTopicName); + builder.internalTopologyBuilder.addInternalTopic(finalRepartitionTopicName, InternalTopicProperties.empty()); final StreamSinkNode> foreignResponseSink = new StreamSinkNode<>( diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/KTableKTableForeignKeyJoinResolutionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/KTableKTableForeignKeyJoinResolutionNode.java index 672625cfc1cd2..90fc177c06e11 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/KTableKTableForeignKeyJoinResolutionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/KTableKTableForeignKeyJoinResolutionNode.java @@ -23,6 +23,7 @@ import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionResponseWrapper; import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper; import org.apache.kafka.streams.processor.FailOnInvalidTimestamp; +import org.apache.kafka.streams.processor.internals.InternalTopicProperties; import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; /** @@ -64,7 +65,7 @@ public KTableKTableForeignKeyJoinResolutionNode(final String nodeName, @Override public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { - topologyBuilder.addInternalTopic(finalRepartitionTopicName); + topologyBuilder.addInternalTopic(finalRepartitionTopicName, InternalTopicProperties.empty()); //Repartition back to the original partitioning structure topologyBuilder.addSink(finalRepartitionSinkName, finalRepartitionTopicName, keySerde.serializer(), subResponseSerde.serializer(), diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index 9c8ff046e31f2..0dffbfc9178f5 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -31,7 +31,8 @@ private OptimizableRepartitionNode(final String nodeName, final Serde keySerde, final Serde valueSerde, final String sinkName, - final String repartitionTopic) { + final String repartitionTopic, + final StreamPartitioner partitioner) { super( nodeName, sourceName, diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java index 4bb8038812cac..38a61dabaed0e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java @@ -26,7 +26,6 @@ * Repartition node that is not subject of optimization algorithm */ public class UnoptimizableRepartitionNode extends BaseRepartitionNode { - private final StreamPartitioner partitioner; private final InternalTopicProperties internalTopicProperties; private UnoptimizableRepartitionNode(final String nodeName, @@ -45,10 +44,9 @@ private UnoptimizableRepartitionNode(final String nodeName, keySerde, valueSerde, sinkName, - repartitionTopic + repartitionTopic, + partitioner ); - - this.partitioner = partitioner; this.internalTopicProperties = internalTopicProperties; } From 3180780be9e40a3a594c7c5bd9152a1629ac682e Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Wed, 9 Oct 2019 10:18:26 +0300 Subject: [PATCH 46/68] KAFKA-8611 / fix compileTestJava --- .../internals/KStreamRepartitionTest.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java index a8273207b2cd2..6a2a37613b6fd 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java @@ -16,14 +16,16 @@ */ package org.apache.kafka.streams.kstream.internals; +import org.apache.kafka.common.serialization.IntegerSerializer; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.streams.KeyValueTimestamp; import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.TestInputTopic; import org.apache.kafka.streams.TopologyTestDriver; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.Repartitioned; import org.apache.kafka.streams.processor.StreamPartitioner; -import org.apache.kafka.streams.test.ConsumerRecordFactory; import org.apache.kafka.test.MockProcessor; import org.apache.kafka.test.MockProcessorSupplier; import org.apache.kafka.test.StreamsTestUtils; @@ -47,11 +49,6 @@ public class KStreamRepartitionTest { private final String inputTopic = "input-topic"; private final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String()); - private final ConsumerRecordFactory recordFactory = new ConsumerRecordFactory<>( - Serdes.Integer().serializer(), - Serdes.String().serializer(), - 0L - ); private StreamsBuilder builder; @@ -81,8 +78,14 @@ public void shouldInvokePartitionerWhenSet() { repartitionedStream.process(supplier); try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { + final TestInputTopic testInputTopic = driver.createInputTopic( + inputTopic, + new IntegerSerializer(), + new StringSerializer() + ); + for (int i = 0; i < 2; i++) { - driver.pipeInput(recordFactory.create(inputTopic, expectedKeys[i], "X" + expectedKeys[i], i + 10)); + testInputTopic.pipeInput(expectedKeys[i], "X" + expectedKeys[i], i + 10); } final MockProcessor proc = supplier.theCapturedProcessor(); proc.checkAndClearProcessResult( From 88886b785acbc88a38af0bf40bccd69fa78d87a2 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 20 Oct 2019 13:32:21 +0300 Subject: [PATCH 47/68] KAFKA-8611 / Make copartitionSources(Collection) method synchronized --- .../streams/processor/internals/InternalTopologyBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index ba04dd1cd7918..7b97b096769a3 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -616,7 +616,7 @@ public final void addInternalTopic(final String topicName, internalTopicNamesWithProperties.put(topicName, internalTopicProperties); } - public final void copartitionSources(final Collection sourceNodes) { + public final synchronized void copartitionSources(final Collection sourceNodes) { copartitionSourceGroups.add(new HashSet<>(sourceNodes)); } From 83f264f9ed3705d91941b48a4844685af13fd304 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 20 Oct 2019 13:41:22 +0300 Subject: [PATCH 48/68] KAFKA-8611 / Rename maybeUpdateCopartitionSourceGroups method parameters: replacedNodeName -> oldNodeName, optimizedNodeName -> newNodeName --- .../processor/internals/InternalTopologyBuilder.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index 7b97b096769a3..4950fda2d7199 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -620,13 +620,13 @@ public final synchronized void copartitionSources(final Collection sourc copartitionSourceGroups.add(new HashSet<>(sourceNodes)); } - public synchronized void maybeUpdateCopartitionSourceGroups(final String replacedNodeName, - final String optimizedNodeName) { + public final synchronized void maybeUpdateCopartitionSourceGroups(final String oldNodeName, + final String newNodeName) { for (final Set copartitionSourceGroup : copartitionSourceGroups) { - if (copartitionSourceGroup.contains(replacedNodeName)) { - copartitionSourceGroup.remove(replacedNodeName); - copartitionSourceGroup.add(optimizedNodeName); - log.debug("replaced node '{}' with '{}' in copartition source group", replacedNodeName, optimizedNodeName); + if (copartitionSourceGroup.contains(oldNodeName)) { + copartitionSourceGroup.remove(oldNodeName); + copartitionSourceGroup.add(newNodeName); + log.debug("replaced node '{}' with '{}' in copartition source group", oldNodeName, newNodeName); } } } From 3b25b49ae2047a8480f67d8199d756dae4924311 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Wed, 13 Nov 2019 02:02:54 +0200 Subject: [PATCH 49/68] KAFKA-8611 / use BaseRepartitionNode#internalTopicProperties from its implementations --- .../internals/graph/GroupedTableOperationRepartitionNode.java | 2 +- .../kstream/internals/graph/OptimizableRepartitionNode.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java index 9bc7ed6c5a034..ce8ec1f958d48 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java @@ -80,7 +80,7 @@ public String toString() { @Override public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { - topologyBuilder.addInternalTopic(repartitionTopic, InternalTopicProperties.empty()); + topologyBuilder.addInternalTopic(repartitionTopic, internalTopicProperties); topologyBuilder.addSink( sinkName, diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index bf408d217d2f2..0b05f9e64767f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -66,7 +66,7 @@ public String toString() { @Override public void writeToTopology(final InternalTopologyBuilder topologyBuilder) { - topologyBuilder.addInternalTopic(repartitionTopic, InternalTopicProperties.empty()); + topologyBuilder.addInternalTopic(repartitionTopic, internalTopicProperties); topologyBuilder.addProcessor( processorParameters.processorName(), From d35bb44e29200366ece34b24f16ae25a4ddfb105 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Fri, 15 Nov 2019 18:11:10 +0200 Subject: [PATCH 50/68] KAFKA-8611 / throw TopologyException when number of partitions during join operation do not match --- .../internals/InternalStreamsBuilder.java | 5 - .../ImmutableRepartitionTopicConfig.java | 74 ++++++ .../internals/InternalTopicConfig.java | 14 +- .../internals/InternalTopologyBuilder.java | 30 ++- .../internals/RepartitionTopicConfig.java | 13 +- .../CopartitionedTopicsEnforcer.java | 64 ++++- .../KStreamRepartitionIntegrationTest.java | 238 +++++++++++------- .../CopartitionedTopicsEnforcerTest.java | 79 +++++- .../InternalTopologyBuilderTest.java | 20 +- 9 files changed, 380 insertions(+), 157 deletions(-) create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/ImmutableRepartitionTopicConfig.java diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java index a0be7ce863727..aedeb9c7d8b79 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java @@ -382,11 +382,6 @@ private void maybeOptimizeRepartitionOperations() { } repartitionNodeToBeReplaced.clearChildren(); - // if replaced repartition node is part of any copartition group, - // we need to update it with the new node name so that co-partitioning won't break. - internalTopologyBuilder.maybeUpdateCopartitionSourceGroups(repartitionNodeToBeReplaced.nodeName(), - optimizedSingleRepartition.nodeName()); - LOG.debug("Updated node {} children {}", optimizedSingleRepartition, optimizedSingleRepartition.children()); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ImmutableRepartitionTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ImmutableRepartitionTopicConfig.java new file mode 100644 index 0000000000000..14127c75538dd --- /dev/null +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ImmutableRepartitionTopicConfig.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.processor.internals; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * repartition topic config which doesn't permit changing number of partitions + */ +public class ImmutableRepartitionTopicConfig extends RepartitionTopicConfig { + private final int numberOfPartitions; + + ImmutableRepartitionTopicConfig(final String name, + final int numberOfPartitions, + final Map topicConfigs) { + super(name, topicConfigs); + validateNumberOfPartitions(numberOfPartitions); + this.numberOfPartitions = numberOfPartitions; + } + + @Override + public Optional numberOfPartitions() { + return Optional.of(numberOfPartitions); + } + + @Override + public void setNumberOfPartitions(final int numberOfPartitions) { + //no-op + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final ImmutableRepartitionTopicConfig that = (ImmutableRepartitionTopicConfig) o; + return Objects.equals(name, that.name) && + Objects.equals(topicConfigs, that.topicConfigs) && + Objects.equals(numberOfPartitions, that.numberOfPartitions); + } + + @Override + public int hashCode() { + return Objects.hash(name, topicConfigs, numberOfPartitions); + } + + @Override + public String toString() { + return "ImmutableRepartitionTopicConfig(" + + "name=" + name + + ", topicConfigs=" + topicConfigs + + ", numberOfPartitions=" + numberOfPartitions + + ")"; + } +} diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java index 1e20cbb5b9f33..35151a338a224 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java @@ -30,21 +30,15 @@ public abstract class InternalTopicConfig { final String name; final Map topicConfigs; - Optional numberOfPartitions; + private Optional numberOfPartitions = Optional.empty(); InternalTopicConfig(final String name, - final Optional numberOfPartitions, final Map topicConfigs) { Objects.requireNonNull(name, "name can't be null"); Topic.validate(name); this.name = name; this.topicConfigs = topicConfigs; - this.numberOfPartitions = numberOfPartitions; - } - - InternalTopicConfig(final String name, final Map topicConfigs) { - this(name, Optional.empty(), topicConfigs); } /** @@ -65,10 +59,14 @@ public Optional numberOfPartitions() { } public void setNumberOfPartitions(final int numberOfPartitions) { + validateNumberOfPartitions(numberOfPartitions); + this.numberOfPartitions = Optional.of(numberOfPartitions); + } + + protected void validateNumberOfPartitions(final int numberOfPartitions) { if (numberOfPartitions < 1) { throw new IllegalArgumentException("Number of partitions must be at least 1."); } - this.numberOfPartitions = Optional.of(numberOfPartitions); } @Override diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index 14d32e0204210..31e6317f76b3d 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -46,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.regex.Pattern; @@ -615,19 +616,8 @@ public final void addInternalTopic(final String topicName, internalTopicNamesWithProperties.put(topicName, internalTopicProperties); } - public final synchronized void copartitionSources(final Collection sourceNodes) { - copartitionSourceGroups.add(new HashSet<>(sourceNodes)); - } - - public final synchronized void maybeUpdateCopartitionSourceGroups(final String oldNodeName, - final String newNodeName) { - for (final Set copartitionSourceGroup : copartitionSourceGroups) { - if (copartitionSourceGroup.contains(oldNodeName)) { - copartitionSourceGroup.remove(oldNodeName); - copartitionSourceGroup.add(newNodeName); - log.debug("replaced node '{}' with '{}' in copartition source group", oldNodeName, newNodeName); - } - } + public final void copartitionSources(final Collection sourceNodes) { + copartitionSourceGroups.add(Collections.unmodifiableSet(new HashSet<>(sourceNodes))); } private void validateGlobalStoreArguments(final String sourceName, @@ -1014,10 +1004,9 @@ public synchronized Map topicGroups() { // prefix the internal topic name with the application id final String internalTopic = decorateTopic(topic); - final RepartitionTopicConfig repartitionTopicConfig = new RepartitionTopicConfig( + final RepartitionTopicConfig repartitionTopicConfig = buildRepartitionTopicConfig( internalTopic, - internalTopicNamesWithProperties.get(topic).getNumberOfPartitions(), - Collections.emptyMap() + internalTopicNamesWithProperties.get(topic).getNumberOfPartitions() ); repartitionTopics.put(repartitionTopicConfig.name(), repartitionTopicConfig); @@ -1066,6 +1055,15 @@ public synchronized Map topicGroups() { return Collections.unmodifiableMap(topicGroups); } + private RepartitionTopicConfig buildRepartitionTopicConfig(final String internalTopic, + final Optional numberOfPartitions) { + if (numberOfPartitions.isPresent()) { + return new ImmutableRepartitionTopicConfig(internalTopic, numberOfPartitions.get(), Collections.emptyMap()); + } else { + return new RepartitionTopicConfig(internalTopic, Collections.emptyMap()); + } + } + private void setRegexMatchedTopicsToSourceNodes() { if (subscriptionUpdates.hasUpdates()) { for (final Map.Entry stringPatternEntry : nodeToSourcePatterns.entrySet()) { diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java index 14495891d65b6..7161a3fc88030 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java @@ -22,7 +22,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.Optional; /** * RepartitionTopicConfig captures the properties required for configuring @@ -43,12 +42,6 @@ public class RepartitionTopicConfig extends InternalTopicConfig { super(name, topicConfigs); } - RepartitionTopicConfig(final String name, - final Optional numberOfPartitions, - final Map topicConfigs) { - super(name, numberOfPartitions, topicConfigs); - } - /** * Get the configured properties for this topic. If retentionMs is set then * we add additionalRetentionMs to work out the desired retention when cleanup.policy=compact,delete @@ -77,13 +70,12 @@ public boolean equals(final Object o) { } final RepartitionTopicConfig that = (RepartitionTopicConfig) o; return Objects.equals(name, that.name) && - Objects.equals(topicConfigs, that.topicConfigs) && - Objects.equals(numberOfPartitions, that.numberOfPartitions); + Objects.equals(topicConfigs, that.topicConfigs); } @Override public int hashCode() { - return Objects.hash(name, topicConfigs, numberOfPartitions); + return Objects.hash(name, topicConfigs); } @Override @@ -91,7 +83,6 @@ public String toString() { return "RepartitionTopicConfig(" + "name=" + name + ", topicConfigs=" + topicConfigs + - ", numberOfPartitions=" + numberOfPartitions + ")"; } } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java index c277cc6f7cc2c..e110c58c24e3f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java @@ -19,9 +19,11 @@ import org.apache.kafka.common.Cluster; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.streams.errors.TopologyException; +import org.apache.kafka.streams.processor.internals.ImmutableRepartitionTopicConfig; import org.apache.kafka.streams.processor.internals.InternalTopicConfig; import org.slf4j.Logger; +import java.util.Collection; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -64,20 +66,70 @@ public void enforce(final Set copartitionGroup, })); final int numPartitionsToUseForRepartitionTopics; + final Collection internalTopicConfigs = repartitionTopicConfigs.values(); + if (copartitionGroup.equals(repartitionTopicConfigs.keySet())) { - // If all topics for this co-partition group is repartition topics, - // then set the number of partitions to be the maximum of the number of partitions. - numPartitionsToUseForRepartitionTopics = getMaxPartitions(repartitionTopicConfigs); + if (internalTopicConfigs.stream().anyMatch(this::isMutableRepartitionTopicConfig)) { + // If all topics for this co-partition group is repartition topics, + // then set the number of partitions to be the maximum of the number of partitions. + numPartitionsToUseForRepartitionTopics = getMaxPartitions(repartitionTopicConfigs); + } else { + // if all repartition topics are immutable repartition topics, + // we must ensure that they all have same number of partitions. + numPartitionsToUseForRepartitionTopics = validateAndGetNumOfPartitionsOfImmutableTopics(repartitionTopicConfigs, + internalTopicConfigs); + } } else { // Otherwise, use the number of partitions from external topics (which must all be the same) numPartitionsToUseForRepartitionTopics = getSamePartitions(nonRepartitionTopicPartitions); - } // coerce all the repartition topics to use the decided number of partitions. - for (final InternalTopicConfig config : repartitionTopicConfigs.values()) { - config.setNumberOfPartitions(numPartitionsToUseForRepartitionTopics); + for (final InternalTopicConfig config : internalTopicConfigs) { + if (isMutableRepartitionTopicConfig(config)) { + config.setNumberOfPartitions(numPartitionsToUseForRepartitionTopics); + } else { + final int numberOfPartitionsOfInternalTopic = config.numberOfPartitions().get(); + + if (numberOfPartitionsOfInternalTopic != numPartitionsToUseForRepartitionTopics) { + final String msg = String.format("%sNumber of partitions [%s] of repartition topic [%s] " + + "doesn't match number of partitions [%s] of the source topic.", + logPrefix, + numberOfPartitionsOfInternalTopic, + config.name(), + numPartitionsToUseForRepartitionTopics); + throw new TopologyException(msg); + } + } + } + } + + private int validateAndGetNumOfPartitionsOfImmutableTopics(final Map repartitionTopicConfigs, + final Collection internalTopicConfigs) { + final int numberOfPartitionsOfInternalTopic = internalTopicConfigs.iterator() + .next() + .numberOfPartitions() + .get(); + + for (final InternalTopicConfig internalTopicConfig : internalTopicConfigs) { + if (internalTopicConfig.numberOfPartitions().get() != numberOfPartitionsOfInternalTopic) { + final Map repartitionTopics = repartitionTopicConfigs + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().numberOfPartitions().get())); + + final String msg = String.format("%sFollowing topics do not have the same number of partitions: [%s]", + logPrefix, + new TreeMap<>(repartitionTopics)); + throw new TopologyException(msg); + } } + + return numberOfPartitionsOfInternalTopic; + } + + private boolean isMutableRepartitionTopicConfig(final InternalTopicConfig internalTopicConfig) { + return !(internalTopicConfig instanceof ImmutableRepartitionTopicConfig); } private int getSamePartitions(final Map nonRepartitionTopicsInCopartitionGroup) { diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 89ee9b8130c8e..49e721f462f6b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -28,6 +28,7 @@ import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.KafkaStreams.State; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; @@ -47,24 +48,31 @@ import org.junit.experimental.categories.Category; import java.io.IOException; +import java.lang.Thread.UncaughtExceptionHandler; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Properties; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.kafka.streams.KafkaStreams.State.ERROR; import static org.apache.kafka.streams.KafkaStreams.State.REBALANCING; import static org.apache.kafka.streams.KafkaStreams.State.RUNNING; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @Category({IntegrationTest.class}) @@ -97,6 +105,7 @@ public void before() throws InterruptedException { streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100); streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass()); streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); + streamsConfiguration.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.OPTIMIZE); } @After @@ -111,110 +120,139 @@ public void whenShuttingDown() throws IOException { } @Test - public void shouldChooseMaxNumberOfPartitionedBetweenTwoRepartitionOperationsWhenJoining() throws ExecutionException, InterruptedException { + public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationsDoNotMatchWhenJoining() throws InterruptedException { final String topicB = "topic-b-" + TEST_NUM.get(); final String topicBRepartitionedName = "topic-b-scale-up"; final String inputTopicRepartitionedName = "input-topic-scale-up"; - - final long timestamp = System.currentTimeMillis(); + final int topicBNumberOfPartitions = 2; + final int inputTopicNumberOfPartitions = 4; + final AtomicReference throwable = new AtomicReference<>(); CLUSTER.createTopic(topicB, 1, 1); - final List> expectedRecords = Arrays.asList( - new KeyValue<>(1, "A"), - new KeyValue<>(2, "B") - ); - - sendEvents(timestamp, expectedRecords); - sendEvents(topicB, timestamp, expectedRecords); - final StreamsBuilder builder = new StreamsBuilder(); final Repartitioned inputTopicRepartitioned = Repartitioned .as(inputTopicRepartitionedName) - .withNumberOfPartitions(4); + .withNumberOfPartitions(inputTopicNumberOfPartitions); final Repartitioned topicBRepartitioned = Repartitioned .as(topicBRepartitionedName) - .withNumberOfPartitions(2); + .withNumberOfPartitions(topicBNumberOfPartitions); final KStream topicBStream = builder .stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())) .repartition(topicBRepartitioned); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition(inputTopicRepartitioned) - .join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))) - .to(outputTopic); + .repartition(inputTopicRepartitioned) + .join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))) + .to(outputTopic); - startStreams(builder); + startStreams(builder, REBALANCING, ERROR, (t, e) -> throwable.set(e)); - assertEquals(4, getNumberOfPartitionsForTopic(toRepartitionTopicName(topicBRepartitionedName))); - assertEquals(4, getNumberOfPartitionsForTopic(toRepartitionTopicName(inputTopicRepartitionedName))); + final Map repartitionTopicsWithNumberOfPartitions = new HashMap<>(); + repartitionTopicsWithNumberOfPartitions.put(toRepartitionTopicName(topicBRepartitionedName), topicBNumberOfPartitions); + repartitionTopicsWithNumberOfPartitions.put(toRepartitionTopicName(inputTopicRepartitionedName), inputTopicNumberOfPartitions); - validateReceivedMessages( - new IntegerDeserializer(), - new StringDeserializer(), - expectedRecords - ); + final String expectedErrorMessage = String.format("Following topics do not have the same " + + "number of partitions: [%s]", + new TreeMap<>(repartitionTopicsWithNumberOfPartitions)); + + assertNotNull(throwable.get()); + assertTrue(throwable.get().getMessage().contains(expectedErrorMessage)); } @Test - public void shouldChooseMaxPartitionNumberFromSourceTopicsForJoinOperation() throws ExecutionException, InterruptedException { + public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDoNotMatchSourceTopicWhenJoining() throws ExecutionException, InterruptedException { final String topicBMapperName = "topic-b-mapper"; final String topicB = "topic-b-" + TEST_NUM.get(); final int topicBNumberOfPartitions = 6; final String inputTopicRepartitionName = "join-repartition-test"; + final AtomicReference expectedThrowable = new AtomicReference<>(); + final int inputTopicRepartitionedNumOfPartitions = 2; + + CLUSTER.createTopic(topicB, topicBNumberOfPartitions, 1); + + final StreamsBuilder builder = new StreamsBuilder(); + + final Repartitioned inputTopicRepartitioned = Repartitioned + .as(inputTopicRepartitionName) + .withNumberOfPartitions(inputTopicRepartitionedNumOfPartitions); + + final KStream topicBStream = builder + .stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())) + .map(KeyValue::new, Named.as(topicBMapperName)); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition(inputTopicRepartitioned) + .join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))) + .to(outputTopic); + + builder.build(streamsConfiguration); + + startStreams(builder, REBALANCING, ERROR, (t, e) -> expectedThrowable.set(e)); + + final String expectedMsg = String.format("Number of partitions [%s] of repartition topic [%s] " + + "doesn't match number of partitions [%s] of the source topic.", + inputTopicRepartitionedNumOfPartitions, + toRepartitionTopicName(inputTopicRepartitionName), + topicBNumberOfPartitions); + assertNotNull(expectedThrowable.get()); + assertTrue(expectedThrowable.get().getMessage().contains(expectedMsg)); + } + + @Test + public void shouldDoProperJoiningWhenNumberOfPartitionsAreValidWhenUsingRepartitionOperation() throws ExecutionException, InterruptedException { + final String topicB = "topic-b-" + TEST_NUM.get(); + final String topicBRepartitionedName = "topic-b-scale-up"; + final String inputTopicRepartitionedName = "input-topic-scale-up"; final long timestamp = System.currentTimeMillis(); - CLUSTER.createTopic(topicB, topicBNumberOfPartitions, 1); + CLUSTER.createTopic(topicB, 1, 1); final List> expectedRecords = Arrays.asList( new KeyValue<>(1, "A"), new KeyValue<>(2, "B") ); - sendEvents(inputTopic, timestamp, expectedRecords); + sendEvents(timestamp, expectedRecords); sendEvents(topicB, timestamp, expectedRecords); final StreamsBuilder builder = new StreamsBuilder(); final Repartitioned inputTopicRepartitioned = Repartitioned - .as(inputTopicRepartitionName) - .withNumberOfPartitions(2); + .as(inputTopicRepartitionedName) + .withNumberOfPartitions(4); + + final Repartitioned topicBRepartitioned = Repartitioned + .as(topicBRepartitionedName) + .withNumberOfPartitions(4); final KStream topicBStream = builder .stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())) - .map(KeyValue::new, Named.as(topicBMapperName)); + .repartition(topicBRepartitioned); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition(inputTopicRepartitioned) - .join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))) - .to(outputTopic); + .repartition(inputTopicRepartitioned) + .join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))) + .to(outputTopic); startStreams(builder); - final String repartitionTopicName = toRepartitionTopicName(inputTopicRepartitionName); - final String mapperRepartitionTopicName = toRepartitionTopicName(topicBMapperName); - - final int mapperNumOfPartitions = getNumberOfPartitionsForTopic(mapperRepartitionTopicName); - final int repartitionTopicNumOfPartitions = getNumberOfPartitionsForTopic(repartitionTopicName); + assertEquals(4, getNumberOfPartitionsForTopic(toRepartitionTopicName(topicBRepartitionedName))); + assertEquals(4, getNumberOfPartitionsForTopic(toRepartitionTopicName(inputTopicRepartitionedName))); validateReceivedMessages( new IntegerDeserializer(), new StringDeserializer(), expectedRecords ); - - assertEquals(topicBNumberOfPartitions, mapperNumOfPartitions); - assertEquals(topicBNumberOfPartitions, repartitionTopicNumOfPartitions); } @Test - public void shouldChooseMaxPartitionNumberFromSourceTopicsForJoinOperationWhenTopologyOptimizationIsSpecified() throws ExecutionException, InterruptedException { - streamsConfiguration.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.OPTIMIZE); - + public void shouldDoJoiningWhenNumberOfPartitionsOfRepartitionOperationMatchesNumberOfPartitionsOfSourceTopic() throws ExecutionException, InterruptedException { final String topicBMapperName = "topic-b-mapper"; final String topicB = "topic-b-" + TEST_NUM.get(); final int topicBNumberOfPartitions = 6; @@ -236,16 +274,16 @@ public void shouldChooseMaxPartitionNumberFromSourceTopicsForJoinOperationWhenTo final Repartitioned inputTopicRepartitioned = Repartitioned .as(inputTopicRepartitionName) - .withNumberOfPartitions(2); + .withNumberOfPartitions(6); final KStream topicBStream = builder .stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())) .map(KeyValue::new, Named.as(topicBMapperName)); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition(inputTopicRepartitioned) - .join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))) - .to(outputTopic); + .repartition(inputTopicRepartitioned) + .join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))) + .to(outputTopic); startStreams(builder); @@ -289,8 +327,8 @@ public void shouldUseStreamPartitionerForRepartitionOperation() throws Execution }); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition(repartitioned) - .to(outputTopic); + .repartition(repartitioned) + .to(outputTopic); startStreams(builder); @@ -321,8 +359,8 @@ public void shouldPerformSelectKeyWithRepartitionOperation() throws ExecutionExc final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition((key, value) -> Integer.valueOf(value)) - .to(outputTopic); + .repartition((key, value) -> Integer.valueOf(value)) + .to(outputTopic); startStreams(builder); @@ -356,8 +394,8 @@ public void shouldCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition(Repartitioned.as(repartitionName)) - .to(outputTopic); + .repartition(Repartitioned.as(repartitionName)) + .to(outputTopic); startStreams(builder); @@ -395,11 +433,11 @@ public void shouldPerformKeySelectOperationWhenRepartitionOperationIsUsedWithKey .withKeySerde(Serdes.String()); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition((key, value) -> key.toString(), repartitioned) - .groupByKey() - .count() - .toStream() - .to(outputTopic); + .repartition((key, value) -> key.toString(), repartitioned) + .groupByKey() + .count() + .toStream() + .to(outputTopic); startStreams(builder); @@ -436,11 +474,11 @@ public void shouldCreateRepartitionTopicWithSpecifiedNumberOfPartitions() throws final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition(Repartitioned.as(repartitionName).withNumberOfPartitions(2)) - .groupByKey() - .count() - .toStream() - .to(outputTopic); + .repartition(Repartitioned.as(repartitionName).withNumberOfPartitions(2)) + .groupByKey() + .count() + .toStream() + .to(outputTopic); startStreams(builder); @@ -475,11 +513,11 @@ public void shouldInheritRepartitionTopicPartitionNumberFromUpstreamTopicWhenNum final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition(Repartitioned.as(repartitionName)) - .groupByKey() - .count() - .toStream() - .to(outputTopic); + .repartition(Repartitioned.as(repartitionName)) + .groupByKey() + .count() + .toStream() + .to(outputTopic); startStreams(builder); @@ -519,12 +557,12 @@ public void shouldCreateOnlyOneRepartitionTopicWhenRepartitionIsFollowedByGroupB .withNumberOfPartitions(1); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .selectKey((key, value) -> key.toString()) - .repartition(repartitioned) - .groupByKey() - .count() - .toStream() - .to(outputTopic); + .selectKey((key, value) -> key.toString()) + .repartition(repartitioned) + .groupByKey() + .count() + .toStream() + .to(outputTopic); startStreams(builder); @@ -558,9 +596,9 @@ public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws Execut final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .selectKey((key, value) -> key.toString()) - .repartition(Repartitioned.with(Serdes.String(), Serdes.String())) - .to(outputTopic); + .selectKey((key, value) -> key.toString()) + .repartition(Repartitioned.with(Serdes.String(), Serdes.String())) + .to(outputTopic); startStreams(builder); @@ -599,12 +637,12 @@ public void shouldGoThroughRebalancingCorrectly() throws ExecutionException, Int .withNumberOfPartitions(2); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .selectKey((key, value) -> key.toString()) - .repartition(repartitioned) - .groupByKey() - .count() - .toStream() - .to(outputTopic); + .selectKey((key, value) -> key.toString()) + .repartition(repartitioned) + .groupByKey() + .count() + .toStream() + .to(outputTopic); startStreams(builder); final KafkaStreams kafkaStreamsToClose = startStreams(builder); @@ -646,9 +684,9 @@ public void shouldGoThroughRebalancingCorrectly() throws ExecutionException, Int private int getNumberOfPartitionsForTopic(final String topic) throws ExecutionException, InterruptedException { try (final AdminClient adminClient = createAdminClient()) { final TopicDescription topicDescription = adminClient.describeTopics(Collections.singleton(topic)) - .values() - .get(topic) - .get(); + .values() + .get(topic) + .get(); return topicDescription.partitions().size(); } @@ -657,8 +695,8 @@ private int getNumberOfPartitionsForTopic(final String topic) throws ExecutionEx private boolean topicExists(final String topic) throws InterruptedException, ExecutionException { try (final AdminClient adminClient = createAdminClient()) { final Set topics = adminClient.listTopics() - .names() - .get(); + .names() + .get(); return topics.contains(topic); } @@ -709,11 +747,28 @@ private void sendEvents(final String topic, } private KafkaStreams startStreams(final StreamsBuilder builder) throws InterruptedException { - final CountDownLatch latch = new CountDownLatch(1); + return startStreams(builder, REBALANCING, RUNNING, null); + } + + private KafkaStreams startStreams(final StreamsBuilder builder, + final State expectedOldState, + final State expectedNewState, + final UncaughtExceptionHandler uncaughtExceptionHandler) throws InterruptedException { + final CountDownLatch latch; final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(streamsConfiguration), streamsConfiguration); + if (uncaughtExceptionHandler == null) { + latch = new CountDownLatch(1); + } else { + latch = new CountDownLatch(2); + kafkaStreams.setUncaughtExceptionHandler((t, e) -> { + uncaughtExceptionHandler.uncaughtException(t, e); + latch.countDown(); + }); + } + kafkaStreams.setStateListener((newState, oldState) -> { - if (REBALANCING == oldState && RUNNING == newState) { + if (expectedOldState == oldState && expectedNewState == newState) { latch.countDown(); } }); @@ -722,6 +777,7 @@ private KafkaStreams startStreams(final StreamsBuilder builder) throws Interrupt latch.await(IntegrationTestUtils.DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS); kafkaStreamsInstances.add(kafkaStreams); + return kafkaStreams; } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java index 87177f24f0b5c..60262c30a09bb 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java @@ -29,13 +29,16 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.TreeMap; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; public class CopartitionedTopicsEnforcerTest { - private final CopartitionedTopicsEnforcer validator = new CopartitionedTopicsEnforcer("thread"); + private final CopartitionedTopicsEnforcer validator = new CopartitionedTopicsEnforcer("thread "); private final Map partitions = new HashMap<>(); private final Cluster cluster = Cluster.empty(); @@ -106,8 +109,73 @@ public void shouldSetNumPartitionsToMaximumPartitionsWhenAllTopicsAreRepartition assertThat(three.numberOfPartitions(), equalTo(Optional.of(15))); } + @Test + public void shouldThrowAnExceptionIfImmutableRepartitionTopicsDoNotHaveSameNumberOfPartitions() { + final InternalTopicConfig topic1 = createImmutableTopicConfig("repartitioned-1", 10); + final InternalTopicConfig topic2 = createImmutableTopicConfig("repartitioned-2", 5); + + final TopologyException ex = assertThrows( + TopologyException.class, + () -> validator.enforce(Utils.mkSet(topic1.name(), topic2.name()), + Utils.mkMap(Utils.mkEntry(topic1.name(), topic1), + Utils.mkEntry(topic2.name(), topic2)), + cluster.withPartitions(partitions)) + ); + + final TreeMap sorted = new TreeMap<>( + Utils.mkMap(Utils.mkEntry(topic1.name(), topic1.numberOfPartitions().get()), + Utils.mkEntry(topic2.name(), topic2.numberOfPartitions().get())) + ); + + assertEquals(String.format("Invalid topology: thread " + + "Following topics do not have the same number of partitions: " + + "[%s]", sorted), ex.getMessage()); + } + + @Test + public void shouldNotThrowAnExceptionWhenNumberOfPartitionsOfImmutableTopicConfigsAreTheSame() { + final InternalTopicConfig topic1 = createImmutableTopicConfig("repartitioned-1", 10); + final InternalTopicConfig topic2 = createImmutableTopicConfig("repartitioned-2", 10); + + validator.enforce(Utils.mkSet(topic1.name(), topic2.name()), + Utils.mkMap(Utils.mkEntry(topic1.name(), topic1), + Utils.mkEntry(topic2.name(), topic2)), + cluster.withPartitions(partitions)); + + assertThat(topic1.numberOfPartitions(), equalTo(Optional.of(10))); + assertThat(topic2.numberOfPartitions(), equalTo(Optional.of(10))); + } + + @Test + public void shouldThrowAnExceptionWhenNumberOfPartitionsOfNonRepartitionTopicAndImmutableRepartitionTopicDoNotMatch() { + final InternalTopicConfig topic1 = createImmutableTopicConfig("repartitioned-1", 10); + + final TopologyException ex = assertThrows( + TopologyException.class, + () -> validator.enforce(Utils.mkSet(topic1.name(), "second"), + Utils.mkMap(Utils.mkEntry(topic1.name(), topic1)), + cluster.withPartitions(partitions)) + ); + + assertEquals(String.format("Invalid topology: thread Number of partitions [%s] " + + "of repartition topic [%s] " + + "doesn't match number of partitions [%s] of the source topic.", + topic1.numberOfPartitions().get(), topic1.name(), 2), ex.getMessage()); + } + + @Test + public void shouldNotThrowAnExceptionWhenNumberOfPartitionsOfNonRepartitionTopicAndImmutableRepartitionTopicMatch() { + final InternalTopicConfig topic1 = createImmutableTopicConfig("repartitioned-1", 2); + + validator.enforce(Utils.mkSet(topic1.name(), "second"), + Utils.mkMap(Utils.mkEntry(topic1.name(), topic1)), + cluster.withPartitions(partitions)); + + assertThat(topic1.numberOfPartitions(), equalTo(Optional.of(2))); + } + private InternalTopicConfig createTopicConfig(final String repartitionTopic, - final int partitions) { + final int partitions) { final InternalTopicConfig repartitionTopicConfig = new RepartitionTopicConfig(repartitionTopic, Collections.emptyMap()); @@ -115,4 +183,11 @@ private InternalTopicConfig createTopicConfig(final String repartitionTopic, return repartitionTopicConfig; } + private InternalTopicConfig createImmutableTopicConfig(final String repartitionTopic, + final int partitions) { + return new ImmutableRepartitionTopicConfig(repartitionTopic, + partitions, + Collections.emptyMap()); + } + } \ No newline at end of file diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java index 1985cde8e3d1b..7543632754403 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java @@ -42,7 +42,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.regex.Pattern; @@ -886,9 +885,9 @@ public void shouldHaveCorrectInternalTopicConfigWhenInternalTopicPropertiesArePr assertEquals( repartitionSourceTopics.get("Z-topic-1z"), - new RepartitionTopicConfig( + new ImmutableRepartitionTopicConfig( "Z-topic-1z", - Optional.of(numberOfPartitions), + numberOfPartitions, Collections.emptyMap() ) ); @@ -928,19 +927,4 @@ public void shouldHaveCorrectInternalTopicConfigWhenInternalTopicPropertiesAreNo new RepartitionTopicConfig("Y-topic-1y", Collections.emptyMap()) ); } - - @Test - public void shouldUpdateCopartitionGroups() { - builder.copartitionSources(mkSet("SOURCE-NODE-1", "SOURCE-NODE-2", "SOURCE-NODE-3")); - - builder.addSource(null, "SOURCE-NODE-1", null, null, null, "topic-1"); - builder.addSource(null, "OPTIMIZED-SOURCE-NODE-2", null, null, null, "topic-2"); - builder.addSource(null, "SOURCE-NODE-3", null, null, null, "topic-3"); - - builder.maybeUpdateCopartitionSourceGroups("SOURCE-NODE-2", "OPTIMIZED-SOURCE-NODE-2"); - - final Collection> copartitionGroups = builder.copartitionGroups(); - - assertEquals(copartitionGroups, Collections.singletonList(mkSet("topic-1", "topic-2", "topic-3"))); - } } From e897952f1d37e492615bef6f0e93a9681ce3cd47 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Fri, 15 Nov 2019 19:01:27 +0200 Subject: [PATCH 51/68] KAFKA-8611 / Minor: user Utils.mkMap instead of HashMap --- .../KStreamRepartitionIntegrationTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 49e721f462f6b..b6cf2aa6daceb 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -27,6 +27,7 @@ import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KafkaStreams.State; import org.apache.kafka.streams.KeyValue; @@ -53,7 +54,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -110,10 +110,9 @@ public void before() throws InterruptedException { @After public void whenShuttingDown() throws IOException { - kafkaStreamsInstances - .stream() - .filter(Objects::nonNull) - .forEach(KafkaStreams::close); + kafkaStreamsInstances.stream() + .filter(Objects::nonNull) + .forEach(KafkaStreams::close); IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration); TEST_NUM.incrementAndGet(); @@ -151,20 +150,21 @@ public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationsD startStreams(builder, REBALANCING, ERROR, (t, e) -> throwable.set(e)); - final Map repartitionTopicsWithNumberOfPartitions = new HashMap<>(); - repartitionTopicsWithNumberOfPartitions.put(toRepartitionTopicName(topicBRepartitionedName), topicBNumberOfPartitions); - repartitionTopicsWithNumberOfPartitions.put(toRepartitionTopicName(inputTopicRepartitionedName), inputTopicNumberOfPartitions); + final Map repartitionTopicsWithNumOfPartitions = Utils.mkMap( + Utils.mkEntry(toRepartitionTopicName(topicBRepartitionedName), topicBNumberOfPartitions), + Utils.mkEntry(toRepartitionTopicName(inputTopicRepartitionedName), inputTopicNumberOfPartitions) + ); final String expectedErrorMessage = String.format("Following topics do not have the same " + "number of partitions: [%s]", - new TreeMap<>(repartitionTopicsWithNumberOfPartitions)); + new TreeMap<>(repartitionTopicsWithNumOfPartitions)); assertNotNull(throwable.get()); assertTrue(throwable.get().getMessage().contains(expectedErrorMessage)); } @Test - public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDoNotMatchSourceTopicWhenJoining() throws ExecutionException, InterruptedException { + public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDoNotMatchSourceTopicWhenJoining() throws InterruptedException { final String topicBMapperName = "topic-b-mapper"; final String topicB = "topic-b-" + TEST_NUM.get(); final int topicBNumberOfPartitions = 6; From f211f801dfd4a0d4adbb3b891ff0fe537e2705c1 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Fri, 15 Nov 2019 21:27:21 +0200 Subject: [PATCH 52/68] KAFKA-8611 / Minor: createRepartition method overload --- .../GroupedStreamAggregateBuilder.java | 1 - .../internals/InternalStreamsBuilder.java | 1 - .../kstream/internals/KStreamImpl.java | 26 ++++++++++++++----- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java index b6fbb8a65d8d9..b77a2303d9d29 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java @@ -128,7 +128,6 @@ private String createRepartitionSource(final String repartitionTopicNamePrefix, keySerde, valueSerde, repartitionTopicNamePrefix, - null, optimizableRepartitionNodeBuilder); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java index aedeb9c7d8b79..f8c23e63a74e5 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java @@ -426,7 +426,6 @@ private OptimizableRepartitionNode createRepartitionNode(final String repartitio keySerde, valueSerde, repartitionTopicName, - null, repartitionNodeBuilder); // ensures setting the repartition topic to the name of the diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 3eb031c6769ac..e2e80a9da930c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -915,7 +915,6 @@ private KStreamImpl repartitionForJoin(final String repartitionName, repartitionKeySerde, repartitionValueSerde, repartitionName, - null, optimizableRepartitionNodeBuilder); final OptimizableRepartitionNode optimizableRepartitionNode = optimizableRepartitionNodeBuilder.build(); @@ -924,12 +923,25 @@ private KStreamImpl repartitionForJoin(final String repartitionName, return new KStreamImpl<>(repartitionedSourceName, repartitionKeySerde, repartitionValueSerde, Collections.singleton(repartitionedSourceName), false, optimizableRepartitionNode, builder); } - static > String createRepartitionedSource(final InternalStreamsBuilder builder, - final Serde keySerde, - final Serde valSerde, - final String repartitionTopicNamePrefix, - final StreamPartitioner streamPartitioner, - final BaseRepartitionNodeBuilder baseRepartitionNodeBuilder) { + static > String createRepartitionedSource(final InternalStreamsBuilder builder, + final Serde keySerde, + final Serde valSerde, + final String repartitionTopicNamePrefix, + final BaseRepartitionNodeBuilder baseRepartitionNodeBuilder) { + return createRepartitionedSource(builder, + keySerde, + valSerde, + repartitionTopicNamePrefix, + null, + baseRepartitionNodeBuilder); + } + + static > String createRepartitionedSource(final InternalStreamsBuilder builder, + final Serde keySerde, + final Serde valSerde, + final String repartitionTopicNamePrefix, + final StreamPartitioner streamPartitioner, + final BaseRepartitionNodeBuilder baseRepartitionNodeBuilder) { final String repartitionTopicName = repartitionTopicNamePrefix.endsWith(REPARTITION_TOPIC_SUFFIX) ? repartitionTopicNamePrefix : repartitionTopicNamePrefix + REPARTITION_TOPIC_SUFFIX; From a5b41a4249bacf77e33dc32f0183640733265409 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Wed, 15 Jan 2020 11:41:54 +0400 Subject: [PATCH 53/68] KAFKA-8611 / fix raw types warning in BaseRepartitionNode implementations; rename UnoptimizableRepartitionNode#repartitionNodeBuilder -> UnoptimizableRepartitionNode#unoptimizableRepartitionNodeBuilder --- .../kafka/streams/kstream/internals/KStreamImpl.java | 2 +- .../kstream/internals/graph/BaseRepartitionNode.java | 8 ++++---- .../graph/GroupedTableOperationRepartitionNode.java | 2 +- .../internals/graph/OptimizableRepartitionNode.java | 2 +- .../internals/graph/UnoptimizableRepartitionNode.java | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 5b19be77f1e79..de652a9743452 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -596,7 +596,7 @@ private KStream doRepartition(final Repartitioned repartition final Serde valueSerde = repartitionedInternal.valueSerde() == null ? valSerde : repartitionedInternal.valueSerde(); final UnoptimizableRepartitionNodeBuilder unoptimizableRepartitionNodeBuilder = UnoptimizableRepartitionNode - .repartitionNodeBuilder(); + .unoptimizableRepartitionNodeBuilder(); final InternalTopicProperties internalTopicProperties = repartitionedInternal.toInternalTopicProperties(); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java index f08e46695db16..1e415c7fa4ba1 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/BaseRepartitionNode.java @@ -30,13 +30,13 @@ public abstract class BaseRepartitionNode extends StreamsGraphNode { protected final String sinkName; protected final String sourceName; protected final String repartitionTopic; - protected final ProcessorParameters processorParameters; + protected final ProcessorParameters processorParameters; protected final StreamPartitioner partitioner; protected final InternalTopicProperties internalTopicProperties; BaseRepartitionNode(final String nodeName, final String sourceName, - final ProcessorParameters processorParameters, + final ProcessorParameters processorParameters, final Serde keySerde, final Serde valueSerde, final String sinkName, @@ -88,7 +88,7 @@ public String toString() { public abstract static class BaseRepartitionNodeBuilder> { protected String nodeName; - protected ProcessorParameters processorParameters; + protected ProcessorParameters processorParameters; protected Serde keySerde; protected Serde valueSerde; protected String sinkName; @@ -97,7 +97,7 @@ public abstract static class BaseRepartitionNodeBuilder partitioner; protected InternalTopicProperties internalTopicProperties = InternalTopicProperties.empty(); - public BaseRepartitionNodeBuilder withProcessorParameters(final ProcessorParameters processorParameters) { + public BaseRepartitionNodeBuilder withProcessorParameters(final ProcessorParameters processorParameters) { this.processorParameters = processorParameters; return this; } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java index ce8ec1f958d48..6d01a5fb8ea11 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java @@ -35,7 +35,7 @@ private GroupedTableOperationRepartitionNode(final String nodeName, final String sinkName, final String sourceName, final String repartitionTopic, - final ProcessorParameters processorParameters, + final ProcessorParameters processorParameters, final InternalTopicProperties internalTopicProperties) { super( diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index 0b05f9e64767f..27abaa942b22e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -27,7 +27,7 @@ public class OptimizableRepartitionNode extends BaseRepartitionNode private OptimizableRepartitionNode(final String nodeName, final String sourceName, - final ProcessorParameters processorParameters, + final ProcessorParameters processorParameters, final Serde keySerde, final Serde valueSerde, final String sinkName, diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java index ed4165664765b..0d228a5c46334 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/UnoptimizableRepartitionNode.java @@ -29,7 +29,7 @@ public class UnoptimizableRepartitionNode extends BaseRepartitionNode processorParameters, final Serde keySerde, final Serde valueSerde, final String sinkName, @@ -83,7 +83,7 @@ public String toString() { return "UnoptimizableRepartitionNode{" + super.toString() + " }"; } - public static UnoptimizableRepartitionNodeBuilder repartitionNodeBuilder() { + public static UnoptimizableRepartitionNodeBuilder unoptimizableRepartitionNodeBuilder() { return new UnoptimizableRepartitionNodeBuilder<>(); } From de1d14d5ca203677d04d77d694be4028bbdcd81b Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Wed, 15 Jan 2020 11:50:13 +0400 Subject: [PATCH 54/68] KAFKA-8611 / use methods from Optional for building repartition topic config --- .../processor/internals/InternalTopologyBuilder.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index 31e6317f76b3d..437e63146c2f8 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -1057,11 +1057,12 @@ public synchronized Map topicGroups() { private RepartitionTopicConfig buildRepartitionTopicConfig(final String internalTopic, final Optional numberOfPartitions) { - if (numberOfPartitions.isPresent()) { - return new ImmutableRepartitionTopicConfig(internalTopic, numberOfPartitions.get(), Collections.emptyMap()); - } else { - return new RepartitionTopicConfig(internalTopic, Collections.emptyMap()); - } + return numberOfPartitions + .map(partitions -> (RepartitionTopicConfig) new ImmutableRepartitionTopicConfig(internalTopic, + partitions, + Collections.emptyMap())) + + .orElse(new RepartitionTopicConfig(internalTopic, Collections.emptyMap())); } private void setRegexMatchedTopicsToSourceNodes() { From c9979b2f2f284285af338b5b7b45fc46b5fb56fe Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 2 Feb 2020 12:34:24 +0200 Subject: [PATCH 55/68] KAFKA-8611 / do not speecify key serder in repartition operations --- .../streams/kstream/internals/KStreamImpl.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index de652a9743452..051adcfe67ef1 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -562,14 +562,12 @@ public KStream repartition() { @Override public KStream repartition(final Repartitioned repartitioned) { - Objects.requireNonNull(repartitioned, "repartitioned parameter can't be null"); - return doRepartition(repartitioned, null); } @Override public KStream repartition(final KeyValueMapper selector) { - Objects.requireNonNull(selector, "selector parameter can't be null"); + Objects.requireNonNull(selector, "selector can't be null"); return doRepartition(Repartitioned.as(null), selector); } @@ -577,22 +575,20 @@ public KStream repartition(final KeyValueMapper KStream repartition(final KeyValueMapper selector, final Repartitioned repartitioned) { - Objects.requireNonNull(selector, "selector parameter can't be null"); - Objects.requireNonNull(repartitioned, "repartitioned parameter can't be null"); + Objects.requireNonNull(selector, "selector can't be null"); return doRepartition(repartitioned, selector); } - @SuppressWarnings("unchecked") private KStream doRepartition(final Repartitioned repartitioned, final KeyValueMapper selector) { + Objects.requireNonNull(repartitioned, "repartitioned can't be null"); + final RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); final String name = repartitionedInternal.name() != null ? repartitionedInternal.name() : builder .newProcessorName(REPARTITION_NAME); - final Serde keySerde = repartitionedInternal.keySerde() == null ? (Serde) this.keySerde : repartitionedInternal.keySerde(); - final Serde valueSerde = repartitionedInternal.valueSerde() == null ? valSerde : repartitionedInternal.valueSerde(); final UnoptimizableRepartitionNodeBuilder unoptimizableRepartitionNodeBuilder = UnoptimizableRepartitionNode @@ -602,7 +598,7 @@ private KStream doRepartition(final Repartitioned repartition final String repartitionSourceName = createRepartitionedSource( builder, - keySerde, + repartitionedInternal.keySerde(), valueSerde, name, repartitionedInternal.streamPartitioner(), @@ -630,10 +626,9 @@ private KStream doRepartition(final Repartitioned repartition return new KStreamImpl<>( repartitionSourceName, - keySerde, + repartitionedInternal.keySerde(), valueSerde, Collections.unmodifiableSet(sourceNodes), - // explicitly set repartition required as false false, unoptimizableRepartitionNode, builder From 765d4f409755a3e7dd35d1dc84ed763ba31b3fec Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 2 Feb 2020 12:50:55 +0200 Subject: [PATCH 56/68] KAFKA-8611 / fix all the NITs --- .../apache/kafka/streams/kstream/KStream.java | 36 +++++++++---------- .../kafka/streams/kstream/Repartitioned.java | 10 +++--- .../kstream/internals/KStreamImpl.java | 12 +++---- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java index ab33a02fdcfa6..2ad7a984150bc 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java @@ -847,9 +847,9 @@ KStream through(final String topic, * Materialize this stream to an auto-generated repartition topic and create a new {@code KStream} * from the auto-generated topic using default serializers, deserializers, and producers {@link DefaultPartitioner}. * The number of partitions is determined based on the upstream topics partition numbers. - * - * This operation is similar to {@link #through}, however, Kafka Streams manages the used topic automatically. - * Created topic is considered as internal topic and is meant to be used only by current Kafka Streams instance. + *

+ * This operation is similar to {@link #through(String)}, however, Kafka Streams manages the used topic automatically. + * The Created topic is considered as an internal topic and is meant to be used only by current Kafka Streams instance. * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be purged by Kafka Streams explicitly. * The topic will be named as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, @@ -865,9 +865,9 @@ KStream through(final String topic, * Materialize this stream to an auto-generated repartition topic and create a new {@code KStream} * from the auto-generated topic using {@link Serde key serde}, {@link Serde value serde}, {@link StreamPartitioner}, * number of partitions, and topic name part as defined by {@link Repartitioned}. - * - * This operation is similar to {@link #through}, however, Kafka Streams manages the used topic automatically. - * Created topic is considered as internal topic and is meant to be used only by current Kafka Streams instance. + *

+ * This operation is similar to {@link #through(String)}, however, Kafka Streams manages the used topic automatically. + * The Created topic is considered as an internal topic and is meant to be used only by current Kafka Streams instance. * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be purged by Kafka Streams explicitly. * The topic will be named as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, @@ -887,41 +887,41 @@ KStream through(final String topic, * Materialize this stream to an auto-generated repartition topic and create a new {@code KStream} * from the auto-generated topic using default serializers, deserializers, and producers {@link DefaultPartitioner}. * The number of partitions is determined based on the upstream topics partition numbers. - * - * This operation is similar to {@link #through}, however, Kafka Streams manages the used topic automatically. - * Created topic is considered as internal topic and is meant to be used only by current Kafka Streams instance. + *

+ * This operation is similar to {@link #through(String)}, however, Kafka Streams manages the used topic automatically. + * The Created topic is considered as an internal topic and is meant to be used only by current Kafka Streams instance. * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be purged by Kafka Streams explicitly. * The topic will be named as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, * "<name>" is an internally generated name, and "-repartition" is a fixed suffix. - * + *

* The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. * Thus, an input record {@code } can be transformed into an output record {@code }. * - * @param selector a {@link KeyValueMapper} that computes a new key for each record + * @param keySelector a {@link KeyValueMapper} that computes a new key for each record * @return a {@code KStream} that contains records with new key (possibly of different type) and unmodified value * @see #through(String) * @see #through(String, Produced) */ - KStream repartition(final KeyValueMapper selector); + KStream repartition(final KeyValueMapper keySelector); /** * Materialize this stream to an auto-generated repartition topic and create a new {@code KStream} * from the auto-generated topic using {@link Serde key serde}, {@link Serde value serde}, {@link StreamPartitioner}, * number of partitions, and topic name part as defined by {@link Repartitioned}. - * - * This operation is similar to {@link #through}, however, Kafka Streams manages the used topic automatically. - * Created topic is considered as internal topic and is meant to be used only by current Kafka Streams instance. + *

+ * This operation is similar to {@link #through(String)}, however, Kafka Streams manages the used topic automatically. + * The Created topic is considered as an internal topic and is meant to be used only by current Kafka Streams instance. * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be purged by Kafka Streams explicitly. * The topic will be named as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, * "<name>" is either provided via {@link Repartitioned#as(String)} or an internally * generated name, and "-repartition" is a fixed suffix. - * + *

* The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. * Thus, an input record {@code } can be transformed into an output record {@code }. * - * @param selector a {@link KeyValueMapper} that computes a new key for each record + * @param keySelector a {@link KeyValueMapper} that computes a new key for each record * @param repartitioned the {@link Repartitioned} instance used to specify {@link Serdes}, * {@link StreamPartitioner} which determines how records are distributed among partitions of the topic, * part of the topic name, and number of partitions for a repartition topic. @@ -929,7 +929,7 @@ KStream through(final String topic, * @see #through(String) * @see #through(String, Produced) */ - KStream repartition(final KeyValueMapper selector, + KStream repartition(final KeyValueMapper keySelector, final Repartitioned repartitioned); /** diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java index a188677b080cd..ad5d8d070450f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java @@ -23,14 +23,14 @@ import org.apache.kafka.streams.processor.StreamPartitioner; /** - * This class is used to provide the optional parameters for internal repartitioned topics when using: - *

    - *
  • {@link KStream#repartition(Repartitioned)}
    • - *
  • {@link KStream#repartition(KeyValueMapper, Repartitioned)}
    • - *
+ * This class is used to provide the optional parameters for internal repartition topics. * * @param key type * @param value type + * @see KStream#repartition() + * @see KStream#repartition(Repartitioned) + * @see KStream#repartition(KeyValueMapper) + * @see KStream#repartition(KeyValueMapper, Repartitioned) */ public class Repartitioned implements NamedOperation> { diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 051adcfe67ef1..23f8c273cdfd2 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -566,18 +566,18 @@ public KStream repartition(final Repartitioned repartitioned) { } @Override - public KStream repartition(final KeyValueMapper selector) { - Objects.requireNonNull(selector, "selector can't be null"); + public KStream repartition(final KeyValueMapper keySelector) { + Objects.requireNonNull(keySelector, "selector can't be null"); - return doRepartition(Repartitioned.as(null), selector); + return doRepartition(Repartitioned.as(null), keySelector); } @Override - public KStream repartition(final KeyValueMapper selector, + public KStream repartition(final KeyValueMapper keySelector, final Repartitioned repartitioned) { - Objects.requireNonNull(selector, "selector can't be null"); + Objects.requireNonNull(keySelector, "selector can't be null"); - return doRepartition(repartitioned, selector); + return doRepartition(repartitioned, keySelector); } private KStream doRepartition(final Repartitioned repartitioned, From 9efe7fc51d26e440194d5a77545f48f29c8a17a5 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 2 Feb 2020 14:24:11 +0200 Subject: [PATCH 57/68] KAFKA-8611 / remove ImmutableRepartitionTopicConfig --- .../ImmutableRepartitionTopicConfig.java | 74 ------------------- .../internals/InternalTopicConfig.java | 33 ++++++++- .../internals/InternalTopologyBuilder.java | 8 +- .../internals/RepartitionTopicConfig.java | 13 +++- .../internals/StreamsPartitionAssignor.java | 5 +- .../UnwindowedChangelogTopicConfig.java | 6 +- .../WindowedChangelogTopicConfig.java | 8 +- .../CopartitionedTopicsEnforcer.java | 3 +- .../CopartitionedTopicsEnforcerTest.java | 23 +++--- .../InternalTopologyBuilderTest.java | 5 +- 10 files changed, 73 insertions(+), 105 deletions(-) delete mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/ImmutableRepartitionTopicConfig.java diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ImmutableRepartitionTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ImmutableRepartitionTopicConfig.java deleted file mode 100644 index 14127c75538dd..0000000000000 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ImmutableRepartitionTopicConfig.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.streams.processor.internals; - -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -/** - * repartition topic config which doesn't permit changing number of partitions - */ -public class ImmutableRepartitionTopicConfig extends RepartitionTopicConfig { - private final int numberOfPartitions; - - ImmutableRepartitionTopicConfig(final String name, - final int numberOfPartitions, - final Map topicConfigs) { - super(name, topicConfigs); - validateNumberOfPartitions(numberOfPartitions); - this.numberOfPartitions = numberOfPartitions; - } - - @Override - public Optional numberOfPartitions() { - return Optional.of(numberOfPartitions); - } - - @Override - public void setNumberOfPartitions(final int numberOfPartitions) { - //no-op - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final ImmutableRepartitionTopicConfig that = (ImmutableRepartitionTopicConfig) o; - return Objects.equals(name, that.name) && - Objects.equals(topicConfigs, that.topicConfigs) && - Objects.equals(numberOfPartitions, that.numberOfPartitions); - } - - @Override - public int hashCode() { - return Objects.hash(name, topicConfigs, numberOfPartitions); - } - - @Override - public String toString() { - return "ImmutableRepartitionTopicConfig(" + - "name=" + name + - ", topicConfigs=" + topicConfigs + - ", numberOfPartitions=" + numberOfPartitions + - ")"; - } -} diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java index 8d45bb222cce2..c37e199a62bf7 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java @@ -16,10 +16,10 @@ */ package org.apache.kafka.streams.processor.internals; -import java.util.HashMap; import org.apache.kafka.common.config.TopicConfig; import org.apache.kafka.common.internals.Topic; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -31,6 +31,7 @@ public abstract class InternalTopicConfig { final String name; final Map topicConfigs; + final boolean enforceNumberOfPartitions; private Optional numberOfPartitions = Optional.empty(); @@ -45,6 +46,22 @@ public abstract class InternalTopicConfig { this.name = name; this.topicConfigs = topicConfigs; + this.enforceNumberOfPartitions = false; + } + + InternalTopicConfig(final String name, + final Map topicConfigs, + final int numberOfPartitions, + final boolean enforceNumberOfPartitions) { + Objects.requireNonNull(name, "name can't be null"); + Topic.validate(name); + + validateNumberOfPartitions(numberOfPartitions); + + this.name = name; + this.topicConfigs = topicConfigs; + this.numberOfPartitions = Optional.of(numberOfPartitions); + this.enforceNumberOfPartitions = enforceNumberOfPartitions; } /** @@ -54,7 +71,11 @@ public abstract class InternalTopicConfig { * @param additionalRetentionMs - added to retention to allow for clock drift etc * @return Properties to be used when creating the topic */ - abstract public Map getProperties(final Map defaultProperties, final long additionalRetentionMs); + public abstract Map getProperties(final Map defaultProperties, final long additionalRetentionMs); + + public boolean hasEnforcedNumberOfPartitions() { + return enforceNumberOfPartitions; + } public String name() { return name; @@ -65,11 +86,16 @@ public Optional numberOfPartitions() { } public void setNumberOfPartitions(final int numberOfPartitions) { + if (hasEnforcedNumberOfPartitions() && this.numberOfPartitions.isPresent()) { + throw new UnsupportedOperationException("number of partitions is enforced and can't be altered"); + } + validateNumberOfPartitions(numberOfPartitions); + this.numberOfPartitions = Optional.of(numberOfPartitions); } - protected void validateNumberOfPartitions(final int numberOfPartitions) { + private static void validateNumberOfPartitions(final int numberOfPartitions) { if (numberOfPartitions < 1) { throw new IllegalArgumentException("Number of partitions must be at least 1."); } @@ -80,6 +106,7 @@ public String toString() { return "InternalTopicConfig(" + "name=" + name + ", topicConfigs=" + topicConfigs + + ", enforceNumberOfPartitions=" + enforceNumberOfPartitions + ")"; } } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index 6dc6305c40567..f34d021dec3bd 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -1068,10 +1068,10 @@ public synchronized Map topicGroups() { private RepartitionTopicConfig buildRepartitionTopicConfig(final String internalTopic, final Optional numberOfPartitions) { return numberOfPartitions - .map(partitions -> (RepartitionTopicConfig) new ImmutableRepartitionTopicConfig(internalTopic, - partitions, - Collections.emptyMap())) - + .map(partitions -> new RepartitionTopicConfig(internalTopic, + Collections.emptyMap(), + partitions, + true)) .orElse(new RepartitionTopicConfig(internalTopic, Collections.emptyMap())); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java index 78dcf64ccdf1b..098bb9f58a130 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java @@ -42,6 +42,13 @@ public class RepartitionTopicConfig extends InternalTopicConfig { super(name, topicConfigs); } + RepartitionTopicConfig(final String name, + final Map topicConfigs, + final int numberOfPartitions, + final boolean enforceNumberOfPartitions) { + super(name, topicConfigs, numberOfPartitions, enforceNumberOfPartitions); + } + /** * Get the configured properties for this topic. If retentionMs is set then * we add additionalRetentionMs to work out the desired retention when cleanup.policy=compact,delete @@ -71,12 +78,13 @@ public boolean equals(final Object o) { } final RepartitionTopicConfig that = (RepartitionTopicConfig) o; return Objects.equals(name, that.name) && - Objects.equals(topicConfigs, that.topicConfigs); + Objects.equals(topicConfigs, that.topicConfigs) && + Objects.equals(enforceNumberOfPartitions, that.enforceNumberOfPartitions); } @Override public int hashCode() { - return Objects.hash(name, topicConfigs); + return Objects.hash(name, topicConfigs, enforceNumberOfPartitions); } @Override @@ -84,6 +92,7 @@ public String toString() { return "RepartitionTopicConfig(" + "name=" + name + ", topicConfigs=" + topicConfigs + + ", enforceNumberOfPartitions=" + enforceNumberOfPartitions + ")"; } } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java index d3baf6e9951c3..0a872e949cbbb 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java @@ -1238,8 +1238,9 @@ private void prepareTopic(final Map topicPartitions logPrefix, topic.name()) ); } - - topic.setNumberOfPartitions(numPartitions.get()); + if (!topic.hasEnforcedNumberOfPartitions()) { + topic.setNumberOfPartitions(numPartitions.get()); + } topicsToMakeReady.put(topic.name(), topic); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/UnwindowedChangelogTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/UnwindowedChangelogTopicConfig.java index fa455a92a164f..b0c2fdae59ff8 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/UnwindowedChangelogTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/UnwindowedChangelogTopicConfig.java @@ -68,12 +68,13 @@ public boolean equals(final Object o) { } final UnwindowedChangelogTopicConfig that = (UnwindowedChangelogTopicConfig) o; return Objects.equals(name, that.name) && - Objects.equals(topicConfigs, that.topicConfigs); + Objects.equals(topicConfigs, that.topicConfigs) && + Objects.equals(enforceNumberOfPartitions, that.enforceNumberOfPartitions); } @Override public int hashCode() { - return Objects.hash(name, topicConfigs); + return Objects.hash(name, topicConfigs, enforceNumberOfPartitions); } @Override @@ -81,6 +82,7 @@ public String toString() { return "UnwindowedChangelogTopicConfig(" + "name=" + name + ", topicConfigs=" + topicConfigs + + ", enforceNumberOfPartitions=" + enforceNumberOfPartitions + ")"; } } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/WindowedChangelogTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/WindowedChangelogTopicConfig.java index 3f65272dc70b1..6e1dc86575c96 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/WindowedChangelogTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/WindowedChangelogTopicConfig.java @@ -86,13 +86,14 @@ public boolean equals(final Object o) { } final WindowedChangelogTopicConfig that = (WindowedChangelogTopicConfig) o; return Objects.equals(name, that.name) && - Objects.equals(topicConfigs, that.topicConfigs) && - Objects.equals(retentionMs, that.retentionMs); + Objects.equals(topicConfigs, that.topicConfigs) && + Objects.equals(retentionMs, that.retentionMs) && + Objects.equals(enforceNumberOfPartitions, that.enforceNumberOfPartitions); } @Override public int hashCode() { - return Objects.hash(name, topicConfigs, retentionMs); + return Objects.hash(name, topicConfigs, retentionMs, enforceNumberOfPartitions); } @Override @@ -101,6 +102,7 @@ public String toString() { "name=" + name + ", topicConfigs=" + topicConfigs + ", retentionMs=" + retentionMs + + ", enforceNumberOfPartitions=" + enforceNumberOfPartitions + ")"; } } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java index e110c58c24e3f..55fb0eb4dc82f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java @@ -19,7 +19,6 @@ import org.apache.kafka.common.Cluster; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.streams.errors.TopologyException; -import org.apache.kafka.streams.processor.internals.ImmutableRepartitionTopicConfig; import org.apache.kafka.streams.processor.internals.InternalTopicConfig; import org.slf4j.Logger; @@ -129,7 +128,7 @@ private int validateAndGetNumOfPartitionsOfImmutableTopics(final Map nonRepartitionTopicsInCopartitionGroup) { diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java index 60262c30a09bb..6fa280c2c35f3 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java @@ -111,8 +111,8 @@ public void shouldSetNumPartitionsToMaximumPartitionsWhenAllTopicsAreRepartition @Test public void shouldThrowAnExceptionIfImmutableRepartitionTopicsDoNotHaveSameNumberOfPartitions() { - final InternalTopicConfig topic1 = createImmutableTopicConfig("repartitioned-1", 10); - final InternalTopicConfig topic2 = createImmutableTopicConfig("repartitioned-2", 5); + final InternalTopicConfig topic1 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-1", 10); + final InternalTopicConfig topic2 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-2", 5); final TopologyException ex = assertThrows( TopologyException.class, @@ -134,8 +134,8 @@ public void shouldThrowAnExceptionIfImmutableRepartitionTopicsDoNotHaveSameNumbe @Test public void shouldNotThrowAnExceptionWhenNumberOfPartitionsOfImmutableTopicConfigsAreTheSame() { - final InternalTopicConfig topic1 = createImmutableTopicConfig("repartitioned-1", 10); - final InternalTopicConfig topic2 = createImmutableTopicConfig("repartitioned-2", 10); + final InternalTopicConfig topic1 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-1", 10); + final InternalTopicConfig topic2 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-2", 10); validator.enforce(Utils.mkSet(topic1.name(), topic2.name()), Utils.mkMap(Utils.mkEntry(topic1.name(), topic1), @@ -148,7 +148,7 @@ public void shouldNotThrowAnExceptionWhenNumberOfPartitionsOfImmutableTopicConfi @Test public void shouldThrowAnExceptionWhenNumberOfPartitionsOfNonRepartitionTopicAndImmutableRepartitionTopicDoNotMatch() { - final InternalTopicConfig topic1 = createImmutableTopicConfig("repartitioned-1", 10); + final InternalTopicConfig topic1 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-1", 10); final TopologyException ex = assertThrows( TopologyException.class, @@ -165,7 +165,7 @@ public void shouldThrowAnExceptionWhenNumberOfPartitionsOfNonRepartitionTopicAnd @Test public void shouldNotThrowAnExceptionWhenNumberOfPartitionsOfNonRepartitionTopicAndImmutableRepartitionTopicMatch() { - final InternalTopicConfig topic1 = createImmutableTopicConfig("repartitioned-1", 2); + final InternalTopicConfig topic1 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-1", 2); validator.enforce(Utils.mkSet(topic1.name(), "second"), Utils.mkMap(Utils.mkEntry(topic1.name(), topic1)), @@ -183,11 +183,12 @@ private InternalTopicConfig createTopicConfig(final String repartitionTopic, return repartitionTopicConfig; } - private InternalTopicConfig createImmutableTopicConfig(final String repartitionTopic, - final int partitions) { - return new ImmutableRepartitionTopicConfig(repartitionTopic, - partitions, - Collections.emptyMap()); + private InternalTopicConfig createRepartitionTopicConfigWithEnforcedNumberOfPartitions(final String repartitionTopic, + final int partitions) { + return new RepartitionTopicConfig(repartitionTopic, + Collections.emptyMap(), + partitions, + true); } } \ No newline at end of file diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java index 06ce4127873ff..067ad434d4326 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java @@ -888,10 +888,11 @@ public void shouldHaveCorrectInternalTopicConfigWhenInternalTopicPropertiesArePr assertEquals( repartitionSourceTopics.get("Z-topic-1z"), - new ImmutableRepartitionTopicConfig( + new RepartitionTopicConfig( "Z-topic-1z", + Collections.emptyMap(), numberOfPartitions, - Collections.emptyMap() + true ) ); } From 66717dc7835a9a3dbc494f4ece971c566d67fb72 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 2 Feb 2020 16:13:22 +0200 Subject: [PATCH 58/68] KAFKA-8611 / Deduct num of partitions when applicable --- .../internals/InternalTopicConfig.java | 5 +- .../CopartitionedTopicsEnforcer.java | 73 +++++++++++-------- .../KStreamRepartitionIntegrationTest.java | 56 +++++++++++++- .../CopartitionedTopicsEnforcerTest.java | 24 +++++- .../internals/RepartitionTopicConfigTest.java | 58 +++++++++++++++ 5 files changed, 175 insertions(+), 41 deletions(-) create mode 100644 streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfigTest.java diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java index c37e199a62bf7..95d279c35014e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java @@ -86,8 +86,9 @@ public Optional numberOfPartitions() { } public void setNumberOfPartitions(final int numberOfPartitions) { - if (hasEnforcedNumberOfPartitions() && this.numberOfPartitions.isPresent()) { - throw new UnsupportedOperationException("number of partitions is enforced and can't be altered"); + if (hasEnforcedNumberOfPartitions()) { + throw new UnsupportedOperationException("number of partitions are enforced on topic " + + "" + name() + " and can't be altered."); } validateNumberOfPartitions(numberOfPartitions); diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java index 55fb0eb4dc82f..2d13e06e46f0e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java @@ -68,15 +68,17 @@ public void enforce(final Set copartitionGroup, final Collection internalTopicConfigs = repartitionTopicConfigs.values(); if (copartitionGroup.equals(repartitionTopicConfigs.keySet())) { - if (internalTopicConfigs.stream().anyMatch(this::isMutableRepartitionTopicConfig)) { - // If all topics for this co-partition group is repartition topics, + // if there is at least one repartiton topic with enforced number of partitions + // 1) validate that they all have same number of partitions + // 2) set all other non-enforced repartition topic + if (internalTopicConfigs.stream().anyMatch(InternalTopicConfig::hasEnforcedNumberOfPartitions)) { + + numPartitionsToUseForRepartitionTopics = validateAndGetNumOfPartitions(repartitionTopicConfigs, + internalTopicConfigs); + } else { + // If all topics for this co-partition group are repartition topics, // then set the number of partitions to be the maximum of the number of partitions. numPartitionsToUseForRepartitionTopics = getMaxPartitions(repartitionTopicConfigs); - } else { - // if all repartition topics are immutable repartition topics, - // we must ensure that they all have same number of partitions. - numPartitionsToUseForRepartitionTopics = validateAndGetNumOfPartitionsOfImmutableTopics(repartitionTopicConfigs, - internalTopicConfigs); } } else { // Otherwise, use the number of partitions from external topics (which must all be the same) @@ -85,32 +87,43 @@ public void enforce(final Set copartitionGroup, // coerce all the repartition topics to use the decided number of partitions. for (final InternalTopicConfig config : internalTopicConfigs) { - if (isMutableRepartitionTopicConfig(config)) { - config.setNumberOfPartitions(numPartitionsToUseForRepartitionTopics); - } else { - final int numberOfPartitionsOfInternalTopic = config.numberOfPartitions().get(); - - if (numberOfPartitionsOfInternalTopic != numPartitionsToUseForRepartitionTopics) { - final String msg = String.format("%sNumber of partitions [%s] of repartition topic [%s] " + - "doesn't match number of partitions [%s] of the source topic.", - logPrefix, - numberOfPartitionsOfInternalTopic, - config.name(), - numPartitionsToUseForRepartitionTopics); - throw new TopologyException(msg); - } + maybeSetNumberOfPartitionsForInternalTopic(numPartitionsToUseForRepartitionTopics, config); + + final int numberOfPartitionsOfInternalTopic = config.numberOfPartitions().get(); + + if (numberOfPartitionsOfInternalTopic != numPartitionsToUseForRepartitionTopics) { + final String msg = String.format("%sNumber of partitions [%s] of repartition topic [%s] " + + "doesn't match number of partitions [%s] of the source topic.", + logPrefix, + numberOfPartitionsOfInternalTopic, + config.name(), + numPartitionsToUseForRepartitionTopics); + throw new TopologyException(msg); } } } - private int validateAndGetNumOfPartitionsOfImmutableTopics(final Map repartitionTopicConfigs, - final Collection internalTopicConfigs) { - final int numberOfPartitionsOfInternalTopic = internalTopicConfigs.iterator() - .next() - .numberOfPartitions() - .get(); + private static void maybeSetNumberOfPartitionsForInternalTopic(final int numPartitionsToUseForRepartitionTopics, + final InternalTopicConfig config) { + if (!config.hasEnforcedNumberOfPartitions()) { + config.setNumberOfPartitions(numPartitionsToUseForRepartitionTopics); + } + } + + private int validateAndGetNumOfPartitions(final Map repartitionTopicConfigs, + final Collection internalTopicConfigs) { + final Collection internalTopicConfigsWithEnforcedNumberOfPartitions = internalTopicConfigs + .stream() + .filter(InternalTopicConfig::hasEnforcedNumberOfPartitions) + .collect(Collectors.toList()); - for (final InternalTopicConfig internalTopicConfig : internalTopicConfigs) { + final int numberOfPartitionsOfInternalTopic = internalTopicConfigsWithEnforcedNumberOfPartitions + .iterator() + .next() + .numberOfPartitions() + .get(); + + for (final InternalTopicConfig internalTopicConfig : internalTopicConfigsWithEnforcedNumberOfPartitions) { if (internalTopicConfig.numberOfPartitions().get() != numberOfPartitionsOfInternalTopic) { final Map repartitionTopics = repartitionTopicConfigs .entrySet() @@ -127,10 +140,6 @@ private int validateAndGetNumOfPartitionsOfImmutableTopics(final Map nonRepartitionTopicsInCopartitionGroup) { final int partitions = nonRepartitionTopicsInCopartitionGroup.values().iterator().next(); for (final Map.Entry entry : nonRepartitionTopicsInCopartitionGroup.entrySet()) { diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index b6cf2aa6daceb..3f39a7861b9b6 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -165,7 +165,6 @@ public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationsD @Test public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDoNotMatchSourceTopicWhenJoining() throws InterruptedException { - final String topicBMapperName = "topic-b-mapper"; final String topicB = "topic-b-" + TEST_NUM.get(); final int topicBNumberOfPartitions = 6; final String inputTopicRepartitionName = "join-repartition-test"; @@ -181,8 +180,7 @@ public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDo .withNumberOfPartitions(inputTopicRepartitionedNumOfPartitions); final KStream topicBStream = builder - .stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())) - .map(KeyValue::new, Named.as(topicBMapperName)); + .stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) .repartition(inputTopicRepartitioned) @@ -202,6 +200,58 @@ public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDo assertTrue(expectedThrowable.get().getMessage().contains(expectedMsg)); } + @Test + public void shouldDeductNumberOfPartitionsFromRepartitionOperation() throws InterruptedException, ExecutionException { + final String topicBMapperName = "topic-b-mapper"; + final String topicB = "topic-b-" + TEST_NUM.get(); + final int topicBNumberOfPartitions = 6; + final String inputTopicRepartitionName = "join-repartition-test"; + final int inputTopicRepartitionedNumOfPartitions = 3; + + final long timestamp = System.currentTimeMillis(); + + CLUSTER.createTopic(topicB, topicBNumberOfPartitions, 1); + + final List> expectedRecords = Arrays.asList( + new KeyValue<>(1, "A"), + new KeyValue<>(2, "B") + ); + + sendEvents(timestamp, expectedRecords); + sendEvents(topicB, timestamp, expectedRecords); + + final StreamsBuilder builder = new StreamsBuilder(); + + final Repartitioned inputTopicRepartitioned = Repartitioned + .as(inputTopicRepartitionName) + .withNumberOfPartitions(inputTopicRepartitionedNumOfPartitions); + + final KStream topicBStream = builder + .stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())) + .map(KeyValue::new, Named.as(topicBMapperName)); + + builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) + .repartition(inputTopicRepartitioned) + .join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))) + .to(outputTopic); + + builder.build(streamsConfiguration); + + startStreams(builder); + + assertEquals(inputTopicRepartitionedNumOfPartitions, + getNumberOfPartitionsForTopic(toRepartitionTopicName(inputTopicRepartitionName))); + + assertEquals(inputTopicRepartitionedNumOfPartitions, + getNumberOfPartitionsForTopic(toRepartitionTopicName(topicBMapperName))); + + validateReceivedMessages( + new IntegerDeserializer(), + new StringDeserializer(), + expectedRecords + ); + } + @Test public void shouldDoProperJoiningWhenNumberOfPartitionsAreValidWhenUsingRepartitionOperation() throws ExecutionException, InterruptedException { final String topicB = "topic-b-" + TEST_NUM.get(); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java index 6fa280c2c35f3..bad7390574d02 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/CopartitionedTopicsEnforcerTest.java @@ -110,7 +110,7 @@ public void shouldSetNumPartitionsToMaximumPartitionsWhenAllTopicsAreRepartition } @Test - public void shouldThrowAnExceptionIfImmutableRepartitionTopicsDoNotHaveSameNumberOfPartitions() { + public void shouldThrowAnExceptionIfRepartitionTopicConfigsWithEnforcedNumOfPartitionsHaveDifferentNumOfPartitiones() { final InternalTopicConfig topic1 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-1", 10); final InternalTopicConfig topic2 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-2", 5); @@ -133,7 +133,7 @@ public void shouldThrowAnExceptionIfImmutableRepartitionTopicsDoNotHaveSameNumbe } @Test - public void shouldNotThrowAnExceptionWhenNumberOfPartitionsOfImmutableTopicConfigsAreTheSame() { + public void shouldNotThrowAnExceptionWhenRepartitionTopicConfigsWithEnforcedNumOfPartitionsAreValid() { final InternalTopicConfig topic1 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-1", 10); final InternalTopicConfig topic2 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-2", 10); @@ -147,7 +147,7 @@ public void shouldNotThrowAnExceptionWhenNumberOfPartitionsOfImmutableTopicConfi } @Test - public void shouldThrowAnExceptionWhenNumberOfPartitionsOfNonRepartitionTopicAndImmutableRepartitionTopicDoNotMatch() { + public void shouldThrowAnExceptionWhenNumberOfPartitionsOfNonRepartitionTopicAndRepartitionTopicWithEnforcedNumOfPartitionsDoNotMatch() { final InternalTopicConfig topic1 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-1", 10); final TopologyException ex = assertThrows( @@ -164,7 +164,7 @@ public void shouldThrowAnExceptionWhenNumberOfPartitionsOfNonRepartitionTopicAnd } @Test - public void shouldNotThrowAnExceptionWhenNumberOfPartitionsOfNonRepartitionTopicAndImmutableRepartitionTopicMatch() { + public void shouldNotThrowAnExceptionWhenNumberOfPartitionsOfNonRepartitionTopicAndRepartitionTopicWithEnforcedNumOfPartitionsMatch() { final InternalTopicConfig topic1 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-1", 2); validator.enforce(Utils.mkSet(topic1.name(), "second"), @@ -174,6 +174,22 @@ public void shouldNotThrowAnExceptionWhenNumberOfPartitionsOfNonRepartitionTopic assertThat(topic1.numberOfPartitions(), equalTo(Optional.of(2))); } + @Test + public void shouldDeductNumberOfPartitionsFromRepartitionTopicWithEnforcedNumberOfPartitions() { + final InternalTopicConfig topic1 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-1", 2); + final InternalTopicConfig topic2 = createTopicConfig("repartitioned-2", 5); + final InternalTopicConfig topic3 = createRepartitionTopicConfigWithEnforcedNumberOfPartitions("repartitioned-3", 2); + + validator.enforce(Utils.mkSet(topic1.name(), topic2.name()), + Utils.mkMap(Utils.mkEntry(topic1.name(), topic1), + Utils.mkEntry(topic2.name(), topic2), + Utils.mkEntry(topic3.name(), topic3)), + cluster.withPartitions(partitions)); + + assertEquals(topic1.numberOfPartitions(), topic2.numberOfPartitions()); + assertEquals(topic2.numberOfPartitions(), topic3.numberOfPartitions()); + } + private InternalTopicConfig createTopicConfig(final String repartitionTopic, final int partitions) { final InternalTopicConfig repartitionTopicConfig = diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfigTest.java new file mode 100644 index 0000000000000..01fff7dfef9ae --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfigTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.processor.internals; + +import org.junit.Test; + +import java.util.Collections; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +public class RepartitionTopicConfigTest { + + @Test + public void shouldThrowAnExceptionWhenSettingNumberOfPartitionsIfTheyAreEnforced() { + final String name = "my-topic"; + final RepartitionTopicConfig repartitionTopicConfig = new RepartitionTopicConfig(name, + Collections.emptyMap(), + 10, + true); + + final UnsupportedOperationException ex = assertThrows( + UnsupportedOperationException.class, + () -> repartitionTopicConfig.setNumberOfPartitions(2) + ); + + assertEquals(String.format("number of partitions are enforced on topic " + + "%s and can't be altered.", name), ex.getMessage()); + } + + @Test + public void shouldNotThrowAnExceptionWhenSettingNumberOfPartitionsIfTheyAreNotEnforced() { + final String name = "my-topic"; + final RepartitionTopicConfig repartitionTopicConfig = new RepartitionTopicConfig(name, + Collections.emptyMap(), + 10, + false); + + repartitionTopicConfig.setNumberOfPartitions(4); + + assertEquals(repartitionTopicConfig.numberOfPartitions(), Optional.of(4)); + } +} \ No newline at end of file From 1d582dd3b73c8041beb23bf17cb54baa4e56568a Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 2 Feb 2020 16:31:48 +0200 Subject: [PATCH 59/68] KAFKA-8611 / update comment --- .../internals/assignment/CopartitionedTopicsEnforcer.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java index 2d13e06e46f0e..10d2ed77e84e7 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java @@ -68,9 +68,8 @@ public void enforce(final Set copartitionGroup, final Collection internalTopicConfigs = repartitionTopicConfigs.values(); if (copartitionGroup.equals(repartitionTopicConfigs.keySet())) { - // if there is at least one repartiton topic with enforced number of partitions - // 1) validate that they all have same number of partitions - // 2) set all other non-enforced repartition topic + // if there's at least one repartition topic with enforced number of partitions + // validate that they all have same number of partitions if (internalTopicConfigs.stream().anyMatch(InternalTopicConfig::hasEnforcedNumberOfPartitions)) { numPartitionsToUseForRepartitionTopics = validateAndGetNumOfPartitions(repartitionTopicConfigs, From 0cb229b66725f2ef368d4819adcb9381ace651bc Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sun, 2 Feb 2020 20:13:23 +0200 Subject: [PATCH 60/68] KAFKA-8611 / Minor refactoring of CopartitionedTopicsEnforcer class --- .../CopartitionedTopicsEnforcer.java | 51 ++++++++++++------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java index 10d2ed77e84e7..8324532032cda 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/CopartitionedTopicsEnforcer.java @@ -24,9 +24,11 @@ import java.util.Collection; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.TreeMap; +import java.util.function.Supplier; import java.util.stream.Collectors; public class CopartitionedTopicsEnforcer { @@ -68,12 +70,18 @@ public void enforce(final Set copartitionGroup, final Collection internalTopicConfigs = repartitionTopicConfigs.values(); if (copartitionGroup.equals(repartitionTopicConfigs.keySet())) { + final Collection internalTopicConfigsWithEnforcedNumberOfPartitions = internalTopicConfigs + .stream() + .filter(InternalTopicConfig::hasEnforcedNumberOfPartitions) + .collect(Collectors.toList()); + // if there's at least one repartition topic with enforced number of partitions // validate that they all have same number of partitions - if (internalTopicConfigs.stream().anyMatch(InternalTopicConfig::hasEnforcedNumberOfPartitions)) { - - numPartitionsToUseForRepartitionTopics = validateAndGetNumOfPartitions(repartitionTopicConfigs, - internalTopicConfigs); + if (!internalTopicConfigsWithEnforcedNumberOfPartitions.isEmpty()) { + numPartitionsToUseForRepartitionTopics = validateAndGetNumOfPartitions( + repartitionTopicConfigs, + internalTopicConfigsWithEnforcedNumberOfPartitions + ); } else { // If all topics for this co-partition group are repartition topics, // then set the number of partitions to be the maximum of the number of partitions. @@ -88,7 +96,9 @@ public void enforce(final Set copartitionGroup, for (final InternalTopicConfig config : internalTopicConfigs) { maybeSetNumberOfPartitionsForInternalTopic(numPartitionsToUseForRepartitionTopics, config); - final int numberOfPartitionsOfInternalTopic = config.numberOfPartitions().get(); + final int numberOfPartitionsOfInternalTopic = config + .numberOfPartitions() + .orElseThrow(emptyNumberOfPartitionsExceptionSupplier(config.name())); if (numberOfPartitionsOfInternalTopic != numPartitionsToUseForRepartitionTopics) { final String msg = String.format("%sNumber of partitions [%s] of repartition topic [%s] " + @@ -111,23 +121,22 @@ private static void maybeSetNumberOfPartitionsForInternalTopic(final int numPart private int validateAndGetNumOfPartitions(final Map repartitionTopicConfigs, final Collection internalTopicConfigs) { - final Collection internalTopicConfigsWithEnforcedNumberOfPartitions = internalTopicConfigs - .stream() - .filter(InternalTopicConfig::hasEnforcedNumberOfPartitions) - .collect(Collectors.toList()); - - final int numberOfPartitionsOfInternalTopic = internalTopicConfigsWithEnforcedNumberOfPartitions - .iterator() - .next() + final InternalTopicConfig firstInternalTopicConfig = internalTopicConfigs.iterator().next(); + + final int firstNumberOfPartitionsOfInternalTopic = firstInternalTopicConfig .numberOfPartitions() - .get(); + .orElseThrow(emptyNumberOfPartitionsExceptionSupplier(firstInternalTopicConfig.name())); - for (final InternalTopicConfig internalTopicConfig : internalTopicConfigsWithEnforcedNumberOfPartitions) { - if (internalTopicConfig.numberOfPartitions().get() != numberOfPartitionsOfInternalTopic) { + for (final InternalTopicConfig internalTopicConfig : internalTopicConfigs) { + final Integer numberOfPartitions = internalTopicConfig + .numberOfPartitions() + .orElseThrow(emptyNumberOfPartitionsExceptionSupplier(internalTopicConfig.name())); + + if (numberOfPartitions != firstNumberOfPartitionsOfInternalTopic) { final Map repartitionTopics = repartitionTopicConfigs .entrySet() .stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().numberOfPartitions().get())); + .collect(Collectors.toMap(Entry::getKey, entry -> entry.getValue().numberOfPartitions().get())); final String msg = String.format("%sFollowing topics do not have the same number of partitions: [%s]", logPrefix, @@ -136,12 +145,16 @@ private int validateAndGetNumOfPartitions(final Map } } - return numberOfPartitionsOfInternalTopic; + return firstNumberOfPartitionsOfInternalTopic; + } + + private static Supplier emptyNumberOfPartitionsExceptionSupplier(final String topic) { + return () -> new TopologyException("Number of partitions is not set for topic: " + topic); } private int getSamePartitions(final Map nonRepartitionTopicsInCopartitionGroup) { final int partitions = nonRepartitionTopicsInCopartitionGroup.values().iterator().next(); - for (final Map.Entry entry : nonRepartitionTopicsInCopartitionGroup.entrySet()) { + for (final Entry entry : nonRepartitionTopicsInCopartitionGroup.entrySet()) { if (entry.getValue() != partitions) { final TreeMap sorted = new TreeMap<>(nonRepartitionTopicsInCopartitionGroup); throw new TopologyException( From a8e3ebf579e7a5e1d13007b83f9f5ca067ff9afa Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Fri, 7 Feb 2020 09:56:15 +0200 Subject: [PATCH 61/68] KAFKA-8611 / fix test --- .../processor/internals/InternalTopologyBuilderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java index ecf6cbb6b4080..efb41272244ae 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java @@ -255,8 +255,8 @@ public void testPatternAndNameSourceTopics() { builder.addSource(null, "source-3", null, null, null, "topic-3"); builder.addSource(null, "source-4", null, null, null, sourcePattern); - builder.initializeSubscription(); builder.addInternalTopic("topic-3", InternalTopicProperties.empty()); + builder.initializeSubscription(); final Pattern expectedPattern = Pattern.compile("X-topic-3|topic-1|topic-2|topic-4|topic-5"); From a73b2114792aec172aa611379a80a677c61414a1 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Thu, 13 Feb 2020 18:03:04 +0200 Subject: [PATCH 62/68] KAFKA-8611 / fix StreamsPartitionAssignorTest --- .../processor/internals/StreamsPartitionAssignorTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java index 2865c2a55d9d1..765673a6ae97d 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java @@ -1748,7 +1748,7 @@ public void shouldNotFailOnBranchedMultiLevelRepartitionConnectedTopology() { builder.addProcessor("KSTREAM-BRANCHCHILD-0000000004", new MockProcessorSupplier(), "KSTREAM-BRANCH-0000000002"); builder.addProcessor("KSTREAM-MAP-0000000005", new MockProcessorSupplier(), "KSTREAM-BRANCHCHILD-0000000003"); - builder.addInternalTopic("odd_store-repartition"); + builder.addInternalTopic("odd_store-repartition", InternalTopicProperties.empty()); builder.addProcessor("odd_store-repartition-filter", new MockProcessorSupplier(), "KSTREAM-MAP-0000000005"); builder.addSink("odd_store-repartition-sink", "odd_store-repartition", null, null, null, "odd_store-repartition-filter"); builder.addSource(null, "odd_store-repartition-source", null, null, null, "odd_store-repartition"); @@ -1757,14 +1757,14 @@ public void shouldNotFailOnBranchedMultiLevelRepartitionConnectedTopology() { builder.addProcessor("KSTREAM-PEEK-0000000011", new MockProcessorSupplier(), "KTABLE-TOSTREAM-0000000010"); builder.addProcessor("KSTREAM-MAP-0000000012", new MockProcessorSupplier(), "KSTREAM-PEEK-0000000011"); - builder.addInternalTopic("odd_store_2-repartition"); + builder.addInternalTopic("odd_store_2-repartition", InternalTopicProperties.empty()); builder.addProcessor("odd_store_2-repartition-filter", new MockProcessorSupplier(), "KSTREAM-MAP-0000000012"); builder.addSink("odd_store_2-repartition-sink", "odd_store_2-repartition", null, null, null, "odd_store_2-repartition-filter"); builder.addSource(null, "odd_store_2-repartition-source", null, null, null, "odd_store_2-repartition"); builder.addProcessor("KSTREAM-REDUCE-0000000013", new MockProcessorSupplier(), "odd_store_2-repartition-source"); builder.addProcessor("KSTREAM-MAP-0000000017", new MockProcessorSupplier(), "KSTREAM-BRANCHCHILD-0000000004"); - builder.addInternalTopic("even_store-repartition"); + builder.addInternalTopic("even_store-repartition", InternalTopicProperties.empty()); builder.addProcessor("even_store-repartition-filter", new MockProcessorSupplier(), "KSTREAM-MAP-0000000017"); builder.addSink("even_store-repartition-sink", "even_store-repartition", null, null, null, "even_store-repartition-filter"); builder.addSource(null, "even_store-repartition-source", null, null, null, "even_store-repartition"); @@ -1773,7 +1773,7 @@ public void shouldNotFailOnBranchedMultiLevelRepartitionConnectedTopology() { builder.addProcessor("KSTREAM-PEEK-0000000023", new MockProcessorSupplier(), "KTABLE-TOSTREAM-0000000022"); builder.addProcessor("KSTREAM-MAP-0000000024", new MockProcessorSupplier(), "KSTREAM-PEEK-0000000023"); - builder.addInternalTopic("even_store_2-repartition"); + builder.addInternalTopic("even_store_2-repartition", InternalTopicProperties.empty()); builder.addProcessor("even_store_2-repartition-filter", new MockProcessorSupplier(), "KSTREAM-MAP-0000000024"); builder.addSink("even_store_2-repartition-sink", "even_store_2-repartition", null, null, null, "even_store_2-repartition-filter"); builder.addSource(null, "even_store_2-repartition-source", null, null, null, "even_store_2-repartition"); From 5b4e4158fa28a00b1611260a7fccb36bfdc55ef2 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Fri, 27 Mar 2020 08:56:52 +0200 Subject: [PATCH 63/68] KAFKA-8611 / remove KeyValueMapper overloads for KStream#repartition operation --- .../apache/kafka/streams/kstream/KStream.java | 49 ------------------- .../kstream/internals/KStreamImpl.java | 36 ++------------ .../KStreamRepartitionIntegrationTest.java | 6 ++- 3 files changed, 8 insertions(+), 83 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java index ec4b69ac386d9..49aa19a579298 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java @@ -885,55 +885,6 @@ KStream through(final String topic, */ KStream repartition(final Repartitioned repartitioned); - /** - * Materialize this stream to an auto-generated repartition topic and create a new {@code KStream} - * from the auto-generated topic using default serializers, deserializers, and producers {@link DefaultPartitioner}. - * The number of partitions is determined based on the upstream topics partition numbers. - *

- * This operation is similar to {@link #through(String)}, however, Kafka Streams manages the used topic automatically. - * The Created topic is considered as an internal topic and is meant to be used only by current Kafka Streams instance. - * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be purged by Kafka Streams explicitly. - * The topic will be named as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in - * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, - * "<name>" is an internally generated name, and "-repartition" is a fixed suffix. - *

- * The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. - * Thus, an input record {@code } can be transformed into an output record {@code }. - * - * @param keySelector a {@link KeyValueMapper} that computes a new key for each record - * @return a {@code KStream} that contains records with new key (possibly of different type) and unmodified value - * @see #through(String) - * @see #through(String, Produced) - */ - KStream repartition(final KeyValueMapper keySelector); - - /** - * Materialize this stream to an auto-generated repartition topic and create a new {@code KStream} - * from the auto-generated topic using {@link Serde key serde}, {@link Serde value serde}, {@link StreamPartitioner}, - * number of partitions, and topic name part as defined by {@link Repartitioned}. - *

- * This operation is similar to {@link #through(String)}, however, Kafka Streams manages the used topic automatically. - * The Created topic is considered as an internal topic and is meant to be used only by current Kafka Streams instance. - * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be purged by Kafka Streams explicitly. - * The topic will be named as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in - * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, - * "<name>" is either provided via {@link Repartitioned#as(String)} or an internally - * generated name, and "-repartition" is a fixed suffix. - *

- * The provided {@link KeyValueMapper} is applied to each input record and computes a new key for it. - * Thus, an input record {@code } can be transformed into an output record {@code }. - * - * @param keySelector a {@link KeyValueMapper} that computes a new key for each record - * @param repartitioned the {@link Repartitioned} instance used to specify {@link Serdes}, - * {@link StreamPartitioner} which determines how records are distributed among partitions of the topic, - * part of the topic name, and number of partitions for a repartition topic. - * @return a {@code KStream} that contains records with new key (possibly of different type) and unmodified value - * @see #through(String) - * @see #through(String, Produced) - */ - KStream repartition(final KeyValueMapper keySelector, - final Repartitioned repartitioned); - /** * Materialize this stream to a topic using default serializers specified in the config and producer's * {@link DefaultPartitioner}. diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 9cd972c2ef96c..c0bb3c5294cfe 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -565,31 +565,15 @@ public KStream through(final String topic, @Override public KStream repartition() { - return doRepartition(Repartitioned.as(null), null); + return doRepartition(Repartitioned.as(null)); } @Override public KStream repartition(final Repartitioned repartitioned) { - return doRepartition(repartitioned, null); + return doRepartition(repartitioned); } - @Override - public KStream repartition(final KeyValueMapper keySelector) { - Objects.requireNonNull(keySelector, "selector can't be null"); - - return doRepartition(Repartitioned.as(null), keySelector); - } - - @Override - public KStream repartition(final KeyValueMapper keySelector, - final Repartitioned repartitioned) { - Objects.requireNonNull(keySelector, "selector can't be null"); - - return doRepartition(repartitioned, keySelector); - } - - private KStream doRepartition(final Repartitioned repartitioned, - final KeyValueMapper selector) { + private KStream doRepartition(final Repartitioned repartitioned) { Objects.requireNonNull(repartitioned, "repartitioned can't be null"); final RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); @@ -615,19 +599,7 @@ private KStream doRepartition(final Repartitioned repartition final UnoptimizableRepartitionNode unoptimizableRepartitionNode = unoptimizableRepartitionNodeBuilder.build(); - if (selector == null) { - builder.addGraphNode(streamsGraphNode, unoptimizableRepartitionNode); - } else { - final NamedInternal namedInternal = new NamedInternal(repartitionedInternal.name()); - - final ProcessorGraphNode selectKeyNode = internalSelectKey(selector, namedInternal); - - selectKeyNode.keyChangingOperation(true); - - builder.addGraphNode(streamsGraphNode, selectKeyNode); - - builder.addGraphNode(selectKeyNode, unoptimizableRepartitionNode); - } + builder.addGraphNode(streamsGraphNode, unoptimizableRepartitionNode); final Set sourceNodes = new HashSet<>(); sourceNodes.add(unoptimizableRepartitionNode.nodeName()); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 3f39a7861b9b6..f8cbe9f33aa42 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -409,7 +409,8 @@ public void shouldPerformSelectKeyWithRepartitionOperation() throws ExecutionExc final StreamsBuilder builder = new StreamsBuilder(); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition((key, value) -> Integer.valueOf(value)) + .selectKey((key, value) -> Integer.valueOf(value)) + .repartition() .to(outputTopic); startStreams(builder); @@ -483,7 +484,8 @@ public void shouldPerformKeySelectOperationWhenRepartitionOperationIsUsedWithKey .withKeySerde(Serdes.String()); builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition((key, value) -> key.toString(), repartitioned) + .selectKey((key, value) -> key.toString(), Named.as(repartitionedName)) + .repartition(repartitioned) .groupByKey() .count() .toStream() From 5054ef1dd61b88e583347d0b1222c7dad862d0bd Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Fri, 27 Mar 2020 08:58:26 +0200 Subject: [PATCH 64/68] KAFKA-8611 / update javadocs --- .../main/java/org/apache/kafka/streams/kstream/KStream.java | 4 ---- .../org/apache/kafka/streams/kstream/Repartitioned.java | 6 ------ 2 files changed, 10 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java index 49aa19a579298..d6c237eaafc96 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java @@ -817,8 +817,6 @@ KStream flatMapValues(final ValueMapperWithKey through(final String topic); @@ -839,8 +837,6 @@ KStream flatMapValues(final ValueMapperWithKey through(final String topic, final Produced produced); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java index ad5d8d070450f..40f66f036c603 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/Repartitioned.java @@ -29,8 +29,6 @@ * @param value type * @see KStream#repartition() * @see KStream#repartition(Repartitioned) - * @see KStream#repartition(KeyValueMapper) - * @see KStream#repartition(KeyValueMapper, Repartitioned) */ public class Repartitioned implements NamedOperation> { @@ -70,7 +68,6 @@ protected Repartitioned(final Repartitioned repartitioned) { * @param value type * @return A new {@code Repartitioned} instance configured with processor name and repartition topic name * @see KStream#repartition(Repartitioned) - * @see KStream#repartition(KeyValueMapper, Repartitioned) */ public static Repartitioned as(final String name) { return new Repartitioned<>(name, null, null, null, null); @@ -85,7 +82,6 @@ public static Repartitioned as(final String name) { * @param value type * @return A new {@code Repartitioned} instance configured with key serde and value serde * @see KStream#repartition(Repartitioned) - * @see KStream#repartition(KeyValueMapper, Repartitioned) */ public static Repartitioned with(final Serde keySerde, final Serde valueSerde) { @@ -102,7 +98,6 @@ public static Repartitioned with(final Serde keySerde, * @param value type * @return A new {@code Repartitioned} instance configured with partitioner * @see KStream#repartition(Repartitioned) - * @see KStream#repartition(KeyValueMapper, Repartitioned) */ public static Repartitioned streamPartitioner(final StreamPartitioner partitioner) { return new Repartitioned<>(null, null, null, null, partitioner); @@ -116,7 +111,6 @@ public static Repartitioned streamPartitioner(final StreamPartition * @param value type * @return A new {@code Repartitioned} instance configured number of partitions * @see KStream#repartition(Repartitioned) - * @see KStream#repartition(KeyValueMapper, Repartitioned) */ public static Repartitioned numberOfPartitions(final int numberOfPartitions) { return new Repartitioned<>(null, null, null, numberOfPartitions, null); From cdbd24c877f6340d013a9e1eda719f3e7d90be51 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Wed, 1 Apr 2020 09:17:58 +0300 Subject: [PATCH 65/68] KAFKA-8611 / revert accidental code change Co-Authored-By: John Roesler --- .../kafka/streams/kstream/internals/graph/StreamsGraphTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/graph/StreamsGraphTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/graph/StreamsGraphTest.java index 956c23880c695..c01bcccc2a4f7 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/graph/StreamsGraphTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/graph/StreamsGraphTest.java @@ -129,7 +129,7 @@ public void shouldNotOptimizeWhenAThroughOperationIsDone() { public void shouldOptimizeSeveralMergeNodesWithCommonKeyChangingParent() { final StreamsBuilder streamsBuilder = new StreamsBuilder(); final KStream parentStream = streamsBuilder.stream("input_topic", Consumed.with(Serdes.Integer(), Serdes.Integer())) - .selectKey(Integer::sum); + .selectKey(Integer::sum); final KStream childStream1 = parentStream.mapValues(v -> v + 1); final KStream childStream2 = parentStream.mapValues(v -> v + 2); From 591e8ff391bf54fac1ebce9ea6962330de3cd2a4 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Fri, 3 Apr 2020 09:57:32 +0300 Subject: [PATCH 66/68] KAFKA-8611 / CR comments --- .../apache/kafka/streams/kstream/KStream.java | 10 +++---- .../CogroupedStreamAggregateBuilder.java | 1 + .../GroupedStreamAggregateBuilder.java | 1 + .../internals/InternalStreamsBuilder.java | 1 + .../kstream/internals/KStreamImpl.java | 26 ++++++------------- .../GroupedTableOperationRepartitionNode.java | 8 +++--- .../graph/OptimizableRepartitionNode.java | 8 +++--- .../internals/InternalTopicConfig.java | 7 ++--- 8 files changed, 24 insertions(+), 38 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java index d6c237eaafc96..755a090a53e5c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java @@ -843,12 +843,12 @@ KStream through(final String topic, /** * Materialize this stream to an auto-generated repartition topic and create a new {@code KStream} - * from the auto-generated topic using default serializers, deserializers, and producers {@link DefaultPartitioner}. + * from the auto-generated topic using default serializers, deserializers, and producer's {@link DefaultPartitioner}. * The number of partitions is determined based on the upstream topics partition numbers. *

* This operation is similar to {@link #through(String)}, however, Kafka Streams manages the used topic automatically. - * The Created topic is considered as an internal topic and is meant to be used only by current Kafka Streams instance. - * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be purged by Kafka Streams explicitly. + * The created topic is considered as an internal topic and is meant to be used only by the current Kafka Streams instance. + * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be automatically purged by Kafka Streams. * The topic will be named as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, * "<name>" is an internally generated name, and "-repartition" is a fixed suffix. @@ -865,8 +865,8 @@ KStream through(final String topic, * number of partitions, and topic name part as defined by {@link Repartitioned}. *

* This operation is similar to {@link #through(String)}, however, Kafka Streams manages the used topic automatically. - * The Created topic is considered as an internal topic and is meant to be used only by current Kafka Streams instance. - * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be purged by Kafka Streams explicitly. + * The created topic is considered as an internal topic and is meant to be used only by the current Kafka Streams instance. + * Similar to auto-repartitioning, the topic will be created with infinite retention time and data will be automatically purged by Kafka Streams. * The topic will be named as "${applicationId}-<name>-repartition", where "applicationId" is user-specified in * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, * "<name>" is either provided via {@link Repartitioned#as(String)} or an internally diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/CogroupedStreamAggregateBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/CogroupedStreamAggregateBuilder.java index 81f2d640f62b5..4fdb1f25a2aa8 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/CogroupedStreamAggregateBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/CogroupedStreamAggregateBuilder.java @@ -177,6 +177,7 @@ private void createRepartitionSource(final String repartitionTopicNamePref keySerde, (Serde) valueSerde, repartitionTopicNamePrefix, + null, (OptimizableRepartitionNodeBuilder) optimizableRepartitionNodeBuilder); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java index 3f3c1995ae193..861dfaed14431 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java @@ -126,6 +126,7 @@ private String createRepartitionSource(final String repartitionTopicNamePrefix, keySerde, valueSerde, repartitionTopicNamePrefix, + null, optimizableRepartitionNodeBuilder); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java index a374c42f905d0..ee092924f162b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilder.java @@ -435,6 +435,7 @@ private OptimizableRepartitionNode createRepartitionNode(final Stri keySerde, valueSerde, repartitionTopicName, + null, repartitionNodeBuilder ); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index c0bb3c5294cfe..4d5f6cf23a9be 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -573,17 +573,18 @@ public KStream repartition(final Repartitioned repartitioned) { return doRepartition(repartitioned); } - private KStream doRepartition(final Repartitioned repartitioned) { + private KStream doRepartition(final Repartitioned repartitioned) { Objects.requireNonNull(repartitioned, "repartitioned can't be null"); - final RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); + final RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned); final String name = repartitionedInternal.name() != null ? repartitionedInternal.name() : builder .newProcessorName(REPARTITION_NAME); final Serde valueSerde = repartitionedInternal.valueSerde() == null ? valSerde : repartitionedInternal.valueSerde(); + final Serde keySerde = repartitionedInternal.keySerde() == null ? this.keySerde : repartitionedInternal.keySerde(); - final UnoptimizableRepartitionNodeBuilder unoptimizableRepartitionNodeBuilder = UnoptimizableRepartitionNode + final UnoptimizableRepartitionNodeBuilder unoptimizableRepartitionNodeBuilder = UnoptimizableRepartitionNode .unoptimizableRepartitionNodeBuilder(); final InternalTopicProperties internalTopicProperties = repartitionedInternal.toInternalTopicProperties(); @@ -597,7 +598,7 @@ private KStream doRepartition(final Repartitioned repartition unoptimizableRepartitionNodeBuilder.withInternalTopicProperties(internalTopicProperties) ); - final UnoptimizableRepartitionNode unoptimizableRepartitionNode = unoptimizableRepartitionNodeBuilder.build(); + final UnoptimizableRepartitionNode unoptimizableRepartitionNode = unoptimizableRepartitionNodeBuilder.build(); builder.addGraphNode(streamsGraphNode, unoptimizableRepartitionNode); @@ -606,7 +607,7 @@ private KStream doRepartition(final Repartitioned repartition return new KStreamImpl<>( repartitionSourceName, - repartitionedInternal.keySerde(), + keySerde, valueSerde, Collections.unmodifiableSet(sourceNodes), false, @@ -713,6 +714,7 @@ public KTable toTable(final Named named, keySerdeOverride, valueSerdeOverride, name, + null, repartitionNodeBuilder ); @@ -984,6 +986,7 @@ private KStreamImpl repartitionForJoin(final String repartitionName, repartitionKeySerde, repartitionValueSerde, repartitionName, + null, optimizableRepartitionNodeBuilder); final OptimizableRepartitionNode optimizableRepartitionNode = optimizableRepartitionNodeBuilder.build(); @@ -999,19 +1002,6 @@ private KStreamImpl repartitionForJoin(final String repartitionName, builder); } - static > String createRepartitionedSource(final InternalStreamsBuilder builder, - final Serde keySerde, - final Serde valSerde, - final String repartitionTopicNamePrefix, - final BaseRepartitionNodeBuilder baseRepartitionNodeBuilder) { - return createRepartitionedSource(builder, - keySerde, - valSerde, - repartitionTopicNamePrefix, - null, - baseRepartitionNodeBuilder); - } - static > String createRepartitionedSource(final InternalStreamsBuilder builder, final Serde keySerde, final Serde valSerde, diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java index 6d01a5fb8ea11..1b88f2cbf968a 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/GroupedTableOperationRepartitionNode.java @@ -35,8 +35,7 @@ private GroupedTableOperationRepartitionNode(final String nodeName, final String sinkName, final String sourceName, final String repartitionTopic, - final ProcessorParameters processorParameters, - final InternalTopicProperties internalTopicProperties) { + final ProcessorParameters processorParameters) { super( nodeName, @@ -47,7 +46,7 @@ private GroupedTableOperationRepartitionNode(final String nodeName, sinkName, repartitionTopic, null, - internalTopicProperties + InternalTopicProperties.empty() ); } @@ -117,8 +116,7 @@ public GroupedTableOperationRepartitionNode build() { sinkName, sourceName, repartitionTopic, - processorParameters, - internalTopicProperties + processorParameters ); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java index 27abaa942b22e..24dc8da9fd1bc 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/graph/OptimizableRepartitionNode.java @@ -32,8 +32,7 @@ private OptimizableRepartitionNode(final String nodeName, final Serde valueSerde, final String sinkName, final String repartitionTopic, - final StreamPartitioner partitioner, - final InternalTopicProperties internalTopicProperties) { + final StreamPartitioner partitioner) { super( nodeName, sourceName, @@ -43,7 +42,7 @@ private OptimizableRepartitionNode(final String nodeName, sinkName, repartitionTopic, partitioner, - internalTopicProperties + InternalTopicProperties.empty() ); } @@ -112,8 +111,7 @@ public OptimizableRepartitionNode build() { valueSerde, sinkName, repartitionTopic, - partitioner, - internalTopicProperties + partitioner ); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java index 95d279c35014e..befdebe15fada 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java @@ -53,13 +53,10 @@ public abstract class InternalTopicConfig { final Map topicConfigs, final int numberOfPartitions, final boolean enforceNumberOfPartitions) { - Objects.requireNonNull(name, "name can't be null"); + this.name = Objects.requireNonNull(name, "name can't be null"); Topic.validate(name); - validateNumberOfPartitions(numberOfPartitions); - - this.name = name; - this.topicConfigs = topicConfigs; + this.topicConfigs = Objects.requireNonNull(topicConfigs, "topicConfigs can't be null"); this.numberOfPartitions = Optional.of(numberOfPartitions); this.enforceNumberOfPartitions = enforceNumberOfPartitions; } From 9fdc7ecc901dccade4d45660078172a7b58701c6 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Sat, 4 Apr 2020 02:00:57 +0300 Subject: [PATCH 67/68] KAFKA-8611 / CR comments: part 2 --- .../internals/InternalTopicConfig.java | 8 +- .../KStreamRepartitionIntegrationTest.java | 102 +++++------------- .../internals/KStreamRepartitionTest.java | 47 ++++---- .../internals/InternalTopicConfigTest.java | 5 + 4 files changed, 63 insertions(+), 99 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java index befdebe15fada..0b5a32799d8e9 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopicConfig.java @@ -41,11 +41,9 @@ public abstract class InternalTopicConfig { } InternalTopicConfig(final String name, final Map topicConfigs) { - Objects.requireNonNull(name, "name can't be null"); + this.name = Objects.requireNonNull(name, "name can't be null"); Topic.validate(name); - - this.name = name; - this.topicConfigs = topicConfigs; + this.topicConfigs = Objects.requireNonNull(topicConfigs, "topicConfigs can't be null"); this.enforceNumberOfPartitions = false; } @@ -53,7 +51,7 @@ public abstract class InternalTopicConfig { final Map topicConfigs, final int numberOfPartitions, final boolean enforceNumberOfPartitions) { - this.name = Objects.requireNonNull(name, "name can't be null"); + this.name = Objects.requireNonNull(name, "name can't be null"); Topic.validate(name); validateNumberOfPartitions(numberOfPartitions); this.topicConfigs = Objects.requireNonNull(topicConfigs, "topicConfigs can't be null"); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index f8cbe9f33aa42..25d8cc1c40dbb 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -61,7 +61,6 @@ import java.util.Set; import java.util.TreeMap; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -83,15 +82,23 @@ public class KStreamRepartitionIntegrationTest { @ClassRule public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS); - private final String inputTopic = "input-topic-" + TEST_NUM.get(); - private final String outputTopic = "output-topic-" + TEST_NUM.get(); - private final String applicationId = "kstream-repartition-stream-test-" + TEST_NUM.get(); + private String topicB; + private String inputTopic; + private String outputTopic; + private String applicationId; private Properties streamsConfiguration; private List kafkaStreamsInstances; @Before public void before() throws InterruptedException { + final int testNum = TEST_NUM.incrementAndGet(); + + topicB = "topic-b-" + testNum; + inputTopic = "input-topic-" + testNum; + outputTopic = "output-topic-" + testNum; + applicationId = "kstream-repartition-stream-test-" + testNum; + CLUSTER.createTopic(inputTopic, 4, 1); CLUSTER.createTopic(outputTopic, 1, 1); @@ -115,12 +122,10 @@ public void whenShuttingDown() throws IOException { .forEach(KafkaStreams::close); IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration); - TEST_NUM.incrementAndGet(); } @Test public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationsDoNotMatchWhenJoining() throws InterruptedException { - final String topicB = "topic-b-" + TEST_NUM.get(); final String topicBRepartitionedName = "topic-b-scale-up"; final String inputTopicRepartitionedName = "input-topic-scale-up"; final int topicBNumberOfPartitions = 2; @@ -165,7 +170,6 @@ public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationsD @Test public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDoNotMatchSourceTopicWhenJoining() throws InterruptedException { - final String topicB = "topic-b-" + TEST_NUM.get(); final int topicBNumberOfPartitions = 6; final String inputTopicRepartitionName = "join-repartition-test"; final AtomicReference expectedThrowable = new AtomicReference<>(); @@ -201,9 +205,8 @@ public void shouldThrowAnExceptionWhenNumberOfPartitionsOfRepartitionOperationDo } @Test - public void shouldDeductNumberOfPartitionsFromRepartitionOperation() throws InterruptedException, ExecutionException { + public void shouldDeductNumberOfPartitionsFromRepartitionOperation() throws Exception { final String topicBMapperName = "topic-b-mapper"; - final String topicB = "topic-b-" + TEST_NUM.get(); final int topicBNumberOfPartitions = 6; final String inputTopicRepartitionName = "join-repartition-test"; final int inputTopicRepartitionedNumOfPartitions = 3; @@ -253,8 +256,7 @@ public void shouldDeductNumberOfPartitionsFromRepartitionOperation() throws Inte } @Test - public void shouldDoProperJoiningWhenNumberOfPartitionsAreValidWhenUsingRepartitionOperation() throws ExecutionException, InterruptedException { - final String topicB = "topic-b-" + TEST_NUM.get(); + public void shouldDoProperJoiningWhenNumberOfPartitionsAreValidWhenUsingRepartitionOperation() throws Exception { final String topicBRepartitionedName = "topic-b-scale-up"; final String inputTopicRepartitionedName = "input-topic-scale-up"; @@ -302,59 +304,7 @@ public void shouldDoProperJoiningWhenNumberOfPartitionsAreValidWhenUsingRepartit } @Test - public void shouldDoJoiningWhenNumberOfPartitionsOfRepartitionOperationMatchesNumberOfPartitionsOfSourceTopic() throws ExecutionException, InterruptedException { - final String topicBMapperName = "topic-b-mapper"; - final String topicB = "topic-b-" + TEST_NUM.get(); - final int topicBNumberOfPartitions = 6; - final String inputTopicRepartitionName = "join-repartition-test"; - - final long timestamp = System.currentTimeMillis(); - - CLUSTER.createTopic(topicB, topicBNumberOfPartitions, 1); - - final List> expectedRecords = Arrays.asList( - new KeyValue<>(1, "A"), - new KeyValue<>(2, "B") - ); - - sendEvents(inputTopic, timestamp, expectedRecords); - sendEvents(topicB, timestamp, expectedRecords); - - final StreamsBuilder builder = new StreamsBuilder(); - - final Repartitioned inputTopicRepartitioned = Repartitioned - .as(inputTopicRepartitionName) - .withNumberOfPartitions(6); - - final KStream topicBStream = builder - .stream(topicB, Consumed.with(Serdes.Integer(), Serdes.String())) - .map(KeyValue::new, Named.as(topicBMapperName)); - - builder.stream(inputTopic, Consumed.with(Serdes.Integer(), Serdes.String())) - .repartition(inputTopicRepartitioned) - .join(topicBStream, (value1, value2) -> value2, JoinWindows.of(Duration.ofSeconds(10))) - .to(outputTopic); - - startStreams(builder); - - final String repartitionTopicName = toRepartitionTopicName(inputTopicRepartitionName); - final String mapperRepartitionTopicName = toRepartitionTopicName(topicBMapperName); - - final int mapperNumOfPartitions = getNumberOfPartitionsForTopic(mapperRepartitionTopicName); - final int repartitionTopicNumOfPartitions = getNumberOfPartitionsForTopic(repartitionTopicName); - - validateReceivedMessages( - new IntegerDeserializer(), - new StringDeserializer(), - expectedRecords - ); - - assertEquals(topicBNumberOfPartitions, mapperNumOfPartitions); - assertEquals(topicBNumberOfPartitions, repartitionTopicNumOfPartitions); - } - - @Test - public void shouldUseStreamPartitionerForRepartitionOperation() throws ExecutionException, InterruptedException { + public void shouldUseStreamPartitionerForRepartitionOperation() throws Exception { final int partition = 1; final String repartitionName = "partitioner-test"; final long timestamp = System.currentTimeMillis(); @@ -395,7 +345,7 @@ public void shouldUseStreamPartitionerForRepartitionOperation() throws Execution } @Test - public void shouldPerformSelectKeyWithRepartitionOperation() throws ExecutionException, InterruptedException { + public void shouldPerformSelectKeyWithRepartitionOperation() throws Exception { final long timestamp = System.currentTimeMillis(); sendEvents( @@ -430,7 +380,7 @@ public void shouldPerformSelectKeyWithRepartitionOperation() throws ExecutionExc } @Test - public void shouldCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() throws ExecutionException, InterruptedException { + public void shouldCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() throws Exception { final String repartitionName = "dummy"; final long timestamp = System.currentTimeMillis(); @@ -466,7 +416,7 @@ public void shouldCreateRepartitionTopicIfKeyChangingOperationWasNotPerformed() } @Test - public void shouldPerformKeySelectOperationWhenRepartitionOperationIsUsedWithKeySelector() throws ExecutionException, InterruptedException { + public void shouldPerformKeySelectOperationWhenRepartitionOperationIsUsedWithKeySelector() throws Exception { final String repartitionedName = "new-key"; final long timestamp = System.currentTimeMillis(); @@ -511,7 +461,7 @@ public void shouldPerformKeySelectOperationWhenRepartitionOperationIsUsedWithKey } @Test - public void shouldCreateRepartitionTopicWithSpecifiedNumberOfPartitions() throws ExecutionException, InterruptedException { + public void shouldCreateRepartitionTopicWithSpecifiedNumberOfPartitions() throws Exception { final String repartitionName = "new-partitions"; final long timestamp = System.currentTimeMillis(); @@ -550,7 +500,7 @@ public void shouldCreateRepartitionTopicWithSpecifiedNumberOfPartitions() throws } @Test - public void shouldInheritRepartitionTopicPartitionNumberFromUpstreamTopicWhenNumberOfPartitionsIsNotSpecified() throws InterruptedException, ExecutionException { + public void shouldInheritRepartitionTopicPartitionNumberFromUpstreamTopicWhenNumberOfPartitionsIsNotSpecified() throws Exception { final String repartitionName = "new-topic"; final long timestamp = System.currentTimeMillis(); @@ -589,7 +539,7 @@ public void shouldInheritRepartitionTopicPartitionNumberFromUpstreamTopicWhenNum } @Test - public void shouldCreateOnlyOneRepartitionTopicWhenRepartitionIsFollowedByGroupByKey() throws ExecutionException, InterruptedException { + public void shouldCreateOnlyOneRepartitionTopicWhenRepartitionIsFollowedByGroupByKey() throws Exception { final String repartitionName = "new-partitions"; final long timestamp = System.currentTimeMillis(); @@ -634,7 +584,7 @@ public void shouldCreateOnlyOneRepartitionTopicWhenRepartitionIsFollowedByGroupB } @Test - public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws ExecutionException, InterruptedException { + public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws Exception { final long timestamp = System.currentTimeMillis(); sendEvents( @@ -669,7 +619,7 @@ public void shouldGenerateRepartitionTopicWhenNameIsNotSpecified() throws Execut } @Test - public void shouldGoThroughRebalancingCorrectly() throws ExecutionException, InterruptedException { + public void shouldGoThroughRebalancingCorrectly() throws Exception { final String repartitionName = "rebalancing-test"; final long timestamp = System.currentTimeMillis(); @@ -733,7 +683,7 @@ public void shouldGoThroughRebalancingCorrectly() throws ExecutionException, Int assertEquals(2, getNumberOfPartitionsForTopic(repartitionTopicName)); } - private int getNumberOfPartitionsForTopic(final String topic) throws ExecutionException, InterruptedException { + private int getNumberOfPartitionsForTopic(final String topic) throws Exception { try (final AdminClient adminClient = createAdminClient()) { final TopicDescription topicDescription = adminClient.describeTopics(Collections.singleton(topic)) .values() @@ -744,7 +694,7 @@ private int getNumberOfPartitionsForTopic(final String topic) throws ExecutionEx } } - private boolean topicExists(final String topic) throws InterruptedException, ExecutionException { + private boolean topicExists(final String topic) throws Exception { try (final AdminClient adminClient = createAdminClient()) { final Set topics = adminClient.listTopics() .names() @@ -778,13 +728,13 @@ private static int countOccurrencesInTopology(final String topologyString, } private void sendEvents(final long timestamp, - final List> events) throws ExecutionException, InterruptedException { + final List> events) throws Exception { sendEvents(inputTopic, timestamp, events); } private void sendEvents(final String topic, final long timestamp, - final List> events) throws ExecutionException, InterruptedException { + final List> events) throws Exception { IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp( topic, events, diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java index 6a2a37613b6fd..1aeea2e473c81 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamRepartitionTest.java @@ -16,18 +16,19 @@ */ package org.apache.kafka.streams.kstream.internals; +import org.apache.kafka.common.serialization.IntegerDeserializer; import org.apache.kafka.common.serialization.IntegerSerializer; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.streams.KeyValueTimestamp; import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.TestInputTopic; +import org.apache.kafka.streams.TestOutputTopic; import org.apache.kafka.streams.TopologyTestDriver; -import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.Repartitioned; import org.apache.kafka.streams.processor.StreamPartitioner; -import org.apache.kafka.test.MockProcessor; -import org.apache.kafka.test.MockProcessorSupplier; +import org.apache.kafka.streams.test.TestRecord; import org.apache.kafka.test.StreamsTestUtils; import org.easymock.EasyMock; import org.easymock.EasyMockRunner; @@ -35,6 +36,7 @@ import org.junit.Test; import org.junit.runner.RunWith; +import java.time.Instant; import java.util.Properties; import static org.easymock.EasyMock.anyInt; @@ -43,6 +45,9 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertTrue; @RunWith(EasyMockRunner.class) public class KStreamRepartitionTest { @@ -66,34 +71,40 @@ public void shouldInvokePartitionerWhenSet() { expect(streamPartitionerMock.partition(anyString(), eq(1), eq("X1"), anyInt())).andReturn(1).times(1); replay(streamPartitionerMock); + final String repartitionOperationName = "test"; final Repartitioned repartitioned = Repartitioned .streamPartitioner(streamPartitionerMock) - .withName("repartition"); + .withName(repartitionOperationName); - final KStream repartitionedStream = builder.stream(inputTopic) + builder.stream(inputTopic) .repartition(repartitioned); - final MockProcessorSupplier supplier = new MockProcessorSupplier<>(); + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { + final TestInputTopic testInputTopic = driver.createInputTopic(inputTopic, + new IntegerSerializer(), + new StringSerializer()); - repartitionedStream.process(supplier); + final String topicName = repartitionOutputTopic(props, repartitionOperationName); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) { - final TestInputTopic testInputTopic = driver.createInputTopic( - inputTopic, - new IntegerSerializer(), - new StringSerializer() + final TestOutputTopic testOutputTopic = driver.createOutputTopic( + topicName, + new IntegerDeserializer(), + new StringDeserializer() ); for (int i = 0; i < 2; i++) { testInputTopic.pipeInput(expectedKeys[i], "X" + expectedKeys[i], i + 10); } - final MockProcessor proc = supplier.theCapturedProcessor(); - proc.checkAndClearProcessResult( - new KeyValueTimestamp<>(0, "X0", 10), - new KeyValueTimestamp<>(1, "X1", 11) - ); + + assertThat(testOutputTopic.readRecord(), equalTo(new TestRecord<>(0, "X0", Instant.ofEpochMilli(10)))); + assertThat(testOutputTopic.readRecord(), equalTo(new TestRecord<>(1, "X1", Instant.ofEpochMilli(11)))); + assertTrue(testOutputTopic.readRecordsToList().isEmpty()); } verify(streamPartitionerMock); } + + private String repartitionOutputTopic(final Properties props, final String repartitionOperationName) { + return props.getProperty(StreamsConfig.APPLICATION_ID_CONFIG) + "-" + repartitionOperationName + "-repartition"; + } } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicConfigTest.java index 9af422c816dbc..25207958683dc 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicConfigTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopicConfigTest.java @@ -28,6 +28,11 @@ public class InternalTopicConfigTest { + @Test(expected = NullPointerException.class) + public void shouldThrowNpeIfTopicConfigIsNull() { + new RepartitionTopicConfig("topic", null); + } + @Test(expected = NullPointerException.class) public void shouldThrowIfNameIsNull() { new RepartitionTopicConfig(null, Collections.emptyMap()); From f2bcdfe487ec13f41f4516d6bee6a1847f6d4ce2 Mon Sep 17 00:00:00 2001 From: Levani Kokhreidze Date: Tue, 7 Apr 2020 17:49:57 +0300 Subject: [PATCH 68/68] KAFKA-8611 / Add optimize as test parameter --- .../KStreamRepartitionIntegrationTest.java | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java index 25d8cc1c40dbb..907d4c830102b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamRepartitionIntegrationTest.java @@ -47,12 +47,16 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; import java.io.IOException; import java.lang.Thread.UncaughtExceptionHandler; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -74,6 +78,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +@RunWith(value = Parameterized.class) @Category({IntegrationTest.class}) public class KStreamRepartitionIntegrationTest { private static final int NUM_BROKERS = 1; @@ -90,6 +95,22 @@ public class KStreamRepartitionIntegrationTest { private Properties streamsConfiguration; private List kafkaStreamsInstances; + @Parameters(name = "Optimization = {0}") + public static Collection data() { + final List values = new ArrayList<>(); + + Arrays.asList(StreamsConfig.OPTIMIZE, StreamsConfig.NO_OPTIMIZATION) + .forEach(x -> values.add(new Object[]{x})); + + return values; + } + + public KStreamRepartitionIntegrationTest(final String topologyOptimization) { + streamsConfiguration = new Properties(); + kafkaStreamsInstances = new ArrayList<>(); + streamsConfiguration.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, topologyOptimization); + } + @Before public void before() throws InterruptedException { final int testNum = TEST_NUM.incrementAndGet(); @@ -102,9 +123,6 @@ public void before() throws InterruptedException { CLUSTER.createTopic(inputTopic, 4, 1); CLUSTER.createTopic(outputTopic, 1, 1); - streamsConfiguration = new Properties(); - kafkaStreamsInstances = new ArrayList<>(); - streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId); streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()); @@ -112,7 +130,6 @@ public void before() throws InterruptedException { streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100); streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass()); streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); - streamsConfiguration.put(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.OPTIMIZE); } @After