KAFKA-4936: Add dynamic routing in Streams #5018
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,6 +26,7 @@ | |
| import org.apache.kafka.streams.processor.StateStore; | ||
| import org.apache.kafka.streams.processor.StreamPartitioner; | ||
| import org.apache.kafka.streams.processor.TimestampExtractor; | ||
| import org.apache.kafka.streams.processor.TopicNameExtractor; | ||
| import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; | ||
| import org.apache.kafka.streams.processor.internals.ProcessorNode; | ||
| import org.apache.kafka.streams.processor.internals.ProcessorTopology; | ||
|
|
@@ -411,7 +412,8 @@ public synchronized Topology addSource(final AutoOffsetReset offsetReset, | |
| * @param parentNames the name of one or more source or processor nodes whose output records this sink should consume | ||
| * and write to its topic | ||
| * @return itself | ||
| * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name, | ||
| * or if this processor's name is equal to the parent's name | ||
| * @see #addSink(String, String, StreamPartitioner, String...) | ||
| * @see #addSink(String, String, Serializer, Serializer, String...) | ||
| * @see #addSink(String, String, Serializer, Serializer, StreamPartitioner, String...) | ||
|
|
@@ -443,7 +445,8 @@ public synchronized Topology addSink(final String name, | |
| * @param parentNames the name of one or more source or processor nodes whose output records this sink should consume | ||
| * and write to its topic | ||
| * @return itself | ||
| * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name, | ||
| * or if this processor's name is equal to the parent's name | ||
| * @see #addSink(String, String, String...) | ||
| * @see #addSink(String, String, Serializer, Serializer, String...) | ||
| * @see #addSink(String, String, Serializer, Serializer, StreamPartitioner, String...) | ||
|
|
@@ -471,7 +474,8 @@ public synchronized <K, V> Topology addSink(final String name, | |
| * @param parentNames the name of one or more source or processor nodes whose output records this sink should consume | ||
| * and write to its topic | ||
| * @return itself | ||
| * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name, | ||
| * or if this processor's name is equal to the parent's name | ||
| * @see #addSink(String, String, String...) | ||
| * @see #addSink(String, String, StreamPartitioner, String...) | ||
| * @see #addSink(String, String, Serializer, Serializer, StreamPartitioner, String...) | ||
|
|
@@ -501,7 +505,8 @@ public synchronized <K, V> Topology addSink(final String name, | |
| * @param parentNames the name of one or more source or processor nodes whose output records this sink should consume | ||
| * and write to its topic | ||
| * @return itself | ||
| * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name, | ||
| * or if this processor's name is equal to the parent's name | ||
| * @see #addSink(String, String, String...) | ||
| * @see #addSink(String, String, StreamPartitioner, String...) | ||
| * @see #addSink(String, String, Serializer, Serializer, String...) | ||
|
|
@@ -516,6 +521,130 @@ public synchronized <K, V> Topology addSink(final String name, | |
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * Add a new sink that forwards records from upstream parent processor and/or source nodes to Kafka topics dynamically. | ||
|
Member
There was a problem hiding this comment. IMHO, "dynamically" need a little bit more explanation. Should we also state, that all those topics must be created by the user manually?
Contributor
There was a problem hiding this comment. I agree with "dynamically"... maybe "based on the {@code topicExtractor}"? About the second point, I think that's addressed in the following line. |
||
| * The topics that it may ever send to should be pre-created. | ||
| * The sink will use the {@link StreamsConfig#DEFAULT_KEY_SERDE_CLASS_CONFIG default key serializer} and | ||
| * {@link StreamsConfig#DEFAULT_VALUE_SERDE_CLASS_CONFIG default value serializer} specified in the | ||
| * {@link StreamsConfig stream configuration}. | ||
| * | ||
| * @param name the unique name of the sink | ||
| * @param topicExtractor the extractor to determine the name of the Kafka topic to which this sink should write for each record | ||
| * @param parentNames the name of one or more source or processor nodes whose output records this sink should consume | ||
| * and dynamically write to topics | ||
| * @return itself | ||
| * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name, | ||
| * or if this processor's name is equal to the parent's name | ||
| * @see #addSink(String, String, StreamPartitioner, String...) | ||
| * @see #addSink(String, String, Serializer, Serializer, String...) | ||
| * @see #addSink(String, String, Serializer, Serializer, StreamPartitioner, String...) | ||
| */ | ||
| public synchronized <K, V> Topology addSink(final String name, | ||
| final TopicNameExtractor<K, V> topicExtractor, | ||
| final String... parentNames) { | ||
| internalTopologyBuilder.addSink(name, topicExtractor, null, null, null, parentNames); | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * Add a new sink that forwards records from upstream parent processor and/or source nodes to Kafka topics dynamically, | ||
|
|
||
| * using the supplied partitioner. | ||
| * The topics that it may ever send to should be pre-created. | ||
| * The sink will use the {@link StreamsConfig#DEFAULT_KEY_SERDE_CLASS_CONFIG default key serializer} and | ||
| * {@link StreamsConfig#DEFAULT_VALUE_SERDE_CLASS_CONFIG default value serializer} specified in the | ||
| * {@link StreamsConfig stream configuration}. | ||
| * <p> | ||
| * The sink will also use the specified {@link StreamPartitioner} to determine how records are distributed among | ||
| * the named Kafka topic's partitions. | ||
| * Such control is often useful with topologies that use {@link #addStateStore(StoreBuilder, String...) state | ||
| * stores} in its processors. | ||
| * In most other cases, however, a partitioner needs not be specified and Kafka will automatically distribute | ||
| * records among partitions using Kafka's default partitioning logic. | ||
| * | ||
| * @param name the unique name of the sink | ||
| * @param topicExtractor the extractor to determine the name of the Kafka topic to which this sink should write for each record | ||
| * @param partitioner the function that should be used to determine the partition for each record processed by the sink | ||
| * @param parentNames the name of one or more source or processor nodes whose output records this sink should consume | ||
| * and dynamically write to topics | ||
| * @return itself | ||
| * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name, | ||
| * or if this processor's name is equal to the parent's name | ||
| * @see #addSink(String, String, String...) | ||
| * @see #addSink(String, String, Serializer, Serializer, String...) | ||
| * @see #addSink(String, String, Serializer, Serializer, StreamPartitioner, String...) | ||
| */ | ||
| public synchronized <K, V> Topology addSink(final String name, | ||
| final TopicNameExtractor<K, V> topicExtractor, | ||
| final StreamPartitioner<? super K, ? super V> partitioner, | ||
| final String... parentNames) { | ||
| internalTopologyBuilder.addSink(name, topicExtractor, null, null, partitioner, parentNames); | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * Add a new sink that forwards records from upstream parent processor and/or source nodes to Kafka topics dynamically. | ||
|
|
||
| * The topics that it may ever send to should be pre-created. | ||
| * The sink will use the specified key and value serializers. | ||
| * | ||
| * @param name the unique name of the sink | ||
| * @param topicExtractor the extractor to determine the name of the Kafka topic to which this sink should write for each record | ||
| * @param keySerializer the {@link Serializer key serializer} used when consuming records; may be null if the sink | ||
| * should use the {@link StreamsConfig#DEFAULT_KEY_SERDE_CLASS_CONFIG default key serializer} specified in the | ||
| * {@link StreamsConfig stream configuration} | ||
| * @param valueSerializer the {@link Serializer value serializer} used when consuming records; may be null if the sink | ||
| * should use the {@link StreamsConfig#DEFAULT_VALUE_SERDE_CLASS_CONFIG default value serializer} specified in the | ||
| * {@link StreamsConfig stream configuration} | ||
| * @param parentNames the name of one or more source or processor nodes whose output records this sink should consume | ||
| * and dynamically write to topics | ||
| * @return itself | ||
| * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name, | ||
| * or if this processor's name is equal to the parent's name | ||
| * @see #addSink(String, String, String...) | ||
| * @see #addSink(String, String, StreamPartitioner, String...) | ||
| * @see #addSink(String, String, Serializer, Serializer, StreamPartitioner, String...) | ||
| */ | ||
| public synchronized <K, V> Topology addSink(final String name, | ||
| final TopicNameExtractor<K, V> topicExtractor, | ||
| final Serializer<K> keySerializer, | ||
| final Serializer<V> valueSerializer, | ||
| final String... parentNames) { | ||
| internalTopologyBuilder.addSink(name, topicExtractor, keySerializer, valueSerializer, null, parentNames); | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * Add a new sink that forwards records from upstream parent processor and/or source nodes to Kafka topics dynamically. | ||
|
|
||
| * The topics that it may ever send to should be pre-created. | ||
| * The sink will use the specified key and value serializers, and the supplied partitioner. | ||
| * | ||
| * @param name the unique name of the sink | ||
| * @param topicExtractor the extractor to determine the name of the Kafka topic to which this sink should write for each record | ||
| * @param keySerializer the {@link Serializer key serializer} used when consuming records; may be null if the sink | ||
| * should use the {@link StreamsConfig#DEFAULT_KEY_SERDE_CLASS_CONFIG default key serializer} specified in the | ||
| * {@link StreamsConfig stream configuration} | ||
| * @param valueSerializer the {@link Serializer value serializer} used when consuming records; may be null if the sink | ||
| * should use the {@link StreamsConfig#DEFAULT_VALUE_SERDE_CLASS_CONFIG default value serializer} specified in the | ||
| * {@link StreamsConfig stream configuration} | ||
| * @param partitioner the function that should be used to determine the partition for each record processed by the sink | ||
| * @param parentNames the name of one or more source or processor nodes whose output records this sink should consume | ||
| * and dynamically write to topics | ||
| * @return itself | ||
| * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name, | ||
| * or if this processor's name is equal to the parent's name | ||
| * @see #addSink(String, String, String...) | ||
| * @see #addSink(String, String, StreamPartitioner, String...) | ||
| * @see #addSink(String, String, Serializer, Serializer, String...) | ||
| */ | ||
| public synchronized <K, V> Topology addSink(final String name, | ||
| final TopicNameExtractor<K, V> topicExtractor, | ||
| final Serializer<K> keySerializer, | ||
| final Serializer<V> valueSerializer, | ||
| final StreamPartitioner<? super K, ? super V> partitioner, | ||
| final String... parentNames) { | ||
| internalTopologyBuilder.addSink(name, topicExtractor, keySerializer, valueSerializer, partitioner, parentNames); | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * Add a new processor node that receives and processes records output by one or more parent source or processor | ||
| * node. | ||
|
|
@@ -526,7 +655,8 @@ public synchronized <K, V> Topology addSink(final String name, | |
| * @param parentNames the name of one or more source or processor nodes whose output records this processor should receive | ||
| * and process | ||
| * @return itself | ||
| * @throws TopologyException if parent processor is not added yet, or if this processor's name is equal to the parent's name, | ||
| * or if this processor's name is equal to the parent's name | ||
| */ | ||
| public synchronized Topology addProcessor(final String name, | ||
| final ProcessorSupplier supplier, | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
takes -> take