From 56134c137ef71bc91a6c744b3af0d0b52a63e05f Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Sat, 21 Apr 2018 14:41:58 -0700 Subject: [PATCH 01/13] kstream and ktable deprecated stateful operators --- .../apache/kafka/streams/kstream/KStream.java | 1150 ++--------------- .../apache/kafka/streams/kstream/KTable.java | 822 +----------- .../kstream/internals/KStreamImpl.java | 217 ---- .../streams/kstream/internals/KTableImpl.java | 106 +- ...StreamAggregationDedupIntegrationTest.java | 5 +- ...msFineGrainedAutoResetIntegrationTest.java | 9 +- .../RegexSourceIntegrationTest.java | 19 +- .../kstream/internals/KStreamImplTest.java | 14 +- .../kstream/internals/KTableFilterTest.java | 2 +- .../kstream/internals/KTableImplTest.java | 19 +- .../internals/KTableKTableInnerJoinTest.java | 29 +- .../kafka/streams/perf/YahooBenchmark.java | 3 +- 12 files changed, 140 insertions(+), 2255 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java index 1436e250aa188..853145384b70e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java @@ -359,392 +359,12 @@ public interface KStream { */ KStream flatMapValues(final ValueMapperWithKey> mapper); - /** - * Print the records of this stream to {@code System.out}. - * This function will use the generated name of the parent processor node to label the key/value pairs printed to - * the console. - *

- * The default serde will be used to deserialize the key or value in case the type is {@code byte[]} before calling - * {@code toString()} on the deserialized object. - *

- * Implementors will need to override {@code toString()} for keys and values that are not of type {@link String}, - * {@link Integer} etc. to get meaningful information. - * @deprecated use {@code print(Printed)} - */ - @Deprecated - void print(); - - /** - * Print the records of this stream to {@code System.out}. - * This function will use the given name to label the key/value pairs printed to the console. - *

- * The default serde will be used to deserialize the key or value in case the type is {@code byte[]} before calling - * {@code toString()} on the deserialized object. - *

- * Implementors will need to override {@code toString()} for keys and values that are not of type {@link String}, - * {@link Integer} etc. to get meaningful information. - * - * @param label the name used to label the key/value pairs printed to the console - * @deprecated use {@link #print(Printed) print(Printed.toSysOut()} - */ - @Deprecated - void print(final String label); - - /** - * Print the records of this stream to {@code System.out}. - * This function will use the generated name of the parent processor node to label the key/value pairs printed to - * the console. - *

- * The provided serde will be used to deserialize the key or value in case the type is {@code byte[]} before calling - * {@code toString()} on the deserialized object. - *

- * Implementors will need to override {@code toString()} for keys and values that are not of type {@link String}, - * {@link Integer} etc. to get meaningful information. - * - * @param keySerde key serde used to deserialize key if type is {@code byte[]}, - * @param valSerde value serde used to deserialize value if type is {@code byte[]}, - * @deprecated use {@link #print(Printed) print(Printed.toSysOut().withKeyValueMapper(...)} - */ - @Deprecated - void print(final Serde keySerde, - final Serde valSerde); - - /** - * Print the records of this stream to {@code System.out}. - *

- * The provided serde will be used to deserialize the key or value in case the type is {@code byte[]} before calling - * {@code toString()} on the deserialized object. - *

- * Implementors will need to override {@code toString()} for keys and values that are not of type {@link String}, - * {@link Integer} etc. to get meaningful information. - * - * @param keySerde key serde used to deserialize key if type is {@code byte[]}, - * @param valSerde value serde used to deserialize value if type is {@code byte[]}, - * @param label the name used to label the key/value pairs printed to the console - * @deprecated use {@link #print(Printed) print(Printed.toSysOut().withLabel(label).withKeyValueMapper(...)} - */ - @Deprecated - void print(final Serde keySerde, - final Serde valSerde, - final String label); - - /** - * Print the customized output with {@code System.out}. - *

- * The default serde will be use to deserialize key or value if type is {@code byte[]}. - * The user provided {@link KeyValueMapper} which customizes output is used to print with {@code System.out} - *

- * The example below shows the way to customize output data. - * 

{@code
-     * final KeyValueMapper mapper = new KeyValueMapper() {
-     *     public String apply(Integer key, String value) {
-     *         return String.format("(%d, %s)", key, value);
-     *     }
-     * };
-     * }
- *

- * The KeyValueMapper's mapped value type must be {@code String}. - * - * @param mapper a {@link KeyValueMapper} that computes output type {@code String}. - * @deprecated use {@link #print(Printed) print(Printed.toSysOut().withKeyValueMapper(mapper)} - */ - @Deprecated - void print(final KeyValueMapper mapper); - - /** - * Print the customized output with {@code System.out}. - *

- * The default serde will be used to deserialize key or value if type is {@code byte[]}. - * The user provided {@link KeyValueMapper} which customizes output is used to print with {@code System.out} - *

- * The example below shows the way to customize output data. - * 

{@code
-     * final KeyValueMapper mapper = new KeyValueMapper() {
-     *     public String apply(Integer key, String value) {
-     *         return String.format("(%d, %s)", key, value);
-     *     }
-     * };
-     * }
- *

- * The KeyValueMapper's mapped value type must be {@code String}. - * - * @param mapper a {@link KeyValueMapper} that computes output type {@code String}. - * @param label The given name which labels output will be printed. - * @deprecated use {@link #print(Printed) print(Printed.toSysOut().withLabel(label).withKeyValueMapper(mapper)} - */ - @Deprecated - void print(final KeyValueMapper mapper, final String label); - - /** - * Print the customized output with {@code System.out}. - *

- * The user provided {@link KeyValueMapper} which customizes output is used to print with {@code System.out} - * The provided serde will be use to deserialize key or value if type is {@code byte[]}. - *

- * The example below shows the way to customize output data. - * 

{@code
-     * final KeyValueMapper mapper = new KeyValueMapper() {
-     *     public String apply(Integer key, String value) {
-     *         return String.format("(%d, %s)", key, value);
-     *     }
-     * };
-     * }
- *

- * The provided KeyValueMapper's mapped value type must be {@code String}. - *

- * Implementors will need to override {@code toString()} for keys and values that are not of type {@link String}, - * {@link Integer} etc. to get meaningful information. - * - * @param mapper a {@link KeyValueMapper} that computes output type {@code String}. - * @param keySerde a {@link Serde} used to deserialize key if type is {@code byte[]}. - * @param valSerde a {@link Serde} used to deserialize value if type is {@code byte[]}. - * @deprecated use {@link #print(Printed) print(Printed.toSysOut().withKeyValueMapper(mapper)} - */ - @Deprecated - void print(final KeyValueMapper mapper, final Serde keySerde, final Serde valSerde); - - /** - * Print the customized output with {@code System.out}. - *

- * The user provided {@link KeyValueMapper} which customizes output is used to print with {@code System.out}. - * The provided serde will be use to deserialize key or value if type is {@code byte[]}. - *

- * The example below shows the way to customize output data. - * 

{@code
-     * final KeyValueMapper mapper = new KeyValueMapper() {
-     *     public String apply(Integer key, String value) {
-     *         return String.format("(%d, %s)", key, value);
-     *     }
-     * };
-     * }
- *

- * The provided KeyValueMapper's mapped value type must be {@code String}. - *

- * Implementors will need to override {@code toString()} for keys and values that are not of type {@link String}, - * {@link Integer} etc. to get meaningful information. - * - * @param mapper a {@link KeyValueMapper} that computes output type {@code String}. - * @param keySerde a {@link Serde} used to deserialize key if type is {@code byte[]}. - * @param valSerde a {@link Serde} used to deserialize value if type is {@code byte[]}. - * @param label The given name which labels output will be printed. - * @deprecated use {@link #print(Printed) print(Printed.toSysOut().withLabel(label).withKeyValueMapper(mapper)} - */ - @Deprecated - void print(final KeyValueMapper mapper, final Serde keySerde, final Serde valSerde, final String label); - /** * Print the records of this KStream using the options provided by {@link Printed} * @param printed options for printing */ void print(final Printed printed); - /** - * Merge this stream and the given stream into one larger stream. - *

- * There is no ordering guarantee between records from this {@code KStream} and records from - * the provided {@code KStream} in the merged stream. - * Relative order is preserved within each input stream though (ie, records within one input - * stream are processed in order). - * - * @param stream a stream which is to be merged into this stream - * @return a merged stream containing all records from this and the provided {@code KStream} - */ - KStream merge(final KStream stream); - - /** - * Write the records of this stream to a file at the given path. - * This function will use the generated name of the parent processor node to label the key/value pairs printed to - * the file. - *

- * The default serde will be used to deserialize the key or value in case the type is {@code byte[]} before calling - * {@code toString()} on the deserialized object. - *

- * Implementors will need to override {@code toString()} for keys and values that are not of type {@link String}, - * {@link Integer} etc. to get meaningful information. - * - * @param filePath name of the file to write to - * @deprecated use {@link #print(Printed) print(Printed.toFile(filePath)} - */ - @Deprecated - void writeAsText(final String filePath); - - /** - * Write the records of this stream to a file at the given path. - * This function will use the given name to label the key/value printed to the file. - *

- * The default serde will be used to deserialize the key or value in case the type is {@code byte[]} before calling - * {@code toString()} on the deserialized object. - *

- * Implementors will need to override {@code toString()} for keys and values that are not of type {@link String}, - * {@link Integer} etc. to get meaningful information. - * - * @param filePath name of the file to write to - * @param label the name used to label the key/value pairs written to the file - * @deprecated use {@link #print(Printed) print(Printed.toFile(filePath).withLabel(label)} - */ - @Deprecated - void writeAsText(final String filePath, - final String label); - - /** - * Write the records of this stream to a file at the given path. - * This function will use the generated name of the parent processor node to label the key/value pairs printed to - * the file. - *

- * The provided serde will be used to deserialize the key or value in case the type is {@code byte[]} before calling - * {@code toString()} on the deserialized object. - *

- * Implementors will need to override {@code toString()} for keys and values that are not of type {@link String}, - * {@link Integer} etc. to get meaningful information. - * - * @param filePath name of the file to write to - * @param keySerde key serde used to deserialize key if type is {@code byte[]}, - * @param valSerde value serde used to deserialize value if type is {@code byte[]}, - * @deprecated use {@link #print(Printed) print(Printed.toFile(filePath).withKeyValueMapper(...)} - */ - @Deprecated - void writeAsText(final String filePath, - final Serde keySerde, - final Serde valSerde); - - /** - * Write the records of this stream to a file at the given path. - * This function will use the given name to label the key/value printed to the file. - *

- * The provided serde will be used to deserialize the key or value in case the type is {@code byte[]} - * before calling {@code toString()} on the deserialized object. - *

- * Implementors will need to override {@code toString()} for keys and values that are not of type {@link String}, - * {@link Integer} etc. to get meaningful information. - * - * @param filePath name of the file to write to - * @param label the name used to label the key/value pairs written to the file - * @param keySerde key serde used to deserialize key if type is {@code byte[]}, - * @param valSerde value serde used deserialize value if type is {@code byte[]}, - * @deprecated use {@link #print(Printed) print(Printed.toFile(filePath).withLabel(label).withKeyValueMapper(...)} - */ - @Deprecated - void writeAsText(final String filePath, - final String label, - final Serde keySerde, - final Serde valSerde); - - /** - * Write the customised output to a given file path. - *

- * The user provided {@link KeyValueMapper} which customizes output is used to write to file. - * This function will use default name of stream to label records. - *

- * The default key and value serde will used to deserialize {@code byte[]} records before calling {@code toString()}. - *

- * The example below shows the way to customize output data. - * 

{@code
-     * final KeyValueMapper mapper = new KeyValueMapper() {
-     *     public String apply(Integer key, String value) {
-     *         return String.format("(%d, %s)", key, value);
-     *     }
-     * };
-     * }
- *

- * The KeyValueMapper's mapped value type must be {@code String}. - * - * @param filePath path of the file to write to. - * @param mapper a {@link KeyValueMapper} that computes output type {@code String}. - * @deprecated use {@link #print(Printed) print(Printed.toFile(filePath).withKeyValueMapper(mapper)} - */ - @Deprecated - void writeAsText(final String filePath, final KeyValueMapper mapper); - - /** - * Write the customised output to a given file path. - *

- * The user provided {@link KeyValueMapper} which customizes output is used to write to file. - * This function will use given name of stream to label records. - *

- * The default key and value serde will used to deserialize {@code byte[]} records before calling {@code toString()}. - *

- * The example below shows the way to customize output data. - * 

{@code
-     * final KeyValueMapper mapper = new KeyValueMapper() {
-     *     public String apply(Integer key, String value) {
-     *         return String.format("(%d, %s)", key, value);
-     *     }
-     * };
-     * }
- *

- * The KeyValueMapper's mapped value type must be {@code String}. - * - * @param filePath path of the file to write to. - * @param label the name used to label records written to file. - * @param mapper a {@link KeyValueMapper} that computes output type {@code String}. - * @deprecated use {@link #print(Printed) print(Printed.toFile(filePath).withLabel(label).withKeyValueMapper(mapper)} - */ - @Deprecated - void writeAsText(final String filePath, final String label, final KeyValueMapper mapper); - - /** - * Write the customised output to a given file path. - *

- * The user provided {@link KeyValueMapper} which customizes output is used to write to file. - * This function will use default name of stream to label records. - *

- * The given key and value serde will be used to deserialize {@code byte[]} records before calling {@code toString()}. - *

- * The example below shows the way to customize output data. - * 

{@code
-     * final KeyValueMapper mapper = new KeyValueMapper() {
-     *     public String apply(Integer key, String value) {
-     *         return String.format("(%d, %s)", key, value);
-     *     }
-     * };
-     * }
- *

- * The KeyValueMapper's mapped value type must be {@code String}. - *

- * Implementors will need to override {@code toString()} for keys and values that are not of type {@link String}, - * {@link Integer} etc. to get meaningful information. - * - * @param filePath path of the file to write to. - * @param keySerde key serde used to deserialize key if type is {@code byte[]}. - * @param valSerde value serde used to deserialize value if type is {@code byte[]}. - * @param mapper a {@link KeyValueMapper} that computes output type {@code String}. - * @deprecated use {@link #print(Printed) print(Printed.toFile(filePath).withKeyValueMapper(mapper)} - */ - @Deprecated - void writeAsText(final String filePath, final Serde keySerde, final Serde valSerde, final KeyValueMapper mapper); - - /** - * Write the customised output to a given file path. - *

- * The user provided {@link KeyValueMapper} which customizes output is used to write to file. - * This function will use given name of stream to label records. - *

- * The given key and value serde will be used to deserialize {@code byte[]} records before calling {@code toString()}. - *

- * The example below shows the way to customize output data. - * 

{@code
-     * final KeyValueMapper mapper = new KeyValueMapper() {
-     *     public String apply(Integer key, String value) {
-     *         return String.format("(%d, %s)", key, value);
-     *     }
-     * };
-     * }
- *

- * The KeyValueMapper's mapped value type must be {@code String}. - *

- * Implementors will need to override {@code toString()} for keys and values that are not of type {@link String}, - * {@link Integer} etc. to get meaningful information. - * - * @param filePath path of the file to write to. - * @param label the name used to label records written to file. - * @param keySerde key serde used to deserialize key if type is {@code byte[]}. - * @param valSerde value serde used to deserialize value if type is {@code byte[]}. - * @param mapper a {@link KeyValueMapper} that computes output type {@code String}. - * @deprecated use {@link #print(Printed) print(Printed.toFile(filePath).withLabel(label).withKeyValueMapper(mapper)} - */ - @Deprecated - void writeAsText(final String filePath, final String label, final Serde keySerde, final Serde valSerde, final KeyValueMapper mapper); - /** * Perform an action on each record of {@code KStream}. * This is a stateless record-by-record operation (cf. {@link #process(ProcessorSupplier, String...)}). @@ -786,89 +406,31 @@ void writeAsText(final String filePath, KStream[] branch(final Predicate... predicates); /** - * Materialize this stream to a topic and creates a new {@code KStream} from the topic using default serializers and - * deserializers and producer's {@link DefaultPartitioner}. - * The specified topic should be manually created before it is used (i.e., before the Kafka Streams application is - * started). + * Merge this stream and the given stream into one larger stream. *

- * This is equivalent to calling {@link #to(String) #to(someTopicName)} and - * {@link StreamsBuilder#stream(String) StreamsBuilder#stream(someTopicName)}. + * There is no ordering guarantee between records from this {@code KStream} and records from + * the provided {@code KStream} in the merged stream. + * Relative order is preserved within each input stream though (ie, records within one input + * stream are processed in order). * - * @param topic the topic name - * @return a {@code KStream} that contains the exact same (and potentially repartitioned) records as this {@code KStream} + * @param stream a stream which is to be merged into this stream + * @return a merged stream containing all records from this and the provided {@code KStream} */ - KStream through(final String topic); + KStream merge(final KStream stream); /** * Materialize this stream to a topic and creates a new {@code KStream} from the topic using default serializers and - * deserializers and a customizable {@link StreamPartitioner} to determine the distribution of records to partitions. + * deserializers and producer's {@link DefaultPartitioner}. * The specified topic should be manually created before it is used (i.e., before the Kafka Streams application is * started). *

- * This is equivalent to calling {@link #to(StreamPartitioner, String) #to(StreamPartitioner, someTopicName)} and + * This is equivalent to calling {@link #to(String) #to(someTopicName)} and * {@link StreamsBuilder#stream(String) StreamsBuilder#stream(someTopicName)}. * - * @param partitioner the function used to determine how records are distributed among partitions of the topic, - * if not specified producer's {@link DefaultPartitioner} will be used - * @param topic the topic name - * @return a {@code KStream} that contains the exact same (and potentially repartitioned) records as this {@code KStream} - * @deprecated use {@link #through(String, Produced) through(topic, Produced.withStreamPartitioner(partitioner))} - */ - @Deprecated - KStream through(final StreamPartitioner partitioner, - final String topic); - - /** - * Materialize this stream to a topic, and creates a new {@code KStream} from the topic. - * The specified topic should be manually created before it is used (i.e., before the Kafka Streams application is - * started). - *

- * If {@code keySerde} provides a {@link WindowedSerializer} for the key {@link WindowedStreamPartitioner} is - * used—otherwise producer's {@link DefaultPartitioner} is used. - *

- * This is equivalent to calling {@link #to(Serde, Serde, String) #to(keySerde, valSerde, someTopicName)} and - * {@link KStreamBuilder#stream(Serde, Serde, String...) KStreamBuilder#stream(keySerde, valSerde, someTopicName)}. - * - * @param keySerde key serde used to send key-value pairs, - * if not specified the default key serde defined in the configuration will be used - * @param valSerde value serde used to send key-value pairs, - * if not specified the default value serde defined in the configuration will be used - * @param topic the topic name - * @return a {@code KStream} that contains the exact same (and potentially repartitioned) records as this {@code KStream} - * @deprecated use {@link #through(String, Produced) through(topic, Produced.with(keySerde, valSerde))} - */ - @Deprecated - KStream through(final Serde keySerde, - final Serde valSerde, - final String topic); - - /** - * Materialize this stream to a topic and creates a new {@code KStream} from the topic using a customizable - * {@link StreamPartitioner} to determine the distribution of records to partitions. - * The specified topic should be manually created before it is used (i.e., before the Kafka Streams application is - * started). - *

- * This is equivalent to calling {@link #to(Serde, Serde, StreamPartitioner, String) #to(keySerde, valSerde, - * StreamPartitioner, someTopicName)} and {@link KStreamBuilder#stream(Serde, Serde, String...) - * KStreamBuilder#stream(keySerde, valSerde, someTopicName)}. - * - * @param keySerde key serde used to send key-value pairs, - * if not specified the default key serde defined in the configuration will be used - * @param valSerde value serde used to send key-value pairs, - * if not specified the default value serde defined in the configuration will be used - * @param partitioner the function used to determine how records are distributed among partitions of the topic, - * if not specified and {@code keySerde} provides a {@link WindowedSerializer} for the key - * {@link WindowedStreamPartitioner} will be used—otherwise {@link DefaultPartitioner} will - * be used - * @param topic the topic name + * @param topic the topic name * @return a {@code KStream} that contains the exact same (and potentially repartitioned) records as this {@code KStream} - * @deprecated use {@link #through(String, Produced) through(topic, Produced.with(keySerde, valSerde, partitioner))} */ - @Deprecated - KStream through(final Serde keySerde, - final Serde valSerde, - final StreamPartitioner partitioner, - final String topic); + KStream through(final String topic); /** * Materialize this stream to a topic and creates a new {@code KStream} from the topic using the @@ -897,63 +459,6 @@ KStream through(final String topic, */ void to(final String topic); - /** - * Materialize this stream to a topic using default serializers specified in the config and a customizable - * {@link StreamPartitioner} to determine the distribution of records to partitions. - * The specified topic should be manually created before it is used (i.e., before the Kafka Streams application is - * started). - * - * @param partitioner the function used to determine how records are distributed among partitions of the topic, - * if not specified producer's {@link DefaultPartitioner} will be used - * @param topic the topic name - * @deprecated use {@link #to(String, Produced) to(topic, Produced.withStreamPartitioner(partitioner))} - */ - @Deprecated - void to(final StreamPartitioner partitioner, - final String topic); - - /** - * Materialize this stream to a topic. If {@code keySerde} provides a {@link WindowedSerializer WindowedSerializer} - * for the key {@link WindowedStreamPartitioner} is used—otherwise producer's {@link DefaultPartitioner} is - * used. - * The specified topic should be manually created before it is used (i.e., before the Kafka Streams application is - * started). - * - * @param keySerde key serde used to send key-value pairs, - * if not specified the default serde defined in the configs will be used - * @param valSerde value serde used to send key-value pairs, - * if not specified the default serde defined in the configs will be used - * @param topic the topic name - * @deprecated use {@link #to(String, Produced) to(topic, Produced.with(keySerde, valSerde))} - */ - @Deprecated - void to(final Serde keySerde, - final Serde valSerde, - final String topic); - - /** - * Materialize this stream to a topic using a customizable {@link StreamPartitioner} to determine the distribution - * of records to partitions. - * The specified topic should be manually created before it is used (i.e., before the Kafka Streams application is - * started). - * - * @param keySerde key serde used to send key-value pairs, - * if not specified the default serde defined in the configs will be used - * @param valSerde value serde used to send key-value pairs, - * if not specified the default serde defined in the configs will be used - * @param partitioner the function used to determine how records are distributed among partitions of the topic, - * if not specified and {@code keySerde} provides a {@link WindowedSerializer} for the key - * {@link WindowedStreamPartitioner} will be used—otherwise {@link DefaultPartitioner} will - * be used - * @param topic the topic name - * @deprecated use {@link #to(String, Produced) to(topic, Produced.with(keySerde, valSerde, partitioner)} - */ - @Deprecated - void to(final Serde keySerde, - final Serde valSerde, - final StreamPartitioner partitioner, - final String topic); - /** * Materialize this stream to a topic using the provided {@link Produced} instance. * The specified topic should be manually created before it is used (i.e., before the Kafka Streams application is @@ -1287,36 +792,6 @@ void process(final ProcessorSupplier processorSupplier, */ KGroupedStream groupByKey(final Serialized serialized); - /** - * Group the records by their current key into a {@link KGroupedStream} while preserving the original values. - * Grouping a stream on the record key is required before an aggregation operator can be applied to the data - * (cf. {@link KGroupedStream}). - * If a record key is {@code null} the record will not be included in the resulting {@link KGroupedStream}. - *

- * If a key changing operator was used before this operation (e.g., {@link #selectKey(KeyValueMapper)}, - * {@link #map(KeyValueMapper)}, {@link #flatMap(KeyValueMapper)}, or - * {@link #transform(TransformerSupplier, String...)}), and no data redistribution happened afterwards (e.g., via - * {@link #through(String)}) an internal repartitioning topic will be created in Kafka. - * This topic will be named "${applicationId}-XXX-repartition", where "applicationId" is user-specified in - * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is - * an internally generated name, and "-repartition" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - *

- * For this case, all data of this stream will be redistributed through the repartitioning topic by writing all - * records to it, and rereading all records from it, such that the resulting {@link KGroupedStream} is partitioned - * correctly on its key. - * - * @param keySerde key serdes for materializing this stream, - * if not specified the default serdes defined in the configs will be used - * @param valSerde value serdes for materializing this stream, - * if not specified the default serdes defined in the configs will be used - * @return a {@link KGroupedStream} that contains the grouped records of the original {@code KStream} - * @deprecated use {@link #groupByKey(Serialized) groupByKey(Serialized.with(keySerde, valSerde))} - */ - @Deprecated - KGroupedStream groupByKey(final Serde keySerde, - final Serde valSerde); - /** * Group the records of this {@code KStream} on a new key that is selected using the provided {@link KeyValueMapper} * and default serializers and deserializers. @@ -1369,40 +844,6 @@ KGroupedStream groupByKey(final Serde keySerde, KGroupedStream groupBy(final KeyValueMapper selector, final Serialized serialized); - /** - * Group the records of this {@code KStream} on a new key that is selected using the provided {@link KeyValueMapper}. - * Grouping a stream on the record key is required before an aggregation operator can be applied to the data - * (cf. {@link KGroupedStream}). - * The {@link KeyValueMapper} selects a new key (with potentially different type) while preserving the original values. - * If the new record key is {@code null} the record will not be included in the resulting {@link KGroupedStream}. - *

- * Because a new key is selected, an internal repartitioning topic will be created in Kafka. - * This topic will be named "${applicationId}-XXX-repartition", where "applicationId" is user-specified in - * {@link StreamsConfig StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and - * "-repartition" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - *

- * All data of this stream will be redistributed through the repartitioning topic by writing all records to it, - * and rereading all records from it, such that the resulting {@link KGroupedStream} is partitioned on the new key. - *

- * This is equivalent to calling {@link #selectKey(KeyValueMapper)} followed by {@link #groupByKey(Serde, Serde)}. - * - * @param selector a {@link KeyValueMapper} that computes a new key for grouping - * @param keySerde key serdes for materializing this stream, - * if not specified the default serdes defined in the configs will be used - * @param valSerde value serdes for materializing this stream, - * if not specified the default serdes defined in the configs will be used - * @param the key type of the result {@link KGroupedStream} - * @return a {@link KGroupedStream} that contains the grouped records of the original {@code KStream} - * @see #groupByKey() - * @deprecated use {@link #groupBy(KeyValueMapper, Serialized) groupBy(selector, Serialized.with(keySerde, valSerde))} - */ - @Deprecated - KGroupedStream groupBy(final KeyValueMapper selector, - final Serde keySerde, - final Serde valSerde); - /** * Join records of this stream with another {@code KStream}'s records using windowed inner equi join with default * serializers and deserializers. @@ -1426,250 +867,7 @@ KGroupedStream groupBy(final KeyValueMapper * <K1:A> * - * - * - * - * <K2:B> - * <K2:b> - * <K2:ValueJoiner(B,b)> - * - * - * - * <K3:c> - * - * - * - * Both input streams (or to be more precise, their underlying source topics) need to have the same number of - * partitions. - * If this is not the case, you would need to call {@link #through(String)} (for one input stream) before doing the - * join, using a pre-created topic with the "correct" number of partitions. - * Furthermore, both input streams need to be co-partitioned on the join key (i.e., use the same partitioner). - * If this requirement is not met, Kafka Streams will automatically repartition the data, i.e., it will create an - * internal repartitioning topic in Kafka and write and re-read the data via this topic before the actual join. - * The repartitioning topic will be named "${applicationId}-XXX-repartition", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and - * "-repartition" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - *

- * Repartitioning can happen for one or both of the joining {@code KStream}s. - * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all - * records to it, and rereading all records from it, such that the join input {@code KStream} is partitioned - * correctly on its key. - *

- * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names. - * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified - * in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is an - * internally generated name, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param otherStream the {@code KStream} to be joined with this stream - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param windows the specification of the {@link JoinWindows} - * @param the value type of the other stream - * @param the value type of the result stream - * @return a {@code KStream} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key and within the joining window intervals - * @see #leftJoin(KStream, ValueJoiner, JoinWindows) - * @see #outerJoin(KStream, ValueJoiner, JoinWindows) - */ - KStream join(final KStream otherStream, - final ValueJoiner joiner, - final JoinWindows windows); - - /** - * Join records of this stream with another {@code KStream}'s records using windowed inner equi join with default - * serializers and deserializers. - * The join is computed on the records' key with join attribute {@code thisKStream.key == otherKStream.key}. - * Furthermore, two records are only joined if their timestamps are close to each other as defined by the given - * {@link JoinWindows}, i.e., the window defines an additional join predicate on the record timestamps. - *

- * For each pair of records meeting both join predicates the provided {@link ValueJoiner} will be called to compute - * a value (with arbitrary type) for the result record. - * The key of the result record is the same as for both joining input records. - * If an input record key or value is {@code null} the record will not be included in the join operation and thus no - * output record will be added to the resulting {@code KStream}. - *

- * Example (assuming all input records belong to the correct windows): - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
thisotherresult
<K1:A>
<K2:B><K2:b><K2:ValueJoiner(B,b)>
<K3:c>
- * Both input streams (or to be more precise, their underlying source topics) need to have the same number of - * partitions. - * If this is not the case, you would need to call {@link #through(String)} (for one input stream) before doing the - * join, using a pre-created topic with the "correct" number of partitions. - * Furthermore, both input streams need to be co-partitioned on the join key (i.e., use the same partitioner). - * If this requirement is not met, Kafka Streams will automatically repartition the data, i.e., it will create an - * internal repartitioning topic in Kafka and write and re-read the data via this topic before the actual join. - * The repartitioning topic will be named "${applicationId}-XXX-repartition", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and - * "-repartition" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - *

- * Repartitioning can happen for one or both of the joining {@code KStream}s. - * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all - * records to it, and rereading all records from it, such that the join input {@code KStream} is partitioned - * correctly on its key. - *

- * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names. - * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified - * in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is an - * internally generated name, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param otherStream the {@code KStream} to be joined with this stream - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param windows the specification of the {@link JoinWindows} - * @param joined a {@link Joined} instance that defines the serdes to - * be used to serialize/deserialize inputs and outputs of the joined streams - * @param the value type of the other stream - * @param the value type of the result stream - * @return a {@code KStream} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key and within the joining window intervals - * @see #leftJoin(KStream, ValueJoiner, JoinWindows, Joined) - * @see #outerJoin(KStream, ValueJoiner, JoinWindows, Joined) - */ - KStream join(final KStream otherStream, - final ValueJoiner joiner, - final JoinWindows windows, - final Joined joined); - /** - * Join records of this stream with another {@code KStream}'s records using windowed inner equi join. - * The join is computed on the records' key with join attribute {@code thisKStream.key == otherKStream.key}. - * Furthermore, two records are only joined if their timestamps are close to each other as defined by the given - * {@link JoinWindows}, i.e., the window defines an additional join predicate on the record timestamps. - *

- * For each pair of records meeting both join predicates the provided {@link ValueJoiner} will be called to compute - * a value (with arbitrary type) for the result record. - * The key of the result record is the same as for both joining input records. - * If an input record key or value is {@code null} the record will not be included in the join operation and thus no - * output record will be added to the resulting {@code KStream}. - *

- * Example (assuming all input records belong to the correct windows): - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
thisotherresult
<K1:A>
<K2:B><K2:b><K2:ValueJoiner(B,b)>
<K3:c>
- * Both input streams (or to be more precise, their underlying source topics) need to have the same number of - * partitions. - * If this is not the case, you would need to call {@link #through(String)} (for one input stream) before doing the - * join, using a pre-created topic with the "correct" number of partitions. - * Furthermore, both input streams need to be co-partitioned on the join key (i.e., use the same partitioner). - * If this requirement is not met, Kafka Streams will automatically repartition the data, i.e., it will create an - * internal repartitioning topic in Kafka and write and re-read the data via this topic before the actual join. - * The repartitioning topic will be named "${applicationId}-XXX-repartition", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and - * "-repartition" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - *

- * Repartitioning can happen for one or both of the joining {@code KStream}s. - * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all - * records to it, and rereading all records from it, such that the join input {@code KStream} is partitioned - * correctly on its key. - *

- * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names. - * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified - * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, - * "storeName" is an internally generated name, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param otherStream the {@code KStream} to be joined with this stream - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param windows the specification of the {@link JoinWindows} - * @param keySerde key serdes for materializing both streams, - * if not specified the default serdes defined in the configs will be used - * @param thisValueSerde value serdes for materializing this stream, - * if not specified the default serdes defined in the configs will be used - * @param otherValueSerde value serdes for materializing the other stream, - * if not specified the default serdes defined in the configs will be used - * @param the value type of the other stream - * @param the value type of the result stream - * @return a {@code KStream} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key and within the joining window intervals - * @see #leftJoin(KStream, ValueJoiner, JoinWindows, Joined) - * @see #outerJoin(KStream, ValueJoiner, JoinWindows, Joined) - * @deprecated use {@link #join(KStream, ValueJoiner, JoinWindows, Joined) join(otherStream, joiner, windows, Joined.with(keySerde, thisValueSerde, otherValueSerde))} - */ - @Deprecated - KStream join(final KStream otherStream, - final ValueJoiner joiner, - final JoinWindows windows, - final Serde keySerde, - final Serde thisValueSerde, - final Serde otherValueSerde); - - /** - * Join records of this stream with another {@code KStream}'s records using windowed left equi join with default - * serializers and deserializers. - * In contrast to {@link #join(KStream, ValueJoiner, JoinWindows) inner-join}, all records from this stream will - * produce at least one output record (cf. below). - * The join is computed on the records' key with join attribute {@code thisKStream.key == otherKStream.key}. - * Furthermore, two records are only joined if their timestamps are close to each other as defined by the given - * {@link JoinWindows}, i.e., the window defines an additional join predicate on the record timestamps. - *

- * For each pair of records meeting both join predicates the provided {@link ValueJoiner} will be called to compute - * a value (with arbitrary type) for the result record. - * The key of the result record is the same as for both joining input records. - * Furthermore, for each input record of this {@code KStream} that does not satisfy the join predicate the provided - * {@link ValueJoiner} will be called with a {@code null} value for the other stream. - * If an input record key or value is {@code null} the record will not be included in the join operation and thus no - * output record will be added to the resulting {@code KStream}. - *

- * Example (assuming all input records belong to the correct windows): - * - * - * - * - * - * - * - * - * - * + * * * * @@ -1690,7 +888,7 @@ KStream join(final KStream otherStream, * If this requirement is not met, Kafka Streams will automatically repartition the data, i.e., it will create an * internal repartitioning topic in Kafka and write and re-read the data via this topic before the actual join. * The repartitioning topic will be named "${applicationId}-XXX-repartition", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter + * user-specified in {@link StreamsConfig} via parameter * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and * "-repartition" is a fixed suffix. * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. @@ -1703,8 +901,9 @@ KStream join(final KStream otherStream, * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names. * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka. * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified - * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, - * "storeName" is an internally generated name, and "-changelog" is a fixed suffix. + * in {@link StreamsConfig} via parameter + * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is an + * internally generated name, and "-changelog" is a fixed suffix. * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. * * @param otherStream the {@code KStream} to be joined with this stream @@ -1713,20 +912,17 @@ KStream join(final KStream otherStream, * @param the value type of the other stream * @param the value type of the result stream * @return a {@code KStream} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of - * this {@code KStream} and within the joining window intervals - * @see #join(KStream, ValueJoiner, JoinWindows) + * {@link ValueJoiner}, one for each matched record-pair with the same key and within the joining window intervals + * @see #leftJoin(KStream, ValueJoiner, JoinWindows) * @see #outerJoin(KStream, ValueJoiner, JoinWindows) */ - KStream leftJoin(final KStream otherStream, - final ValueJoiner joiner, - final JoinWindows windows); + KStream join(final KStream otherStream, + final ValueJoiner joiner, + final JoinWindows windows); /** - * Join records of this stream with another {@code KStream}'s records using windowed left equi join with default + * Join records of this stream with another {@code KStream}'s records using windowed inner equi join with default * serializers and deserializers. - * In contrast to {@link #join(KStream, ValueJoiner, JoinWindows) inner-join}, all records from this stream will - * produce at least one output record (cf. below). * The join is computed on the records' key with join attribute {@code thisKStream.key == otherKStream.key}. * Furthermore, two records are only joined if their timestamps are close to each other as defined by the given * {@link JoinWindows}, i.e., the window defines an additional join predicate on the record timestamps. @@ -1734,8 +930,6 @@ KStream leftJoin(final KStream otherStream, * For each pair of records meeting both join predicates the provided {@link ValueJoiner} will be called to compute * a value (with arbitrary type) for the result record. * The key of the result record is the same as for both joining input records. - * Furthermore, for each input record of this {@code KStream} that does not satisfy the join predicate the provided - * {@link ValueJoiner} will be called with a {@code null} value for the other stream. * If an input record key or value is {@code null} the record will not be included in the join operation and thus no * output record will be added to the resulting {@code KStream}. *

@@ -1749,7 +943,7 @@ KStream leftJoin(final KStream otherStream, *

* * - * + * * * * @@ -1770,7 +964,7 @@ KStream leftJoin(final KStream otherStream, * If this requirement is not met, Kafka Streams will automatically repartition the data, i.e., it will create an * internal repartitioning topic in Kafka and write and re-read the data via this topic before the actual join. * The repartitioning topic will be named "${applicationId}-XXX-repartition", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter + * user-specified in {@link StreamsConfig} via parameter * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and * "-repartition" is a fixed suffix. * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. @@ -1783,8 +977,9 @@ KStream leftJoin(final KStream otherStream, * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names. * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka. * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified - * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, - * "storeName" is an internally generated name, and "-changelog" is a fixed suffix. + * in {@link StreamsConfig} via parameter + * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is an + * internally generated name, and "-changelog" is a fixed suffix. * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. * * @param otherStream the {@code KStream} to be joined with this stream @@ -1795,20 +990,20 @@ KStream leftJoin(final KStream otherStream, * @param the value type of the other stream * @param the value type of the result stream * @return a {@code KStream} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of - * this {@code KStream} and within the joining window intervals - * @see #join(KStream, ValueJoiner, JoinWindows, Joined) + * {@link ValueJoiner}, one for each matched record-pair with the same key and within the joining window intervals + * @see #leftJoin(KStream, ValueJoiner, JoinWindows, Joined) * @see #outerJoin(KStream, ValueJoiner, JoinWindows, Joined) */ - KStream leftJoin(final KStream otherStream, - final ValueJoiner joiner, - final JoinWindows windows, - final Joined joined); + KStream join(final KStream otherStream, + final ValueJoiner joiner, + final JoinWindows windows, + final Joined joined); /** - * Join records of this stream with another {@code KStream}'s records using windowed left equi join. - * In contrast to {@link #join(KStream, ValueJoiner, JoinWindows, Joined) inner-join}, all records from - * this stream will produce at least one output record (cf. below). + * Join records of this stream with another {@code KStream}'s records using windowed left equi join with default + * serializers and deserializers. + * In contrast to {@link #join(KStream, ValueJoiner, JoinWindows) inner-join}, all records from this stream will + * produce at least one output record (cf. below). * The join is computed on the records' key with join attribute {@code thisKStream.key == otherKStream.key}. * Furthermore, two records are only joined if their timestamps are close to each other as defined by the given * {@link JoinWindows}, i.e., the window defines an additional join predicate on the record timestamps. @@ -1865,43 +1060,30 @@ KStream leftJoin(final KStream otherStream, * Both of the joining {@code KStream}s will be materialized in local state stores with auto-generated store names. * For failure and recovery each store will be backed by an internal changelog topic that will be created in Kafka. * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified - * in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is an - * internally generated name, and "-changelog" is a fixed suffix. + * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, + * "storeName" is an internally generated name, and "-changelog" is a fixed suffix. * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. * - * @param otherStream the {@code KStream} to be joined with this stream - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param windows the specification of the {@link JoinWindows} - * @param keySerde key serdes for materializing the other stream, - * if not specified the default serdes defined in the configs will be used - * @param thisValSerde value serdes for materializing this stream, - * if not specified the default serdes defined in the configs will be used - * @param otherValueSerde value serdes for materializing the other stream, - * if not specified the default serdes defined in the configs will be used - * @param the value type of the other stream - * @param the value type of the result stream + * @param otherStream the {@code KStream} to be joined with this stream + * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records + * @param windows the specification of the {@link JoinWindows} + * @param the value type of the other stream + * @param the value type of the result stream * @return a {@code KStream} that contains join-records for each key and values computed by the given * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of * this {@code KStream} and within the joining window intervals - * @see #join(KStream, ValueJoiner, JoinWindows, Joined) - * @see #outerJoin(KStream, ValueJoiner, JoinWindows, Joined) - * @deprecated use {@link #leftJoin(KStream, ValueJoiner, JoinWindows, Joined) leftJoin(otherStream, joiner, windows, Joined.with(keySerde, thisValSerde, otherValueSerde))} + * @see #join(KStream, ValueJoiner, JoinWindows) + * @see #outerJoin(KStream, ValueJoiner, JoinWindows) */ - @Deprecated KStream leftJoin(final KStream otherStream, final ValueJoiner joiner, - final JoinWindows windows, - final Serde keySerde, - final Serde thisValSerde, - final Serde otherValueSerde); + final JoinWindows windows); /** - * Join records of this stream with another {@code KStream}'s records using windowed outer equi join with default + * Join records of this stream with another {@code KStream}'s records using windowed left equi join with default * serializers and deserializers. - * In contrast to {@link #join(KStream, ValueJoiner, JoinWindows) inner-join} or - * {@link #leftJoin(KStream, ValueJoiner, JoinWindows) left-join}, all records from both streams will produce at - * least one output record (cf. below). + * In contrast to {@link #join(KStream, ValueJoiner, JoinWindows) inner-join}, all records from this stream will + * produce at least one output record (cf. below). * The join is computed on the records' key with join attribute {@code thisKStream.key == otherKStream.key}. * Furthermore, two records are only joined if their timestamps are close to each other as defined by the given * {@link JoinWindows}, i.e., the window defines an additional join predicate on the record timestamps. @@ -1909,8 +1091,8 @@ KStream leftJoin(final KStream otherStream, * For each pair of records meeting both join predicates the provided {@link ValueJoiner} will be called to compute * a value (with arbitrary type) for the result record. * The key of the result record is the same as for both joining input records. - * Furthermore, for each input record of both {@code KStream}s that does not satisfy the join predicate the provided - * {@link ValueJoiner} will be called with a {@code null} value for the this/other stream, respectively. + * Furthermore, for each input record of this {@code KStream} that does not satisfy the join predicate the provided + * {@link ValueJoiner} will be called with a {@code null} value for the other stream. * If an input record key or value is {@code null} the record will not be included in the join operation and thus no * output record will be added to the resulting {@code KStream}. *

@@ -1929,12 +1111,12 @@ KStream leftJoin(final KStream otherStream, *

* * - * + * * * * * - * + * * *
thisotherresult
<K1:A><K1:ValueJoiner(A,null)>
<K2:B>
<K1:A><K1:ValueJoiner(A,null)>
<K2:B>
<K2:B><K2:b><K2:ValueJoiner(null,b)>
<K2:ValueJoiner(B,b)>		<K2:ValueJoiner(B,b)>
<K3:c><K3:ValueJoiner(null,c)>
* Both input streams (or to be more precise, their underlying source topics) need to have the same number of @@ -1965,17 +1147,20 @@ KStream leftJoin(final KStream otherStream, * @param otherStream the {@code KStream} to be joined with this stream * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records * @param windows the specification of the {@link JoinWindows} + * @param joined a {@link Joined} instance that defines the serdes to + * be used to serialize/deserialize inputs and outputs of the joined streams * @param the value type of the other stream * @param the value type of the result stream * @return a {@code KStream} that contains join-records for each key and values computed by the given * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of - * both {@code KStream} and within the joining window intervals - * @see #join(KStream, ValueJoiner, JoinWindows) - * @see #leftJoin(KStream, ValueJoiner, JoinWindows) + * this {@code KStream} and within the joining window intervals + * @see #join(KStream, ValueJoiner, JoinWindows, Joined) + * @see #outerJoin(KStream, ValueJoiner, JoinWindows, Joined) */ - KStream outerJoin(final KStream otherStream, - final ValueJoiner joiner, - final JoinWindows windows); + KStream leftJoin(final KStream otherStream, + final ValueJoiner joiner, + final JoinWindows windows, + final Joined joined); /** * Join records of this stream with another {@code KStream}'s records using windowed outer equi join with default @@ -2051,19 +1236,19 @@ KStream outerJoin(final KStream otherStream, * @return a {@code KStream} that contains join-records for each key and values computed by the given * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of * both {@code KStream} and within the joining window intervals - * @see #join(KStream, ValueJoiner, JoinWindows, Joined) - * @see #leftJoin(KStream, ValueJoiner, JoinWindows, Joined) + * @see #join(KStream, ValueJoiner, JoinWindows) + * @see #leftJoin(KStream, ValueJoiner, JoinWindows) */ KStream outerJoin(final KStream otherStream, final ValueJoiner joiner, - final JoinWindows windows, - final Joined joined); + final JoinWindows windows); /** - * Join records of this stream with another {@code KStream}'s records using windowed outer equi join. - * In contrast to {@link #join(KStream, ValueJoiner, JoinWindows, Joined) inner-join} or - * {@link #leftJoin(KStream, ValueJoiner, JoinWindows, Joined) left-join}, all records from both - * streams will produce at least one output record (cf. below). + * Join records of this stream with another {@code KStream}'s records using windowed outer equi join with default + * serializers and deserializers. + * In contrast to {@link #join(KStream, ValueJoiner, JoinWindows) inner-join} or + * {@link #leftJoin(KStream, ValueJoiner, JoinWindows) left-join}, all records from both streams will produce at + * least one output record (cf. below). * The join is computed on the records' key with join attribute {@code thisKStream.key == otherKStream.key}. * Furthermore, two records are only joined if their timestamps are close to each other as defined by the given * {@link JoinWindows}, i.e., the window defines an additional join predicate on the record timestamps. @@ -2072,7 +1257,7 @@ KStream outerJoin(final KStream otherStream, * a value (with arbitrary type) for the result record. * The key of the result record is the same as for both joining input records. * Furthermore, for each input record of both {@code KStream}s that does not satisfy the join predicate the provided - * {@link ValueJoiner} will be called with a {@code null} value for this/other stream, respectively. + * {@link ValueJoiner} will be called with a {@code null} value for the this/other stream, respectively. * If an input record key or value is {@code null} the record will not be included in the join operation and thus no * output record will be added to the resulting {@code KStream}. *

@@ -2124,31 +1309,21 @@ KStream outerJoin(final KStream otherStream, * "storeName" is an internally generated name, and "-changelog" is a fixed suffix. * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. * - * @param otherStream the {@code KStream} to be joined with this stream - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param windows the specification of the {@link JoinWindows} - * @param keySerde key serdes for materializing both streams, - * if not specified the default serdes defined in the configs will be used - * @param thisValueSerde value serdes for materializing this stream, - * if not specified the default serdes defined in the configs will be used - * @param otherValueSerde value serdes for materializing the other stream, - * if not specified the default serdes defined in the configs will be used - * @param the value type of the other stream - * @param the value type of the result stream + * @param otherStream the {@code KStream} to be joined with this stream + * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records + * @param windows the specification of the {@link JoinWindows} + * @param the value type of the other stream + * @param the value type of the result stream * @return a {@code KStream} that contains join-records for each key and values computed by the given * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of - * both {@code KStream}s and within the joining window intervals + * both {@code KStream} and within the joining window intervals * @see #join(KStream, ValueJoiner, JoinWindows, Joined) * @see #leftJoin(KStream, ValueJoiner, JoinWindows, Joined) - * @deprecated use {@link #outerJoin(KStream, ValueJoiner, JoinWindows, Joined) outerJoin(otherStream, joiner, windows, Joined.with(keySerde, thisValueSerde, otherValueSerde))} */ - @Deprecated KStream outerJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, - final Serde keySerde, - final Serde thisValueSerde, - final Serde otherValueSerde); + final Joined joined); /** * Join records of this stream with {@link KTable}'s records using non-windowed inner equi join with default @@ -2301,87 +1476,6 @@ KStream join(final KTable table, final ValueJoiner joiner, final Joined joined); - /** - * Join records of this stream with {@link KTable}'s records using non-windowed inner equi join. - * The join is a primary key table lookup join with join attribute {@code stream.key == table.key}. - * "Table lookup join" means, that results are only computed if {@code KStream} records are processed. - * This is done by performing a lookup for matching records in the current (i.e., processing time) internal - * {@link KTable} state. - * In contrast, processing {@link KTable} input records will only update the internal {@link KTable} state and - * will not produce any result records. - *

- * For each {@code KStream} record that finds a corresponding record in {@link KTable} the provided - * {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. - * The key of the result record is the same as for both joining input records. - * If an {@code KStream} input record key or value is {@code null} the record will not be included in the join - * operation and thus no output record will be added to the resulting {@code KStream}. - *

- * Example: - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
KStreamKTablestateresult
<K1:A>
<K1:b><K1:b>
<K1:C><K1:b><K1:ValueJoiner(C,b)>
- * Both input streams (or to be more precise, their underlying source topics) need to have the same number of - * partitions. - * If this is not the case, you would need to call {@link #through(String)} for this {@code KStream} before doing - * the join, using a pre-created topic with the same number of partitions as the given {@link KTable}. - * Furthermore, both input streams need to be co-partitioned on the join key (i.e., use the same partitioner); - * cf. {@link #join(GlobalKTable, KeyValueMapper, ValueJoiner)}. - * If this requirement is not met, Kafka Streams will automatically repartition the data, i.e., it will create an - * internal repartitioning topic in Kafka and write and re-read the data via this topic before the actual join. - * The repartitioning topic will be named "${applicationId}-XXX-repartition", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and - * "-repartition" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - *

- * Repartitioning can happen only for this {@code KStream} but not for the provided {@link KTable}. - * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all - * records to it, and rereading all records from it, such that the join input {@code KStream} is partitioned - * correctly on its key. - * - * @param table the {@link KTable} to be joined with this stream - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param keySerde key serdes for materializing this ({@link KStream} input) stream - * If not specified the default serdes defined in the configs will be used - * @param valSerde value serdes for materializing this ({@link KStream} input) stream, - * if not specified the default serdes defined in the configs will be used - * @param the value type of the table - * @param the value type of the result stream - * @return a {@code KStream} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key - * @see #leftJoin(KTable, ValueJoiner, Joined) - * @see #join(GlobalKTable, KeyValueMapper, ValueJoiner) - * @deprecated use {@link #join(KTable, ValueJoiner, Joined) join(table, joiner, Joined.with(keySerde, valSerde, null))} - */ - @Deprecated - KStream join(final KTable table, - final ValueJoiner joiner, - final Serde keySerde, - final Serde valSerde); - /** * Join records of this stream with {@link KTable}'s records using non-windowed left equi join with default * serializers and deserializers. @@ -2537,90 +1631,6 @@ KStream leftJoin(final KTable table, final ValueJoiner joiner, final Joined joined); - /** - * Join records of this stream with {@link KTable}'s records using non-windowed left equi join. - * In contrast to {@link #join(KTable, ValueJoiner) inner-join}, all records from this stream will produce an - * output record (cf. below). - * The join is a primary key table lookup join with join attribute {@code stream.key == table.key}. - * "Table lookup join" means, that results are only computed if {@code KStream} records are processed. - * This is done by performing a lookup for matching records in the current (i.e., processing time) internal - * {@link KTable} state. - * In contrast, processing {@link KTable} input records will only update the internal {@link KTable} state and - * will not produce any result records. - *

- * For each {@code KStream} record whether or not it finds a corresponding record in {@link KTable} the provided - * {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. - * If no {@link KTable} record was found during lookup, a {@code null} value will be provided to {@link ValueJoiner}. - * The key of the result record is the same as for both joining input records. - * If an {@code KStream} input record key or value is {@code null} the record will not be included in the join - * operation and thus no output record will be added to the resulting {@code KStream}. - *

- * Example: - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
KStreamKTablestateresult
<K1:A><K1:ValueJoiner(A,null)>
<K1:b><K1:b>
<K1:C><K1:b><K1:ValueJoiner(C,b)>
- * Both input streams (or to be more precise, their underlying source topics) need to have the same number of - * partitions. - * If this is not the case, you would need to call {@link #through(String)} for this {@code KStream} before doing - * the join, using a pre-created topic with the same number of partitions as the given {@link KTable}. - * Furthermore, both input streams need to be co-partitioned on the join key (i.e., use the same partitioner); - * cf. {@link #join(GlobalKTable, KeyValueMapper, ValueJoiner)}. - * If this requirement is not met, Kafka Streams will automatically repartition the data, i.e., it will create an - * internal repartitioning topic in Kafka and write and re-read the data via this topic before the actual join. - * The repartitioning topic will be named "${applicationId}-XXX-repartition", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and - * "-repartition" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - *

- * Repartitioning can happen only for this {@code KStream} but not for the provided {@link KTable}. - * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all - * records to it, and rereading all records from it, such that the join input {@code KStream} is partitioned - * correctly on its key. - * - * @param table the {@link KTable} to be joined with this stream - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param keySerde key serdes for materializing this ({@link KStream} input) stream - * If not specified the default serdes defined in the configs will be used - * @param valSerde value serdes for materializing this ({@link KStream} input) stream, - * if not specified the default serdes defined in the configs will be used - * @param the value type of the table - * @param the value type of the result stream - * @return a {@code KStream} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one output for each input {@code KStream} record - * @see #join(KTable, ValueJoiner, Serde, Serde) - * @see #leftJoin(GlobalKTable, KeyValueMapper, ValueJoiner) - * @deprecated use {@link #leftJoin(KTable, ValueJoiner, Joined) leftJoin(table, joiner, Joined.with(keySerde, valSerde, null))} - */ - @Deprecated - KStream leftJoin(final KTable table, - final ValueJoiner joiner, - final Serde keySerde, - final Serde valSerde); - /** * Join records of this stream with {@link GlobalKTable}'s records using non-windowed inner equi join. * The join is a primary key table lookup join with join attribute diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java index 55555a515926d..cbd450b25a37a 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java @@ -129,84 +129,6 @@ public interface KTable { KTable filter(final Predicate predicate, final Materialized> materialized); - /** - * Create a new {@code KTable} that consists of all records of this {@code KTable} which satisfy the given - * predicate. - * All records that do not satisfy the predicate are dropped. - * For each {@code KTable} update the filter is evaluated on the updated record to produce an updated record for the - * result {@code KTable}. - * This is a stateless record-by-record operation. - *

- * Note that {@code filter} for a changelog stream works differently than {@link KStream#filter(Predicate) - * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records) - * have delete semantics. - * Thus, for tombstones the provided filter predicate is not evaluated but the tombstone record is forwarded - * directly if required (i.e., if there is anything to be deleted). - * Furthermore, for each record that gets dropped (i.e., does not satisfy the given predicate) a tombstone record - * is forwarded. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // filtering words
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * K key = "some-word";
-     * V valueForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * - * @param predicate a filter {@link Predicate} that is applied to each record - * @param queryableStoreName a user-provided name of the underlying {@link KTable} that can be - * used to subsequently query the operation results; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then the results cannot be queried - * (i.e., that would be equivalent to calling {@link KTable#filter(Predicate)}. - * @return a {@code KTable} that contains only those records that satisfy the given predicate - * @see #filterNot(Predicate, Materialized) - * @deprecated use {@link #filter(Predicate, Materialized) filter(predicate, Materialized.as(queryableStoreName))} - */ - @Deprecated - KTable filter(final Predicate predicate, final String queryableStoreName); - - /** - * Create a new {@code KTable} that consists of all records of this {@code KTable} which satisfy the given - * predicate. - * All records that do not satisfy the predicate are dropped. - * For each {@code KTable} update the filter is evaluated on the updated record to produce an updated record for the - * result {@code KTable}. - * This is a stateless record-by-record operation. - *

- * Note that {@code filter} for a changelog stream works differently than {@link KStream#filter(Predicate) - * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records) - * have delete semantics. - * Thus, for tombstones the provided filter predicate is not evaluated but the tombstone record is forwarded - * directly if required (i.e., if there is anything to be deleted). - * Furthermore, for each record that gets dropped (i.e., does not satisfy the given predicate) a tombstone record - * is forwarded. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // filtering words
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * K key = "some-word";
-     * V valueForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * - * @param predicate a filter {@link Predicate} that is applied to each record - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a {@code KTable} that contains only those records that satisfy the given predicate - * @see #filterNot(Predicate, Materialized) - * @deprecated use {@link #filter(Predicate, Materialized) filter(predicate, Materialized.as(KeyValueByteStoreSupplier))} - */ - @Deprecated - KTable filter(final Predicate predicate, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - /** * Create a new {@code KTable} that consists all records of this {@code KTable} which do not satisfy the * given predicate. @@ -265,81 +187,6 @@ KTable filter(final Predicate predicate, */ KTable filterNot(final Predicate predicate, final Materialized> materialized); - /** - * Create a new {@code KTable} that consists all records of this {@code KTable} which do not satisfy the - * given predicate. - * All records that do satisfy the predicate are dropped. - * For each {@code KTable} update the filter is evaluated on the updated record to produce an updated record for the - * result {@code KTable}. - * This is a stateless record-by-record operation. - *

- * Note that {@code filterNot} for a changelog stream works differently than {@link KStream#filterNot(Predicate) - * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records) - * have delete semantics. - * Thus, for tombstones the provided filter predicate is not evaluated but the tombstone record is forwarded - * directly if required (i.e., if there is anything to be deleted). - * Furthermore, for each record that gets dropped (i.e., does satisfy the given predicate) a tombstone record is - * forwarded. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // filtering words
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * K key = "some-word";
-     * V valueForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * @param predicate a filter {@link Predicate} that is applied to each record - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a {@code KTable} that contains only those records that do not satisfy the given predicate - * @see #filter(Predicate, Materialized) - * @deprecated use {@link #filterNot(Predicate, Materialized) filterNot(predicate, Materialized.as(KeyValueByteStoreSupplier))} - */ - @Deprecated - KTable filterNot(final Predicate predicate, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - - /** - * Create a new {@code KTable} that consists all records of this {@code KTable} which do not satisfy the - * given predicate. - * All records that do satisfy the predicate are dropped. - * For each {@code KTable} update the filter is evaluated on the updated record to produce an updated record for the - * result {@code KTable}. - * This is a stateless record-by-record operation. - *

- * Note that {@code filterNot} for a changelog stream works differently than {@link KStream#filterNot(Predicate) - * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records) - * have delete semantics. - * Thus, for tombstones the provided filter predicate is not evaluated but the tombstone record is forwarded - * directly if required (i.e., if there is anything to be deleted). - * Furthermore, for each record that gets dropped (i.e., does satisfy the given predicate) a tombstone record is - * forwarded. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // filtering words
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * K key = "some-word";
-     * V valueForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * @param predicate a filter {@link Predicate} that is applied to each record - * @param queryableStoreName a user-provided name of the underlying {@link KTable} that can be - * used to subsequently query the operation results; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then the results cannot be queried - * (i.e., that would be equivalent to calling {@link KTable#filterNot(Predicate)}. - * @return a {@code KTable} that contains only those records that do not satisfy the given predicate - * @see #filter(Predicate, Materialized) - * @deprecated use {@link #filter(Predicate, Materialized) filterNot(predicate, Materialized.as(queryableStoreName))} - */ - @Deprecated - KTable filterNot(final Predicate predicate, final String queryableStoreName); /** @@ -500,100 +347,6 @@ KTable mapValues(final ValueMapper mapper, KTable mapValues(final ValueMapperWithKey mapper, final Materialized> materialized); - /** - * Create a new {@code KTable} by transforming the value of each record in this {@code KTable} into a new value - * (with possible new type) in the new {@code KTable}. - * For each {@code KTable} update the provided {@link ValueMapper} is applied to the value of the updated record and - * computes a new value for it, resulting in an updated record for the result {@code KTable}. - * Thus, an input record {@code } can be transformed into an output record {@code }. - * This is a stateless record-by-record operation. - *

- * The example below counts the number of token of the value string. - * 

{@code
-     * KTable inputTable = builder.table("topic");
-     * KTable outputTable = inputTable.mapValue(new ValueMapper {
-     *     Integer apply(String value) {
-     *         return value.split(" ").length;
-     *     }
-     * });
-     * }
- *

- * To query the local {@link KeyValueStore} representing outputTable above it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- *

- * This operation preserves data co-location with respect to the key. - * Thus, no internal data redistribution is required if a key based operator (like a join) is applied to - * the result {@code KTable}. - *

- * Note that {@code mapValues} for a changelog stream works differently than {@link KStream#mapValues(ValueMapper) - * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records) - * have delete semantics. - * Thus, for tombstones the provided value-mapper is not evaluated but the tombstone record is forwarded directly to - * delete the corresponding record in the result {@code KTable}. - * - * @param mapper a {@link ValueMapper} that computes a new output value - * @param queryableStoreName a user-provided name of the underlying {@link KTable} that can be - * used to subsequently query the operation results; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then the results cannot be queried - * (i.e., that would be equivalent to calling {@link KTable#mapValues(ValueMapper)}. - * @param valueSerde serializer for new value type - * @param the value type of the result {@code KTable} - * - * @return a {@code KTable} that contains records with unmodified keys and new values (possibly of different type) - * @deprecated use {@link #mapValues(ValueMapper, Materialized) mapValues(mapper, Materialized.as(queryableStoreName).withValueSerde(valueSerde))} - */ - @Deprecated - KTable mapValues(final ValueMapper mapper, final Serde valueSerde, final String queryableStoreName); - - /** - * Create a new {@code KTable} by transforming the value of each record in this {@code KTable} into a new value - * (with possible new type) in the new {@code KTable}. - * For each {@code KTable} update the provided {@link ValueMapper} is applied to the value of the updated record and - * computes a new value for it, resulting in an updated record for the result {@code KTable}. - * Thus, an input record {@code } can be transformed into an output record {@code }. - * This is a stateless record-by-record operation. - *

- * The example below counts the number of token of the value string. - * 

{@code
-     * KTable inputTable = builder.table("topic");
-     * KTable outputTable = inputTable.mapValue(new ValueMapper {
-     *     Integer apply(String value) {
-     *         return value.split(" ").length;
-     *     }
-     * });
-     * }
- *

- * To query the local {@link KeyValueStore} representing outputTable above it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- *

- * This operation preserves data co-location with respect to the key. - * Thus, no internal data redistribution is required if a key based operator (like a join) is applied to - * the result {@code KTable}. - *

- * Note that {@code mapValues} for a changelog stream works differently than {@link KStream#mapValues(ValueMapper) - * record stream filters}, because {@link KeyValue records} with {@code null} values (so-called tombstone records) - * have delete semantics. - * Thus, for tombstones the provided value-mapper is not evaluated but the tombstone record is forwarded directly to - * delete the corresponding record in the result {@code KTable}. - * - * @param mapper a {@link ValueMapper} that computes a new output value - * @param valueSerde serializer for new value type - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @param the value type of the result {@code KTable} - * @return a {@code KTable} that contains records with unmodified keys and new values (possibly of different type) - * @deprecated use {@link #mapValues(ValueMapper, Materialized) mapValues(mapper, Materialized.as(KeyValueByteStoreSupplier).withValueSerde(valueSerde))} - */ - @Deprecated - KTable mapValues(final ValueMapper mapper, - final Serde valueSerde, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - /** * Print the updated records of this {@code KTable} to {@code System.out}. @@ -1327,40 +1080,6 @@ void to(final Serde keySerde, KGroupedTable groupBy(final KeyValueMapper> selector, final Serialized serialized); - /** - * Re-groups the records of this {@code KTable} using the provided {@link KeyValueMapper}. - * Each {@link KeyValue} pair of this {@code KTable} is mapped to a new {@link KeyValue} pair by applying the - * provided {@link KeyValueMapper}. - * Re-grouping a {@code KTable} is required before an aggregation operator can be applied to the data - * (cf. {@link KGroupedTable}). - * The {@link KeyValueMapper} selects a new key and value (both with potentially different type). - * If the new record key is {@code null} the record will not be included in the resulting {@link KGroupedTable} - *

- * Because a new key is selected, an internal repartitioning topic will be created in Kafka. - * This topic will be named "${applicationId}-XXX-repartition", where "applicationId" is user-specified in - * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is - * an internally generated name, and "-repartition" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - *

- * All data of this {@code KTable} will be redistributed through the repartitioning topic by writing all update - * records to and rereading all updated records from it, such that the resulting {@link KGroupedTable} is partitioned - * on the new key. - * - * @param selector a {@link KeyValueMapper} that computes a new grouping key and value to be aggregated - * @param keySerde key serdes for materializing this stream, - * if not specified the default serdes defined in the configs will be used - * @param valueSerde value serdes for materializing this stream, - * if not specified the default serdes defined in the configs will be used - * @param the key type of the result {@link KGroupedTable} - * @param the value type of the result {@link KGroupedTable} - * @return a {@link KGroupedTable} that contains the re-grouped records of the original {@code KTable} - * @deprecated use {@link #groupBy(KeyValueMapper, Serialized) groupBy(selector, Serialized.with(keySerde, valueSerde)} - */ - @Deprecated - KGroupedTable groupBy(final KeyValueMapper> selector, - final Serde keySerde, - final Serde valueSerde); - /** * Join records of this {@code KTable} with another {@code KTable}'s records using non-windowed inner equi join. * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. @@ -1511,9 +1230,14 @@ KTable join(final KTable other, KTable join(final KTable other, final ValueJoiner joiner, final Materialized> materialized); + + /** - * Join records of this {@code KTable} with another {@code KTable}'s records using non-windowed inner equi join. + * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using + * non-windowed left equi join. * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. + * In contrast to {@link #join(KTable, ValueJoiner) inner-join}, all records from left {@code KTable} will produce + * an output record (cf. below). * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result * of the join. *

@@ -1522,13 +1246,17 @@ KTable join(final KTable other, * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input * {@code KTable} the result gets updated. *

- * For each {@code KTable} record that finds a corresponding record in the other {@code KTable} the provided - * {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. + * For each {@code KTable} record that finds a corresponding record in the other {@code KTable}'s state the + * provided {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. + * Additionally, for each record of left {@code KTable} that does not find a corresponding record in the + * right {@code KTable}'s state the provided {@link ValueJoiner} will be called with {@code rightValue = + * null} to compute a value (with arbitrary type) for the result record. * The key of the result record is the same as for both joining input records. *

* Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. - * Thus, for input tombstones the provided value-joiner is not called but a tombstone record is forwarded - * directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be deleted). + * For example, for left input tombstones the provided value-joiner is not called but a tombstone record is + * forwarded directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be + * deleted). *

* Input records with {@code null} key will be dropped and no join computation is performed. *

@@ -1546,176 +1274,7 @@ KTable join(final KTable other, * <K1:A> * * - * - * - * - * - * <K1:A> - * <K1:b> - * <K1:b> - * <K1:ValueJoiner(A,b)> - * - * - * <K1:C> - * <K1:C> - * - * <K1:b> - * <K1:ValueJoiner(C,b)> - * - * - * - * <K1:C> - * <K1:null> - * - * <K1:null> - * - * - * Both input streams (or to be more precise, their underlying source topics) need to have the same number of - * partitions. - * - * @param other the other {@code KTable} to be joined with this {@code KTable} - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param the value type of the other {@code KTable} - * @param the value type of the result {@code KTable} - * @param joinSerde serializer for join result value type - * @param queryableStoreName a user-provided name of the underlying {@link KTable} that can be - * used to subsequently query the operation results; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then the results cannot be queried - * (i.e., that would be equivalent to calling {@link KTable#join(KTable, ValueJoiner)}. - * @return a {@code KTable} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key - * @see #leftJoin(KTable, ValueJoiner, Materialized) - * @see #outerJoin(KTable, ValueJoiner, Materialized) - * @deprecated use {@link #join(KTable, ValueJoiner, Materialized) join(other, joiner, Materialized.as(queryableStoreName).withValueSerde(joinSerde)} - */ - @Deprecated - KTable join(final KTable other, - final ValueJoiner joiner, - final Serde joinSerde, - final String queryableStoreName); - - /** - * Join records of this {@code KTable} with another {@code KTable}'s records using non-windowed inner equi join. - * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. - * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result - * of the join. - *

- * The join is computed by (1) updating the internal state of one {@code KTable} and (2) performing a lookup for a - * matching record in the current (i.e., processing time) internal state of the other {@code KTable}. - * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input - * {@code KTable} the result gets updated. - *

- * For each {@code KTable} record that finds a corresponding record in the other {@code KTable} the provided - * {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. - * The key of the result record is the same as for both joining input records. - *

- * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. - * Thus, for input tombstones the provided value-joiner is not called but a tombstone record is forwarded - * directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be deleted). - *

- * Input records with {@code null} key will be dropped and no join computation is performed. - *

- * Example: - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
thisKTablethisStateotherKTableotherStateresult updated record
<K1:A><K1:A>
<K1:A><K1:b><K1:b><K1:ValueJoiner(A,b)>
<K1:C><K1:C><K1:b><K1:ValueJoiner(C,b)>
<K1:C><K1:null><K1:null>
- * Both input streams (or to be more precise, their underlying source topics) need to have the same number of - * partitions. - * - * @param other the other {@code KTable} to be joined with this {@code KTable} - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param the value type of the other {@code KTable} - * @param the value type of the result {@code KTable} - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a {@code KTable} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key - * @see #leftJoin(KTable, ValueJoiner, Materialized) - * @see #outerJoin(KTable, ValueJoiner, Materialized) - * @deprecated use {@link #join(KTable, ValueJoiner, Materialized) join(other, joiner, Materialized.as(KeyValueByteStoreSupplier)} - */ - @Deprecated - KTable join(final KTable other, - final ValueJoiner joiner, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - - - /** - * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using - * non-windowed left equi join. - * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. - * In contrast to {@link #join(KTable, ValueJoiner) inner-join}, all records from left {@code KTable} will produce - * an output record (cf. below). - * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result - * of the join. - *

- * The join is computed by (1) updating the internal state of one {@code KTable} and (2) performing a lookup for a - * matching record in the current (i.e., processing time) internal state of the other {@code KTable}. - * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input - * {@code KTable} the result gets updated. - *

- * For each {@code KTable} record that finds a corresponding record in the other {@code KTable}'s state the - * provided {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. - * Additionally, for each record of left {@code KTable} that does not find a corresponding record in the - * right {@code KTable}'s state the provided {@link ValueJoiner} will be called with {@code rightValue = - * null} to compute a value (with arbitrary type) for the result record. - * The key of the result record is the same as for both joining input records. - *

- * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. - * For example, for left input tombstones the provided value-joiner is not called but a tombstone record is - * forwarded directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be - * deleted). - *

- * Input records with {@code null} key will be dropped and no join computation is performed. - *

- * Example: - * - * - * - * - * - * - * - * - * - * - * - * - * - * + * * * * @@ -1839,182 +1398,6 @@ KTable leftJoin(final KTable other, KTable leftJoin(final KTable other, final ValueJoiner joiner, final Materialized> materialized); - /** - * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using - * non-windowed left equi join. - * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. - * In contrast to {@link #join(KTable, ValueJoiner) inner-join}, all records from left {@code KTable} will produce - * an output record (cf. below). - * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result - * of the join. - *

- * The join is computed by (1) updating the internal state of one {@code KTable} and (2) performing a lookup for a - * matching record in the current (i.e., processing time) internal state of the other {@code KTable}. - * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input - * {@code KTable} the result gets updated. - *

- * For each {@code KTable} record that finds a corresponding record in the other {@code KTable}'s state the - * provided {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. - * Additionally, for each record of left {@code KTable} that does not find a corresponding record in the - * right {@code KTable}'s state the provided {@link ValueJoiner} will be called with {@code rightValue = - * null} to compute a value (with arbitrary type) for the result record. - * The key of the result record is the same as for both joining input records. - *

- * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. - * For example, for left input tombstones the provided value-joiner is not called but a tombstone record is - * forwarded directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be - * deleted). - *

- * Input records with {@code null} key will be dropped and no join computation is performed. - *

- * Example: - *

thisKTablethisStateotherKTableotherStateresult updated record
<K1:A><K1:A><K1:ValueJoiner(A,null)><K1:ValueJoiner(A,null)>
- * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
thisKTablethisStateotherKTableotherStateresult updated record
<K1:A><K1:A><K1:ValueJoiner(A,null)>
<K1:A><K1:b><K1:b><K1:ValueJoiner(A,b)>
<K1:null><K1:b><K1:null>
<K1:null>
- * Both input streams (or to be more precise, their underlying source topics) need to have the same number of - * partitions. - * - * @param other the other {@code KTable} to be joined with this {@code KTable} - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param the value type of the other {@code KTable} - * @param the value type of the result {@code KTable} - * @param joinSerde serializer for join result value type - * @param queryableStoreName a user-provided name of the underlying {@link KTable} that can be - * used to subsequently query the operation results; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then the results cannot be queried - * (i.e., that would be equivalent to calling {@link KTable#leftJoin(KTable, ValueJoiner)}. - * @return a {@code KTable} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of - * left {@code KTable} - * @see #join(KTable, ValueJoiner, Materialized) - * @see #outerJoin(KTable, ValueJoiner, Materialized) - * @deprecated use {@link #leftJoin(KTable, ValueJoiner, Materialized) leftJoin(other, joiner, Materialized.as(queryableStoreName).withValueSerde(joinSerde)} - */ - @Deprecated - KTable leftJoin(final KTable other, - final ValueJoiner joiner, - final Serde joinSerde, - final String queryableStoreName); - - /** - * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using - * non-windowed left equi join. - * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. - * In contrast to {@link #join(KTable, ValueJoiner) inner-join}, all records from left {@code KTable} will produce - * an output record (cf. below). - * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result - * of the join. - *

- * The join is computed by (1) updating the internal state of one {@code KTable} and (2) performing a lookup for a - * matching record in the current (i.e., processing time) internal state of the other {@code KTable}. - * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input - * {@code KTable} the result gets updated. - *

- * For each {@code KTable} record that finds a corresponding record in the other {@code KTable}'s state the - * provided {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. - * Additionally, for each record of left {@code KTable} that does not find a corresponding record in the - * right {@code KTable}'s state the provided {@link ValueJoiner} will be called with {@code rightValue = - * null} to compute a value (with arbitrary type) for the result record. - * The key of the result record is the same as for both joining input records. - *

- * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. - * For example, for left input tombstones the provided value-joiner is not called but a tombstone record is - * forwarded directly to delete a record in the result {@code KTable} if required (i.e., if there is anything to be - * deleted). - *

- * Input records with {@code null} key will be dropped and no join computation is performed. - *

- * Example: - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
thisKTablethisStateotherKTableotherStateresult updated record
<K1:A><K1:A><K1:ValueJoiner(A,null)>
<K1:A><K1:b><K1:b><K1:ValueJoiner(A,b)>
<K1:null><K1:b><K1:null>
<K1:null>
- * Both input streams (or to be more precise, their underlying source topics) need to have the same number of - * partitions. - * - * @param other the other {@code KTable} to be joined with this {@code KTable} - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param the value type of the other {@code KTable} - * @param the value type of the result {@code KTable} - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a {@code KTable} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of - * left {@code KTable} - * @see #join(KTable, ValueJoiner, Materialized) - * @see #outerJoin(KTable, ValueJoiner, Materialized) - * @deprecated use {@link #leftJoin(KTable, ValueJoiner, Materialized) leftJoin(other, joiner, Materialized.as(KeyValueByteStoreSupplier)} - */ - @Deprecated - KTable leftJoin(final KTable other, - final ValueJoiner joiner, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); /** @@ -2182,181 +1565,6 @@ KTable outerJoin(final KTable other, final ValueJoiner joiner, final Materialized> materialized); - /** - * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using - * non-windowed outer equi join. - * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. - * In contrast to {@link #join(KTable, ValueJoiner) inner-join} or {@link #leftJoin(KTable, ValueJoiner) left-join}, - * all records from both input {@code KTable}s will produce an output record (cf. below). - * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result - * of the join. - *

- * The join is computed by (1) updating the internal state of one {@code KTable} and (2) performing a lookup for a - * matching record in the current (i.e., processing time) internal state of the other {@code KTable}. - * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input - * {@code KTable} the result gets updated. - *

- * For each {@code KTable} record that finds a corresponding record in the other {@code KTable}'s state the - * provided {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. - * Additionally, for each record that does not find a corresponding record in the corresponding other - * {@code KTable}'s state the provided {@link ValueJoiner} will be called with {@code null} value for the - * corresponding other value to compute a value (with arbitrary type) for the result record. - * The key of the result record is the same as for both joining input records. - *

- * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. - * Thus, for input tombstones the provided value-joiner is not called but a tombstone record is forwarded directly - * to delete a record in the result {@code KTable} if required (i.e., if there is anything to be deleted). - *

- * Input records with {@code null} key will be dropped and no join computation is performed. - *

- * Example: - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
thisKTablethisStateotherKTableotherStateresult updated record
<K1:A><K1:A><K1:ValueJoiner(A,null)>
<K1:A><K1:b><K1:b><K1:ValueJoiner(A,b)>
<K1:null><K1:b><K1:ValueJoiner(null,b)>
<K1:null><K1:null>
- * Both input streams (or to be more precise, their underlying source topics) need to have the same number of - * partitions. - * - * @param other the other {@code KTable} to be joined with this {@code KTable} - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param the value type of the other {@code KTable} - * @param the value type of the result {@code KTable} - * @param joinSerde serializer for join result value type - * @param queryableStoreName a user-provided name of the underlying {@link KTable} that can be - * used to subsequently query the operation results; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then the results cannot be queried - * (i.e., that would be equivalent to calling {@link KTable#outerJoin(KTable, ValueJoiner)}. - * @return a {@code KTable} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of - * both {@code KTable}s - * @see #join(KTable, ValueJoiner, Materialized) - * @see #leftJoin(KTable, ValueJoiner, Materialized) - * @deprecated use {@link #outerJoin(KTable, ValueJoiner, Materialized) outerJoin(other, joiner, Materialized.as(queryableStoreName).withValueSerde(joinSerde)} - */ - @Deprecated - KTable outerJoin(final KTable other, - final ValueJoiner joiner, - final Serde joinSerde, - final String queryableStoreName); - - /** - * Join records of this {@code KTable} (left input) with another {@code KTable}'s (right input) records using - * non-windowed outer equi join. - * The join is a primary key join with join attribute {@code thisKTable.key == otherKTable.key}. - * In contrast to {@link #join(KTable, ValueJoiner) inner-join} or {@link #leftJoin(KTable, ValueJoiner) left-join}, - * all records from both input {@code KTable}s will produce an output record (cf. below). - * The result is an ever updating {@code KTable} that represents the current (i.e., processing time) result - * of the join. - *

- * The join is computed by (1) updating the internal state of one {@code KTable} and (2) performing a lookup for a - * matching record in the current (i.e., processing time) internal state of the other {@code KTable}. - * This happens in a symmetric way, i.e., for each update of either {@code this} or the {@code other} input - * {@code KTable} the result gets updated. - *

- * For each {@code KTable} record that finds a corresponding record in the other {@code KTable}'s state the - * provided {@link ValueJoiner} will be called to compute a value (with arbitrary type) for the result record. - * Additionally, for each record that does not find a corresponding record in the corresponding other - * {@code KTable}'s state the provided {@link ValueJoiner} will be called with {@code null} value for the - * corresponding other value to compute a value (with arbitrary type) for the result record. - * The key of the result record is the same as for both joining input records. - *

- * Note that {@link KeyValue records} with {@code null} values (so-called tombstone records) have delete semantics. - * Thus, for input tombstones the provided value-joiner is not called but a tombstone record is forwarded directly - * to delete a record in the result {@code KTable} if required (i.e., if there is anything to be deleted). - *

- * Input records with {@code null} key will be dropped and no join computation is performed. - *

- * Example: - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
thisKTablethisStateotherKTableotherStateresult updated record
<K1:A><K1:A><K1:ValueJoiner(A,null)>
<K1:A><K1:b><K1:b><K1:ValueJoiner(A,b)>
<K1:null><K1:b><K1:ValueJoiner(null,b)>
<K1:null><K1:null>
- * Both input streams (or to be more precise, their underlying source topics) need to have the same number of - * partitions. - * - * @param other the other {@code KTable} to be joined with this {@code KTable} - * @param joiner a {@link ValueJoiner} that computes the join result for a pair of matching records - * @param the value type of the other {@code KTable} - * @param the value type of the result {@code KTable} - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a {@code KTable} that contains join-records for each key and values computed by the given - * {@link ValueJoiner}, one for each matched record-pair with the same key plus one for each non-matching record of - * both {@code KTable}s - * @see #join(KTable, ValueJoiner) - * @see #leftJoin(KTable, ValueJoiner) - * @deprecated use {@link #outerJoin(KTable, ValueJoiner, Materialized) outerJoin(other, joiner, Materialized.as(KeyValueByteStoreSupplier)} - */ - @Deprecated - KTable outerJoin(final KTable other, - final ValueJoiner joiner, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - /** * Get the name of the local state store used that can be used to query this {@code KTable}. * diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 857abf7f1b1b6..9a6e5d577dc7b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -192,55 +192,6 @@ public KStream mapValues(final ValueMapperWithKey keySerde, - final Serde valSerde) { - print(defaultKeyValueMapper, keySerde, valSerde, this.name); - } - - @SuppressWarnings("deprecation") - @Override - public void print(final Serde keySerde, - final Serde valSerde, - final String label) { - print(defaultKeyValueMapper, keySerde, valSerde, label); - } - - @SuppressWarnings("deprecation") - @Override - public void print(final KeyValueMapper mapper) { - print(mapper, null, null, this.name); - } - - @SuppressWarnings("deprecation") - @Override - public void print(final KeyValueMapper mapper, - final String label) { - print(mapper, null, null, label); - } - - @SuppressWarnings("deprecation") - @Override - public void print(final KeyValueMapper mapper, - final Serde keySerde, - final Serde valSerde) { - print(mapper, keySerde, valSerde, this.name); - } - - @SuppressWarnings("deprecation") - @Override public void print(final KeyValueMapper mapper, final Serde keySerde, final Serde valSerde, @@ -259,61 +210,6 @@ public void print(final Printed printed) { } @SuppressWarnings("deprecation") - @Override - public void writeAsText(final String filePath) { - writeAsText(filePath, this.name, null, null, defaultKeyValueMapper); - } - - @SuppressWarnings("deprecation") - @Override - public void writeAsText(final String filePath, - final String label) { - writeAsText(filePath, label, null, null, defaultKeyValueMapper); - } - - @SuppressWarnings("deprecation") - @Override - public void writeAsText(final String filePath, - final Serde keySerde, - final Serde valSerde) { - writeAsText(filePath, this.name, keySerde, valSerde, defaultKeyValueMapper); - } - - @SuppressWarnings("deprecation") - @Override - public void writeAsText(final String filePath, - final String label, - final Serde keySerde, - final Serde valSerde) { - writeAsText(filePath, label, keySerde, valSerde, defaultKeyValueMapper); - } - - @SuppressWarnings("deprecation") - @Override - public void writeAsText(final String filePath, - final KeyValueMapper mapper) { - writeAsText(filePath, this.name, null, null, mapper); - } - - @SuppressWarnings("deprecation") - @Override - public void writeAsText(final String filePath, - final String label, - final KeyValueMapper mapper) { - writeAsText(filePath, label, null, null, mapper); - } - - @SuppressWarnings("deprecation") - @Override - public void writeAsText(final String filePath, - final Serde keySerde, - final Serde valSerde, - final KeyValueMapper mapper) { - writeAsText(filePath, this.name, keySerde, valSerde, mapper); - } - - @SuppressWarnings("deprecation") - @Override public void writeAsText(final String filePath, final String label, final Serde keySerde, @@ -400,7 +296,6 @@ private KStream merge(final InternalStreamsBuilder builder, } @SuppressWarnings("deprecation") - @Override public KStream through(final Serde keySerde, final Serde valSerde, final StreamPartitioner partitioner, String topic) { @@ -436,21 +331,6 @@ public KStream peek(final ForeachAction action) { return new KStreamImpl<>(builder, name, sourceNodes, repartitionRequired); } - @SuppressWarnings("deprecation") - @Override - public KStream through(final Serde keySerde, - final Serde valSerde, - final String topic) { - return through(topic, Produced.with(keySerde, valSerde)); - } - - @SuppressWarnings("deprecation") - @Override - public KStream through(final StreamPartitioner partitioner, - final String topic) { - return through(topic, Produced.streamPartitioner(partitioner)); - } - @Override public KStream through(final String topic) { return through(null, null, null, topic); @@ -461,31 +341,6 @@ public void to(final String topic) { to(topic, Produced.with(null, null, null)); } - @SuppressWarnings("deprecation") - @Override - public void to(final StreamPartitioner partitioner, - final String topic) { - to(topic, Produced.streamPartitioner(partitioner)); - } - - @SuppressWarnings("deprecation") - @Override - public void to(final Serde keySerde, - final Serde valSerde, - final String topic) { - to(topic, Produced.with(keySerde, valSerde)); - } - - @SuppressWarnings({"unchecked", "deprecation"}) - @Override - public void to(final Serde keySerde, - final Serde valSerde, - final StreamPartitioner partitioner, - final String topic) { - Objects.requireNonNull(topic, "topic can't be null"); - to(topic, Produced.with(keySerde, valSerde, partitioner)); - } - @SuppressWarnings("unchecked") @Override public void to(final String topic, final Produced produced) { @@ -562,18 +417,6 @@ public void process(final ProcessorSupplier processorSuppl } } - @SuppressWarnings("deprecation") - @Override - public KStream join(final KStream other, - final ValueJoiner joiner, - final JoinWindows windows, - final Serde keySerde, - final Serde thisValueSerde, - final Serde otherValueSerde) { - return doJoin(other, joiner, windows, Joined.with(keySerde, thisValueSerde, otherValueSerde), - new KStreamImplJoin(false, false)); - } - @Override public KStream join(final KStream other, final ValueJoiner joiner, @@ -590,17 +433,6 @@ public KStream join(final KStream otherStream, new KStreamImplJoin(false, false)); } - @SuppressWarnings("deprecation") - @Override - public KStream outerJoin(final KStream other, - final ValueJoiner joiner, - final JoinWindows windows, - final Serde keySerde, - final Serde thisValueSerde, - final Serde otherValueSerde) { - return outerJoin(other, joiner, windows, Joined.with(keySerde, thisValueSerde, otherValueSerde)); - } - @Override public KStream outerJoin(final KStream other, final ValueJoiner joiner, @@ -692,21 +524,6 @@ public boolean test(final K1 key, final V1 value) { return sourceName; } - @SuppressWarnings("deprecation") - @Override - public KStream leftJoin(final KStream other, - final ValueJoiner joiner, - final JoinWindows windows, - final Serde keySerde, - final Serde thisValSerde, - final Serde otherValueSerde) { - return doJoin(other, - joiner, - windows, - Joined.with(keySerde, thisValSerde, otherValueSerde), - new KStreamImplJoin(true, false)); - } - @Override public KStream leftJoin(final KStream other, final ValueJoiner joiner, @@ -748,15 +565,6 @@ public KStream join(final KTable other, } } - @SuppressWarnings("deprecation") - @Override - public KStream join(final KTable other, - final ValueJoiner joiner, - final Serde keySerde, - final Serde valueSerde) { - return join(other, joiner, Joined.with(keySerde, valueSerde, null)); - } - @Override public KStream leftJoin(final GlobalKTable globalTable, final KeyValueMapper keyMapper, @@ -822,15 +630,6 @@ public KStream leftJoin(final KTable other, } } - @SuppressWarnings("deprecation") - @Override - public KStream leftJoin(final KTable other, - final ValueJoiner joiner, - final Serde keySerde, - final Serde valueSerde) { - return leftJoin(other, joiner, Joined.with(keySerde, valueSerde, null)); - } - @Override public KGroupedStream groupBy(final KeyValueMapper selector) { return groupBy(selector, Serialized.with(null, null)); @@ -851,15 +650,6 @@ public KGroupedStream groupBy(final KeyValueMapper KGroupedStream groupBy(final KeyValueMapper selector, - final Serde keySerde, - final Serde valSerde) { - Objects.requireNonNull(selector, "selector can't be null"); - return groupBy(selector, Serialized.with(keySerde, valSerde)); - } - @Override public KGroupedStream groupByKey() { return groupByKey(Serialized.with(null, null)); @@ -877,13 +667,6 @@ public KGroupedStream groupByKey(final Serialized serialized) { } - @SuppressWarnings("deprecation") - @Override - public KGroupedStream groupByKey(final Serde keySerde, - final Serde valSerde) { - return groupByKey(Serialized.with(keySerde, valSerde)); - } - private static StoreBuilder> createWindowedStateStore(final JoinWindows windows, final Serde keySerde, final Serde valueSerde, diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java index 11b8c516a4d25..0c53dbbc9a0ea 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java @@ -194,7 +194,6 @@ public KTable filter(final Predicate predicate, } @SuppressWarnings("deprecation") - @Override public KTable filter(final Predicate predicate, final String queryableStoreName) { org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier = null; @@ -204,14 +203,6 @@ public KTable filter(final Predicate predicate, return doFilter(predicate, storeSupplier, false); } - @SuppressWarnings("deprecation") - @Override - public KTable filter(final Predicate predicate, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - return doFilter(predicate, storeSupplier, false); - } - @Override public KTable filterNot(final Predicate predicate) { return filterNot(predicate, (String) null); @@ -226,7 +217,6 @@ public KTable filterNot(final Predicate predicate, } @SuppressWarnings("deprecation") - @Override public KTable filterNot(final Predicate predicate, final String queryableStoreName) { org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier = null; @@ -236,14 +226,6 @@ public KTable filterNot(final Predicate predicate, return doFilter(predicate, storeSupplier, true); } - @SuppressWarnings("deprecation") - @Override - public KTable filterNot(final Predicate predicate, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - return doFilter(predicate, storeSupplier, true); - } - @SuppressWarnings("deprecation") private KTable doMapValues(final ValueMapperWithKey mapper, final Serde valueSerde, @@ -300,24 +282,6 @@ public KTable mapValues(final ValueMapperWithKey(builder, name, processorSupplier, sourceNodes, this.queryableStoreName, true); } - @SuppressWarnings("deprecation") - @Override - public KTable mapValues(final ValueMapper mapper, - final Serde valueSerde, - final String queryableStoreName) { - return mapValues(withKey(mapper), Materialized.>as(queryableStoreName). - withValueSerde(valueSerde).withKeySerde(this.keySerde)); - } - - @SuppressWarnings("deprecation") - @Override - public KTable mapValues(final ValueMapper mapper, - final Serde valueSerde, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - return doMapValues(withKey(mapper), valueSerde, storeSupplier); - } - @SuppressWarnings("deprecation") @Override public void print() { @@ -524,14 +488,14 @@ public KTable through(final String topic) { @SuppressWarnings("deprecation") @Override public void to(final String topic) { - to(null, null, null, topic); + //to(null, null, null, topic); } @SuppressWarnings("deprecation") @Override public void to(final StreamPartitioner partitioner, final String topic) { - to(null, null, partitioner, topic); + //to(null, null, partitioner, topic); } @SuppressWarnings("deprecation") @@ -539,7 +503,7 @@ public void to(final StreamPartitioner partitioner, public void to(final Serde keySerde, final Serde valSerde, final String topic) { - this.toStream().to(keySerde, valSerde, null, topic); + //this.toStream().to(keySerde, valSerde, null, topic); } @SuppressWarnings("deprecation") @@ -548,7 +512,7 @@ public void to(final Serde keySerde, final Serde valSerde, final StreamPartitioner partitioner, final String topic) { - this.toStream().to(keySerde, valSerde, partitioner, topic); + //this.toStream().to(keySerde, valSerde, partitioner, topic); } @Override @@ -586,24 +550,6 @@ public KTable join(final KTable other, return doJoin(other, joiner, new MaterializedInternal<>(materialized, builder, MERGE_NAME), false, false); } - @SuppressWarnings("deprecation") - @Override - public KTable join(final KTable other, - final ValueJoiner joiner, - final Serde joinSerde, - final String queryableStoreName) { - return doJoin(other, joiner, false, false, joinSerde, queryableStoreName); - } - - @SuppressWarnings("deprecation") - @Override - public KTable join(final KTable other, - final ValueJoiner joiner, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - return doJoin(other, joiner, false, false, storeSupplier); - } - @Override public KTable outerJoin(final KTable other, final ValueJoiner joiner) { @@ -617,24 +563,6 @@ public KTable outerJoin(final KTable other, return doJoin(other, joiner, new MaterializedInternal<>(materialized, builder, MERGE_NAME), true, true); } - @SuppressWarnings("deprecation") - @Override - public KTable outerJoin(final KTable other, - final ValueJoiner joiner, - final Serde joinSerde, - final String queryableStoreName) { - return doJoin(other, joiner, true, true, joinSerde, queryableStoreName); - } - - @SuppressWarnings("deprecation") - @Override - public KTable outerJoin(final KTable other, - final ValueJoiner joiner, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - return doJoin(other, joiner, true, true, storeSupplier); - } - @Override public KTable leftJoin(final KTable other, final ValueJoiner joiner) { @@ -652,24 +580,6 @@ public KTable leftJoin(final KTable other, false); } - @SuppressWarnings("deprecation") - @Override - public KTable leftJoin(final KTable other, - final ValueJoiner joiner, - final Serde joinSerde, - final String queryableStoreName) { - return doJoin(other, joiner, true, false, joinSerde, queryableStoreName); - } - - @SuppressWarnings("deprecation") - @Override - public KTable leftJoin(final KTable other, - final ValueJoiner joiner, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - return doJoin(other, joiner, true, false, storeSupplier); - } - @SuppressWarnings({"unchecked", "deprecation"}) private KTable doJoin(final KTable other, final ValueJoiner joiner, @@ -784,14 +694,6 @@ private KTable buildJoin(final AbstractStream other, return new KTableImpl<>(builder, joinMergeName, joinMerge, allSourceNodes, internalQueryableName, internalQueryableName != null); } - @SuppressWarnings("deprecation") - @Override - public KGroupedTable groupBy(final KeyValueMapper> selector, - final Serde keySerde, - final Serde valueSerde) { - return groupBy(selector, Serialized.with(keySerde, valueSerde)); - } - @Override public KGroupedTable groupBy(final KeyValueMapper> selector) { return this.groupBy(selector, Serialized.with(null, null)); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java index 44e139a28bdbf..e19eed057b334 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java @@ -34,6 +34,7 @@ import org.apache.kafka.streams.kstream.KGroupedStream; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.Reducer; import org.apache.kafka.streams.kstream.Serialized; import org.apache.kafka.streams.kstream.TimeWindows; @@ -180,7 +181,7 @@ public String apply(Windowed windowedKey, String value) { return windowedKey.key() + "@" + windowedKey.window().start(); } }) - .to(Serdes.String(), Serdes.String(), outputTopic); + .to(outputTopic, Produced.with(Serdes.String(), Serdes.String())); startStreams(); @@ -232,7 +233,7 @@ public void shouldGroupByKey() throws Exception { public String apply(final Windowed windowedKey, final Long value) { return windowedKey.key() + "@" + windowedKey.window().start(); } - }).to(Serdes.String(), Serdes.Long(), outputTopic); + }).to(outputTopic, Produced.with(Serdes.String(), Serdes.Long())); startStreams(); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamsFineGrainedAutoResetIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamsFineGrainedAutoResetIntegrationTest.java index 5415b58e3de9b..0f7df6b0f0d8b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamsFineGrainedAutoResetIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamsFineGrainedAutoResetIntegrationTest.java @@ -37,6 +37,7 @@ import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KStreamBuilder; +import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.test.IntegrationTest; import org.apache.kafka.test.StreamsTestUtils; import org.apache.kafka.test.TestCondition; @@ -193,9 +194,9 @@ private void shouldOnlyReadForEarliest( final KStream pattern2Stream = builder.stream(Pattern.compile("topic-[A-D]" + topicSuffix), Consumed.with(Topology.AutoOffsetReset.LATEST)); final KStream namedTopicsStream = builder.stream(Arrays.asList(topicY, topicZ)); - pattern1Stream.to(stringSerde, stringSerde, outputTopic); - pattern2Stream.to(stringSerde, stringSerde, outputTopic); - namedTopicsStream.to(stringSerde, stringSerde, outputTopic); + pattern1Stream.to(outputTopic, Produced.with(stringSerde, stringSerde)); + pattern2Stream.to(outputTopic, Produced.with(stringSerde, stringSerde)); + namedTopicsStream.to(outputTopic, Produced.with(stringSerde, stringSerde)); final Properties producerConfig = TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class); @@ -289,7 +290,7 @@ public void shouldThrowStreamsExceptionNoResetSpecified() throws InterruptedExce final StreamsBuilder builder = new StreamsBuilder(); final KStream exceptionStream = builder.stream(NOOP); - exceptionStream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC); + exceptionStream.to(DEFAULT_OUTPUT_TOPIC, Produced.with(stringSerde, stringSerde)); KafkaStreams streams = new KafkaStreams(builder.build(), localConfig); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/RegexSourceIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/RegexSourceIntegrationTest.java index 57ed161b084c5..d0361dc675c1f 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/RegexSourceIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/RegexSourceIntegrationTest.java @@ -33,6 +33,7 @@ import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.processor.ProcessorSupplier; import org.apache.kafka.streams.processor.TopologyBuilder; import org.apache.kafka.streams.processor.internals.DefaultKafkaClientSupplier; @@ -143,7 +144,7 @@ public void testRegexMatchesTopicsAWhenCreated() throws Exception { final KStream pattern1Stream = builder.stream(Pattern.compile("TEST-TOPIC-\\d")); - pattern1Stream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC); + pattern1Stream.to(DEFAULT_OUTPUT_TOPIC, Produced.with(stringSerde, stringSerde)); final List assignedTopics = new ArrayList<>(); streams = new KafkaStreams(builder.build(), streamsConfiguration, new DefaultKafkaClientSupplier() { @Override @@ -191,7 +192,7 @@ public void testRegexMatchesTopicsAWhenDeleted() throws Exception { final KStream pattern1Stream = builder.stream(Pattern.compile("TEST-TOPIC-[A-Z]")); - pattern1Stream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC); + pattern1Stream.to(DEFAULT_OUTPUT_TOPIC, Produced.with(stringSerde, stringSerde)); final List assignedTopics = new ArrayList<>(); streams = new KafkaStreams(builder.build(), streamsConfiguration, new DefaultKafkaClientSupplier() { @@ -280,9 +281,9 @@ public void testShouldReadFromRegexAndNamedTopics() throws Exception { final KStream pattern2Stream = builder.stream(Pattern.compile("topic-[A-D]")); final KStream namedTopicsStream = builder.stream(Arrays.asList(TOPIC_Y, TOPIC_Z)); - pattern1Stream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC); - pattern2Stream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC); - namedTopicsStream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC); + pattern1Stream.to(DEFAULT_OUTPUT_TOPIC, Produced.with(stringSerde, stringSerde)); + pattern2Stream.to(DEFAULT_OUTPUT_TOPIC, Produced.with(stringSerde, stringSerde)); + namedTopicsStream.to(DEFAULT_OUTPUT_TOPIC, Produced.with(stringSerde, stringSerde)); streams = new KafkaStreams(builder.build(), streamsConfiguration); streams.start(); @@ -326,8 +327,8 @@ public void testMultipleConsumersCanReadFromPartitionedTopic() throws Exception final KStream partitionedStreamFollower = builderFollower.stream(Pattern.compile("partitioned-\\d")); - partitionedStreamLeader.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC); - partitionedStreamFollower.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC); + partitionedStreamLeader.to(DEFAULT_OUTPUT_TOPIC, Produced.with(stringSerde, stringSerde)); + partitionedStreamFollower.to(DEFAULT_OUTPUT_TOPIC, Produced.with(stringSerde, stringSerde)); final List leaderAssignment = new ArrayList<>(); final List followerAssignment = new ArrayList<>(); @@ -397,8 +398,8 @@ public void testNoMessagesSentExceptionFromOverlappingPatterns() throws Exceptio final KStream pattern2Stream = builder.stream(Pattern.compile("f.*")); - pattern1Stream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC); - pattern2Stream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC); + pattern1Stream.to(DEFAULT_OUTPUT_TOPIC, Produced.with(stringSerde, stringSerde)); + pattern2Stream.to(DEFAULT_OUTPUT_TOPIC, Produced.with(stringSerde, stringSerde)); streams = new KafkaStreams(builder.build(), streamsConfiguration); streams.start(); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java index ef65bb32b36d6..98d10010712cd 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java @@ -243,7 +243,7 @@ public void shouldUseRecordMetadataTimestampExtractorWhenInternalRepartitioningT Joined.with(Serdes.String(), Serdes.String(), Serdes.String())) - .to(Serdes.String(), Serdes.String(), "output-topic"); + .to("output-topic", Produced.with(Serdes.String(), Serdes.String())); ProcessorTopology processorTopology = builder.setApplicationId("X").build(null); SourceNode originalSourceNode = processorTopology.source("topic-1"); @@ -262,7 +262,7 @@ public void testToWithNullValueSerdeDoesntNPE() { final StreamsBuilder builder = new StreamsBuilder(); final Consumed consumed = Consumed.with(stringSerde, stringSerde); final KStream inputStream = builder.stream(Collections.singleton("input"), consumed); - inputStream.to(stringSerde, null, "output"); + inputStream.to("output", Produced.with(stringSerde, stringSerde)); } @Test(expected = NullPointerException.class) @@ -295,16 +295,6 @@ public void shouldNotAllowNullMapperOnMapValuesWithKey() { testStream.mapValues((ValueMapperWithKey) null); } - @Test(expected = NullPointerException.class) - public void shouldNotAllowNullFilePathOnWriteAsText() { - testStream.writeAsText(null); - } - - @Test(expected = TopologyException.class) - public void shouldNotAllowEmptyFilePathOnWriteAsText() { - testStream.writeAsText("\t \t"); - } - @Test(expected = NullPointerException.class) public void shouldNotAllowNullMapperOnFlatMap() { testStream.flatMap(null); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java index 657e05d11fa20..a0ca8274782b3 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java @@ -115,7 +115,7 @@ public void testQueryableKTable() { public boolean test(String key, Integer value) { return (value % 2) == 0; } - }, "anyStoreNameFilter"); + }); KTable table3 = table1.filterNot(new Predicate() { @Override public boolean test(String key, Integer value) { diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java index a7aed2e55024c..8646380d80c64 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java @@ -442,24 +442,7 @@ public void shouldNotAllowNullOtherTableOnJoin() { @Test public void shouldAllowNullStoreInJoin() { - table.join(table, MockValueJoiner.TOSTRING_JOINER, null, null); - } - - @SuppressWarnings("unchecked") - @Test(expected = NullPointerException.class) - public void shouldNotAllowNullStoreSupplierInJoin() { - table.join(table, MockValueJoiner.TOSTRING_JOINER, (StateStoreSupplier) null); - } - - @SuppressWarnings("unchecked") - @Test(expected = NullPointerException.class) - public void shouldNotAllowNullStoreSupplierInLeftJoin() { - table.leftJoin(table, MockValueJoiner.TOSTRING_JOINER, (StateStoreSupplier) null); - } - - @Test(expected = NullPointerException.class) - public void shouldNotAllowNullStoreSupplierInOuterJoin() { - table.outerJoin(table, MockValueJoiner.TOSTRING_JOINER, (StateStoreSupplier) null); + table.join(table, MockValueJoiner.TOSTRING_JOINER); } @Test(expected = NullPointerException.class) diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoinTest.java index 98d3e5225749c..54707da59d549 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoinTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableInnerJoinTest.java @@ -18,11 +18,14 @@ import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.Consumed; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsBuilderTest; import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.test.KStreamTestDriver; import org.apache.kafka.test.MockProcessorSupplier; import org.apache.kafka.test.MockValueJoiner; @@ -44,12 +47,14 @@ public class KTableKTableInnerJoinTest { - final private String topic1 = "topic1"; - final private String topic2 = "topic2"; + private final String topic1 = "topic1"; + private final String topic2 = "topic2"; - final private Serde intSerde = Serdes.Integer(); - final private Serde stringSerde = Serdes.String(); + private final Serde intSerde = Serdes.Integer(); + private final Serde stringSerde = Serdes.String(); private final Consumed consumed = Consumed.with(intSerde, stringSerde); + private final Materialized> materialized = Materialized.with(intSerde, stringSerde); + private File stateDir = null; @Rule public final KStreamTestDriver driver = new KStreamTestDriver(); @@ -181,16 +186,16 @@ public void testQueryableJoin() { final KTable table1; final KTable table2; - final KTable joined; + final KTable table3; final MockProcessorSupplier processor; processor = new MockProcessorSupplier<>(); table1 = builder.table(topic1, consumed); table2 = builder.table(topic2, consumed); - joined = table1.join(table2, MockValueJoiner.TOSTRING_JOINER, Serdes.String(), "anyQueryableName"); - joined.toStream().process(processor); + table3 = table1.join(table2, MockValueJoiner.TOSTRING_JOINER, materialized); + table3.toStream().process(processor); - doTestJoin(builder, expectedKeys, processor, joined); + doTestJoin(builder, expectedKeys, processor, table3); } private void doTestSendingOldValues(final StreamsBuilder builder, @@ -302,16 +307,16 @@ public void testQueryableNotSendingOldValues() { final KTable table1; final KTable table2; - final KTable joined; + final KTable table3; final MockProcessorSupplier proc; table1 = builder.table(topic1, consumed); table2 = builder.table(topic2, consumed); - joined = table1.join(table2, MockValueJoiner.TOSTRING_JOINER, Serdes.String(), "anyQueryableName"); + table3 = table1.join(table2, MockValueJoiner.TOSTRING_JOINER, materialized); proc = new MockProcessorSupplier<>(); - builder.build().addProcessor("proc", proc, ((KTableImpl) joined).name); + builder.build().addProcessor("proc", proc, ((KTableImpl) table3).name); - doTestSendingOldValues(builder, expectedKeys, table1, table2, proc, joined, false); + doTestSendingOldValues(builder, expectedKeys, table1, table2, proc, table3, false); } diff --git a/streams/src/test/java/org/apache/kafka/streams/perf/YahooBenchmark.java b/streams/src/test/java/org/apache/kafka/streams/perf/YahooBenchmark.java index 9c7768068668c..e9785322e017b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/perf/YahooBenchmark.java +++ b/streams/src/test/java/org/apache/kafka/streams/perf/YahooBenchmark.java @@ -32,6 +32,7 @@ import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.kstream.ForeachAction; +import org.apache.kafka.streams.kstream.Joined; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.KeyValueMapper; @@ -321,7 +322,7 @@ public CampaignAd apply(String value) { public String apply(ProjectedEvent value1, CampaignAd value2) { return value2.campaignID; } - }, Serdes.String(), Serdes.serdeFrom(projectedEventSerializer, projectedEventDeserializer)); + }, Joined.with(Serdes.String(), Serdes.serdeFrom(projectedEventSerializer, projectedEventDeserializer), null)); // key by campaign rather than by ad as original From dfed53b087b5d9c73517cfb932ef17e41e9032bb Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 23 Apr 2018 23:21:22 -0700 Subject: [PATCH 02/13] remove deprecated apis in KGroupedTable and KGroupedStream --- .../kafka/streams/kstream/KGroupedStream.java | 1370 +---------------- .../kafka/streams/kstream/KGroupedTable.java | 560 ------- .../kstream/internals/KGroupedStreamImpl.java | 161 -- .../kstream/internals/KGroupedTableImpl.java | 135 +- ...StreamAggregationDedupIntegrationTest.java | 19 +- .../KStreamAggregationIntegrationTest.java | 18 +- .../streams/kstream/KStreamBuilderTest.java | 6 +- .../internals/InternalStreamsBuilderTest.java | 4 +- .../internals/KGroupedStreamImplTest.java | 244 +-- .../internals/KGroupedTableImplTest.java | 31 +- .../internals/KStreamWindowAggregateTest.java | 12 +- .../internals/KTableAggregateTest.java | 20 +- .../kstream/internals/KTableFilterTest.java | 4 +- .../kstream/internals/KTableImplTest.java | 4 +- .../internals/KTableKTableLeftJoinTest.java | 5 +- .../kafka/streams/perf/YahooBenchmark.java | 6 +- .../internals/StreamsMetadataStateTest.java | 10 +- 17 files changed, 198 insertions(+), 2411 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java index 29de64c1e67c3..f7069e9074dfe 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java @@ -46,49 +46,6 @@ @InterfaceStability.Evolving public interface KGroupedStream { - /** - * Count the number of records in this stream by the grouped key. - * Records with {@code null} key or value are ignored. - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * that can be queried using the provided {@code queryableStoreName}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // counting words
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-word";
-     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * Therefore, the store name must be a valid Kafka topic name and cannot contain characters other than ASCII - * alphanumerics, '.', '_' and '-'. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param queryableStoreName the name of the underlying {@link KTable} state store; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then this will be equivalent to {@link KGroupedStream#count()}. - * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that - * represent the latest (rolling) count (i.e., number of records) for each key - * @deprecated use {@link #count(Materialized) count(Materialized.as(queryableStoreName))} - */ - @Deprecated - KTable count(final String queryableStoreName); - /** * Count the number of records in this stream by the grouped key. * Records with {@code null} key or value are ignored. @@ -115,41 +72,6 @@ public interface KGroupedStream { */ KTable count(); - /** - * Count the number of records in this stream by the grouped key. - * Records with {@code null} key or value are ignored. - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * provided by the given {@code storeSupplier}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * Use {@link org.apache.kafka.streams.processor.StateStore#name()} to get the store name: - * 

{@code
-     * KafkaStreams streams = ... // counting words
-     * String queryableStoreName = storeSupplier.name();
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-word";
-     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - * - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that - * represent the latest (rolling) count (i.e., number of records) for each key - * @deprecated use {@link #count(Materialized) count(Materialized.as(KeyValueByteStoreSupplier))} - */ - @Deprecated - KTable count(final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - /** * Count the number of records in this stream by the grouped key. * Records with {@code null} key or value are ignored. @@ -183,239 +105,6 @@ public interface KGroupedStream { */ KTable count(final Materialized> materialized); - /** - * Count the number of records in this stream by the grouped key and the defined windows. - * Records with {@code null} key or value are ignored. - * The specified {@code windows} define either hopping time windows that can be overlapping or tumbling (c.f. - * {@link TimeWindows}) or they define landmark windows (c.f. {@link UnlimitedWindows}). - * The result is written into a local windowed {@link KeyValueStore} (which is basically an ever-updating - * materialized view) that can be queried using the provided {@code queryableStoreName}. - * Windows are retained until their retention time expires (c.f. {@link Windows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local windowed {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // counting words
-     * ReadOnlyWindowStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.windowStore());
-     * String key = "some-word";
-     * long fromTime = ...;
-     * long toTime = ...;
-     * WindowStoreIterator countForWordsForWindows = localWindowStore.fetch(key, timeFrom, timeTo); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * Therefore, the store name must be a valid Kafka topic name and cannot contain characters other than ASCII - * alphanumerics, '.', '_' and '-'. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param windows the specification of the aggregation {@link Windows} - * @param queryableStoreName the name of the underlying {@link KTable} state store; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then this will be equivalent to {@link KGroupedStream#count(Windows)}. - * @return a windowed {@link KTable} that contains "update" records with unmodified keys and {@link Long} values - * that represent the latest (rolling) count (i.e., number of records) for each key within a window. - * @deprecated use {@link #windowedBy(Windows) windowedBy(windows)} followed by - * {@link TimeWindowedKStream#count(Materialized) count(Materialized.as(queryableStoreName))} - */ - @Deprecated - KTable, Long> count(final Windows windows, - final String queryableStoreName); - - /** - * Count the number of records in this stream by the grouped key and the defined windows. - * Records with {@code null} key or value are ignored. - * The specified {@code windows} define either hopping time windows that can be overlapping or tumbling (c.f. - * {@link TimeWindows}) or they define landmark windows (c.f. {@link UnlimitedWindows}). - * The result is written into a local windowed {@link KeyValueStore} (which is basically an ever-updating - * materialized view) that can be queried using the provided {@code queryableName}. - * Windows are retained until their retention time expires (c.f. {@link Windows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name - * and "-changelog" is a fixed suffix. - * Note that the internal store name may not be queriable through Interactive Queries. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param windows the specification of the aggregation {@link Windows} - * @return a windowed {@link KTable} that contains "update" records with unmodified keys and {@link Long} values - * that represent the latest (rolling) count (i.e., number of records) for each key within a window - * @deprecated use {@link #windowedBy(Windows) windowedBy(windows)} followed by {@link TimeWindowedKStream#count() count()} - */ - @Deprecated - KTable, Long> count(final Windows windows); - - /** - * Count the number of records in this stream by the grouped key and the defined windows. - * Records with {@code null} key or value are ignored. - * The specified {@code windows} define either hopping time windows that can be overlapping or tumbling (c.f. - * {@link TimeWindows}) or they define landmark windows (c.f. {@link UnlimitedWindows}). - * The result is written into a local windowed {@link KeyValueStore} (which is basically an ever-updating - * materialized view) provided by the given {@code storeSupplier}. - * Windows are retained until their retention time expires (c.f. {@link Windows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local windowed {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * Use {@link org.apache.kafka.streams.processor.StateStoreSupplier#name()} to get the store name: - * 

{@code
-     * KafkaStreams streams = ... // counting words
-     * String queryableStoreName = storeSupplier.name();
-     * ReadOnlyWindowStore localWindowStore = streams.store(queryableName, QueryableStoreTypes.windowStore());
-     * String key = "some-word";
-     * long fromTime = ...;
-     * long toTime = ...;
-     * WindowStoreIterator countForWordsForWindows = localWindowStore.fetch(key, timeFrom, timeTo); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - * - * @param windows the specification of the aggregation {@link Windows} - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a windowed {@link KTable} that contains "update" records with unmodified keys and {@link Long} values - * that represent the latest (rolling) count (i.e., number of records) for each key within a window - * @deprecated use {@link #windowedBy(Windows) windowedBy(windows)} followed by - * {@link TimeWindowedKStream#count(Materialized) count(Materialized.as(KeyValueByteStoreSupplier))} - */ - @Deprecated - KTable, Long> count(final Windows windows, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - - - /** - * Count the number of records in this stream by the grouped key into {@link SessionWindows}. - * Records with {@code null} key or value are ignored. - * The result is written into a local {@link SessionStore} (which is basically an ever-updating - * materialized view) that can be queried using the provided {@code queryableStoreName}. - * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local windowed {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * 

{@code
-     * KafkaStreams streams = ... // compute sum
-     * ReadOnlySessionStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.ReadOnlySessionStore);
-     * String key = "some-key";
-     * KeyValueIterator, Long> sumForKeyForWindows = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - * - * @param sessionWindows the specification of the aggregation {@link SessionWindows} - * @param queryableStoreName the name of the state store created from this operation; valid characters are ASCII - * alphanumerics, '.', '_' and '-. If {@code null} then this will be equivalent to {@link KGroupedStream#count(SessionWindows)}. - * @return a windowed {@link KTable} that contains "update" records with unmodified keys and {@link Long} values - * that represent the latest (rolling) count (i.e., number of records) for each key within a window - * @deprecated use {@link #windowedBy(SessionWindows) windowedBy(sessionWindows)} followed by - * {@link SessionWindowedKStream#count(Materialized) count(Materialized.as(queryableStoreName))} - */ - @Deprecated - KTable, Long> count(final SessionWindows sessionWindows, final String queryableStoreName); - - /** - * Count the number of records in this stream by the grouped key into {@link SessionWindows}. - * Records with {@code null} key or value are ignored. - * The result is written into a local {@link SessionStore} (which is basically an ever-updating - * materialized view) that can be queried using the provided {@code queryableStoreName}. - * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - * - * @param sessionWindows the specification of the aggregation {@link SessionWindows} - * @return a windowed {@link KTable} that contains "update" records with unmodified keys and {@link Long} values - * that represent the latest (rolling) count (i.e., number of records) for each key within a window - * @deprecated use {@link #windowedBy(SessionWindows) windowedBy(sessionWindows)} followed by - * {@link SessionWindowedKStream#count() count()} - */ - @Deprecated - KTable, Long> count(final SessionWindows sessionWindows); - - /** - * Count the number of records in this stream by the grouped key into {@link SessionWindows}. - * Records with {@code null} key or value are ignored. - * The result is written into a local {@link SessionStore} (which is basically an ever-updating materialized view) - * provided by the given {@code storeSupplier}. - * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local windowed {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * Use {@link org.apache.kafka.streams.processor.StateStoreSupplier#name()} to get the store name: - * 

{@code
-     * KafkaStreams streams = ... // compute sum
-     * Sting queryableStoreName = storeSupplier.name();
-     * ReadOnlySessionStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.ReadOnlySessionStore);
-     * String key = "some-key";
-     * KeyValueIterator, Long> sumForKeyForWindows = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - * - * @param sessionWindows the specification of the aggregation {@link SessionWindows} - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a windowed {@link KTable} that contains "update" records with unmodified keys and {@link Long} values - * that represent the latest (rolling) count (i.e., number of records) for each key within a window - * @deprecated use {@link #windowedBy(SessionWindows) windowedBy(sessionWindows)} followed by - * {@link SessionWindowedKStream#count(Materialized) count(Materialized.as(KeyValueByteStoreSupplier))} - */ - @Deprecated - KTable, Long> count(final SessionWindows sessionWindows, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); /** * Combine the values of records in this stream by the grouped key. @@ -452,13 +141,14 @@ KTable, Long> count(final SessionWindows sessionWindows, */ KTable reduce(final Reducer reducer); + /** - * Combine the values of records in this stream by the grouped key. + * Combine the value of records in this stream by the grouped key. * Records with {@code null} key or value are ignored. * Combining implies that the type of the aggregate result is the same as the type of the input value - * (c.f. {@link #aggregate(Initializer, Aggregator, Serde, String)}). + * (c.f. {@link #aggregate(Initializer, Aggregator, Materialized)}). * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * that can be queried using the provided {@code queryableStoreName}. + * provided by the given {@code materialized}. * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. *

* The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current @@ -474,7 +164,8 @@ KTable, Long> count(final SessionWindows sessionWindows, *

* If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's * value as-is. - * Thus, {@code reduce(Reducer, String)} can be used to compute aggregate functions like sum, min, or max. + * Thus, {@code reduce(Reducer, Materialized)} can be used to compute aggregate functions like sum, min, or + * max. *

* Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to * the same key. @@ -484,61 +175,42 @@ KTable, Long> count(final SessionWindows sessionWindows, * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. *

* To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: + * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. * 

{@code
      * KafkaStreams streams = ... // compute sum
+     * String queryableStoreName = "storeName" // the queryableStoreName should be the name of the store as defined by the Materialized instance
      * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
      * String key = "some-key";
      * Long sumForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
      * }
* For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * Therefore, the store name must be a valid Kafka topic name and cannot contain characters other than ASCII - * alphanumerics, '.', '_' and '-'. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. * - * @param reducer a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}. - * @param queryableStoreName the name of the underlying {@link KTable} state store; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then this will be equivalent to {@link KGroupedStream#reduce(Reducer)} ()}. + * @param reducer a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}. + * @param materialized an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}. * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the * latest (rolling) aggregate for each key - * @deprecated use {@link #reduce(Reducer, Materialized) reduce(reducer, Materialized.as(queryableStoreName))} */ - @Deprecated KTable reduce(final Reducer reducer, - final String queryableStoreName); + final Materialized> materialized); /** - * Combine the value of records in this stream by the grouped key. + * Aggregate the values of records in this stream by the grouped key. * Records with {@code null} key or value are ignored. - * Combining implies that the type of the aggregate result is the same as the type of the input value - * (c.f. {@link #aggregate(Initializer, Aggregator, org.apache.kafka.streams.processor.StateStoreSupplier)}). + * Aggregating is a generalization of {@link #reduce(Reducer) combining via reduce(...)} as it, for example, + * allows the result to have a different type than the input values. * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * provided by the given {@code storeSupplier}. + * that can be queried using the provided {@code queryableStoreName}. * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. *

- * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current - * aggregate (first argument) and the record's value (second argument): - * 

{@code
-     * // At the example of a Reducer
-     * new Reducer() {
-     *   public Long apply(Long aggValue, Long currValue) {
-     *     return aggValue + currValue;
-     *   }
-     * }
-     * }
- *

- * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's - * value as-is. - * Thus, {@code reduce(Reducer, org.apache.kafka.streams.processor.StateStoreSupplier)} can be used to compute - * aggregate functions like sum, min, or max. + * The specified {@link Initializer} is applied once directly before the first input record is processed to + * provide an initial intermediate aggregation result that is used to process the first record. + * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current + * aggregate (or for the very first record using the intermediate aggregation result provided via the + * {@link Initializer}) and the record's value. + * Thus, {@code aggregate(Initializer, Aggregator, Serde, String)} can be used to compute aggregate functions like + * count (c.f. {@link #count()}). *

* Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to * the same key. @@ -548,550 +220,39 @@ KTable reduce(final Reducer reducer, * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. *

* To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * Use {@link org.apache.kafka.streams.processor.StateStoreSupplier#name()} to get the store name: + * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: * 

{@code
-     * KafkaStreams streams = ... // compute sum
-     * String queryableStoreName = storeSupplier.name();
+     * KafkaStreams streams = ... // some aggregation on value type double
      * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
      * String key = "some-key";
-     * Long sumForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * Long aggForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
      * }
* For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to * query the value of the key on a parallel running instance of your Kafka Streams application. + *

+ * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. + * Therefore, the store name must be a valid Kafka topic name and cannot contain characters other than ASCII + * alphanumerics, '.', '_' and '-'. + * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is + * user-specified in {@link StreamsConfig} via parameter + * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the + * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. + * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. * - * @param reducer a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}. - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. + * @param initializer an {@link Initializer} that computes an initial intermediate aggregation result + * @param aggregator an {@link Aggregator} that computes a new aggregate result + * @param materialized an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}. + * @param the value type of the resulting {@link KTable} * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the * latest (rolling) aggregate for each key - * @deprecated use {@link #reduce(Reducer, Materialized) reduce(reducer, Materialized.as(KeyValueByteStoreSupplier))} */ - @Deprecated - KTable reduce(final Reducer reducer, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); + KTable aggregate(final Initializer initializer, + final Aggregator aggregator, + final Materialized> materialized); + /** - * Combine the value of records in this stream by the grouped key. - * Records with {@code null} key or value are ignored. - * Combining implies that the type of the aggregate result is the same as the type of the input value - * (c.f. {@link #aggregate(Initializer, Aggregator, Materialized)}). - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * provided by the given {@code materialized}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current - * aggregate (first argument) and the record's value (second argument): - * 

{@code
-     * // At the example of a Reducer
-     * new Reducer() {
-     *   public Long apply(Long aggValue, Long currValue) {
-     *     return aggValue + currValue;
-     *   }
-     * }
-     * }
- *

- * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's - * value as-is. - * Thus, {@code reduce(Reducer, Materialized)} can be used to compute aggregate functions like sum, min, or - * max. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * 

{@code
-     * KafkaStreams streams = ... // compute sum
-     * String queryableStoreName = "storeName" // the queryableStoreName should be the name of the store as defined by the Materialized instance
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-key";
-     * Long sumForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - * - * @param reducer a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}. - * @param materialized an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}. - * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - */ - KTable reduce(final Reducer reducer, - final Materialized> materialized); - - /** - * Combine the number of records in this stream by the grouped key and the defined windows. - * Records with {@code null} key or value are ignored. - * Combining implies that the type of the aggregate result is the same as the type of the input value - * (c.f. {@link #aggregate(Initializer, Aggregator, Windows, Serde, String)}). - * The specified {@code windows} define either hopping time windows that can be overlapping or tumbling (c.f. - * {@link TimeWindows}) or they define landmark windows (c.f. {@link UnlimitedWindows}). - * The result is written into a local windowed {@link KeyValueStore} (which is basically an ever-updating - * materialized view) that can be queried using the provided {@code queryableStoreName}. - * Windows are retained until their retention time expires (c.f. {@link Windows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current - * aggregate (first argument) and the record's value (second argument): - * 

{@code
-     * // At the example of a Reducer
-     * new Reducer() {
-     *   public Long apply(Long aggValue, Long currValue) {
-     *     return aggValue + currValue;
-     *   }
-     * }
-     * }
- *

- * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's - * value as-is. - * Thus, {@code reduce(Reducer, Windows, String)} can be used to compute aggregate functions like sum, min, or max. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local windowed {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // compute sum
-     * ReadOnlyWindowStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.windowStore());
-     * String key = "some-key";
-     * long fromTime = ...;
-     * long toTime = ...;
-     * WindowStoreIterator sumForKeyForWindows = localWindowStore.fetch(key, timeFrom, timeTo); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * Therefore, the store name must be a valid Kafka topic name and cannot contain characters other than ASCII - * alphanumerics, '.', '_' and '-'. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param reducer a {@link Reducer} that computes a new aggregate result - * @param windows the specification of the aggregation {@link Windows} - * @param queryableStoreName the name of the state store created from this operation; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then this will be equivalent to {@link KGroupedStream#reduce(Reducer, Windows)}. - * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent - * the latest (rolling) aggregate for each key within a window - * @deprecated use {@link #windowedBy(Windows) windowedBy(windows)} followed by - * {@link TimeWindowedKStream#reduce(Reducer, Materialized) reduce(reducer, Materialized.as(queryableStoreName))} - */ - @Deprecated - KTable, V> reduce(final Reducer reducer, - final Windows windows, - final String queryableStoreName); - - /** - * Combine the number of records in this stream by the grouped key and the defined windows. - * Records with {@code null} key or value are ignored. - * Combining implies that the type of the aggregate result is the same as the type of the input value - * (c.f. {@link #aggregate(Initializer, Aggregator, Windows, Serde, String)}). - * The specified {@code windows} define either hopping time windows that can be overlapping or tumbling (c.f. - * {@link TimeWindows}) or they define landmark windows (c.f. {@link UnlimitedWindows}). - * The result is written into a local windowed {@link KeyValueStore} (which is basically an ever-updating - * materialized view) that can be queried using the provided {@code queryableStoreName}. - * Windows are retained until their retention time expires (c.f. {@link Windows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current - * aggregate and the record's value. - * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's - * value as-is. - * Thus, {@code reduce(Reducer, Windows, String)} can be used to compute aggregate functions like sum, min, or max. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name - * and "-changelog" is a fixed suffix. - * Note that the internal store name may not be queriable through Interactive Queries. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param reducer a {@link Reducer} that computes a new aggregate result - * @param windows the specification of the aggregation {@link Windows} - * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent - * the latest (rolling) aggregate for each key within a window - * @deprecated use {@link #windowedBy(Windows) windowedBy(windows)} followed by - * {@link TimeWindowedKStream#reduce(Reducer) reduce(reducer)} - */ - @Deprecated - KTable, V> reduce(final Reducer reducer, - final Windows windows); - - /** - * Combine the values of records in this stream by the grouped key and the defined windows. - * Records with {@code null} key or value are ignored. - * Combining implies that the type of the aggregate result is the same as the type of the input value - * (c.f. {@link #aggregate(Initializer, Aggregator, Windows, Serde, String)}). - * The specified {@code windows} define either hopping time windows that can be overlapping or tumbling (c.f. - * {@link TimeWindows}) or they define landmark windows (c.f. {@link UnlimitedWindows}). - * The result is written into a local windowed {@link KeyValueStore} (which is basically an ever-updating - * materialized view) provided by the given {@code storeSupplier}. - * Windows are retained until their retention time expires (c.f. {@link Windows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current - * aggregate (first argument) and the record's value (second argument): - * 

{@code
-     * // At the example of a Reducer
-     * new Reducer() {
-     *   public Long apply(Long aggValue, Long currValue) {
-     *     return aggValue + currValue;
-     *   }
-     * }
-     * }
- *

- * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's - * value as-is. - * Thus, {@code reduce(Reducer, Windows, org.apache.kafka.streams.processor.StateStoreSupplier)} can be used to - * compute aggregate functions like sum, min, or max. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local windowed {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * Use {@link org.apache.kafka.streams.processor.StateStoreSupplier#name()} to get the store name: - * 

{@code
-     * KafkaStreams streams = ... // compute sum
-     * Sting queryableStoreName = storeSupplier.name();
-     * ReadOnlyWindowStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.windowStore());
-     * String key = "some-key";
-     * long fromTime = ...;
-     * long toTime = ...;
-     * WindowStoreIterator sumForKeyForWindows = localWindowStore.fetch(key, timeFrom, timeTo); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - * - * @param reducer a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}. - * @param windows the specification of the aggregation {@link Windows} - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent - * the latest (rolling) aggregate for each key within a window - * @deprecated use {@link #windowedBy(Windows) windowedBy(windows)} followed by - * {@link TimeWindowedKStream#reduce(Reducer, Materialized) reduce(reducer, Materialized.as(KeyValueByteStoreSupplier))} - */ - @Deprecated - KTable, V> reduce(final Reducer reducer, - final Windows windows, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - - /** - * Combine values of this stream by the grouped key into {@link SessionWindows}. - * Records with {@code null} key or value are ignored. - * Combining implies that the type of the aggregate result is the same as the type of the input value - * (c.f. {@link #aggregate(Initializer, Aggregator, Merger, SessionWindows, Serde, String)}). - * The result is written into a local {@link SessionStore} (which is basically an ever-updating - * materialized view) that can be queried using the provided {@code queryableStoreName}. - * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current - * aggregate (first argument) and the record's value (second argument): - * 

{@code
-     * // At the example of a Reducer
-     * new Reducer() {
-     *   public Long apply(Long aggValue, Long currValue) {
-     *     return aggValue + currValue;
-     *   }
-     * }
-     * }
- *

- * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's - * value as-is. - * Thus, {@code reduce(Reducer, SessionWindows, String)} can be used to compute aggregate functions like sum, min, - * or max. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local windowed {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * 

{@code
-     * KafkaStreams streams = ... // compute sum
-     * ReadOnlySessionStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.ReadOnlySessionStore);
-     * String key = "some-key";
-     * KeyValueIterator, Long> sumForKeyForWindows = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * Therefore, the store name must be a valid Kafka topic name and cannot contain characters other than ASCII - * alphanumerics, '.', '_' and '-'. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * @param reducer a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}. - * @param sessionWindows the specification of the aggregation {@link SessionWindows} - * @param queryableStoreName the name of the state store created from this operation; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then this will be equivalent to {@link KGroupedStream#reduce(Reducer, SessionWindows)}. - * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent - * the latest (rolling) aggregate for each key within a window - * @deprecated use {@link #windowedBy(SessionWindows) windowedBy(sessionWindows)} followed by - * {@link SessionWindowedKStream#reduce(Reducer, Materialized) reduce(reducer, Materialized.as(queryableStoreName))} - */ - @Deprecated - KTable, V> reduce(final Reducer reducer, - final SessionWindows sessionWindows, - final String queryableStoreName); - - /** - * Combine values of this stream by the grouped key into {@link SessionWindows}. - * Records with {@code null} key or value are ignored. - * Combining implies that the type of the aggregate result is the same as the type of the input value - * (c.f. {@link #aggregate(Initializer, Aggregator, Merger, SessionWindows, Serde, String)}). - * The result is written into a local {@link SessionStore} (which is basically an ever-updating - * materialized view) that can be queried using the provided {@code queryableStoreName}. - * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current - * aggregate and the record's value. - * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's - * value as-is. - * Thus, {@code reduce(Reducer, SessionWindows, String)} can be used to compute aggregate functions like sum, min, - * or max. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - * @param reducer a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}. - * @param sessionWindows the specification of the aggregation {@link SessionWindows} - * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent - * the latest (rolling) aggregate for each key within a window - * @deprecated use {@link #windowedBy(SessionWindows) windowedBy(sessionWindows)} followed by - * {@link SessionWindowedKStream#reduce(Reducer) reduce(reducer)} - */ - @Deprecated - KTable, V> reduce(final Reducer reducer, - final SessionWindows sessionWindows); - - /** - * Combine values of this stream by the grouped key into {@link SessionWindows}. - * Records with {@code null} key or value are ignored. - * Combining implies that the type of the aggregate result is the same as the type of the input value - * (c.f. {@link #aggregate(Initializer, Aggregator, Merger, SessionWindows, Serde, String)}). - * The result is written into a local {@link SessionStore} (which is basically an ever-updating materialized view) - * provided by the given {@code storeSupplier}. - * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current - * aggregate (first argument) and the record's value (second argument): - * 

{@code
-     * // At the example of a Reducer
-     * new Reducer() {
-     *   public Long apply(Long aggValue, Long currValue) {
-     *     return aggValue + currValue;
-     *   }
-     * }
-     * }
- *

- * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's - * value as-is. - * Thus, {@code reduce(Reducer, SessionWindows, org.apache.kafka.streams.processor.StateStoreSupplier)} can be used - * to compute aggregate functions like sum, min, or max. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local windowed {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * Use {@link org.apache.kafka.streams.processor.StateStoreSupplier#name()} to get the store name: - * 

{@code
-     * KafkaStreams streams = ... // compute sum
-     * Sting queryableStoreName = storeSupplier.name();
-     * ReadOnlySessionStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.ReadOnlySessionStore);
-     * String key = "some-key";
-     * KeyValueIterator, Long> sumForKeyForWindows = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * Therefore, the store name must be a valid Kafka topic name and cannot contain characters other than ASCII - * alphanumerics, '.', '_' and '-'. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * @param reducer a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}. - * @param sessionWindows the specification of the aggregation {@link SessionWindows} - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent - * the latest (rolling) aggregate for each key within a window - * @deprecated use {@link #windowedBy(SessionWindows) windowedBy(sessionWindows)} followed by - * {@link SessionWindowedKStream#reduce(Reducer, Materialized) reduce(reducer, Materialized.as(KeyValueByteStoreSupplier))} - */ - @Deprecated - KTable, V> reduce(final Reducer reducer, - final SessionWindows sessionWindows, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - - - /** - * Aggregate the values of records in this stream by the grouped key. - * Records with {@code null} key or value are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer, String) combining via reduce(...)} as it, for example, - * allows the result to have a different type than the input values. - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * that can be queried using the provided {@code queryableStoreName}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * The specified {@link Initializer} is applied once directly before the first input record is processed to - * provide an initial intermediate aggregation result that is used to process the first record. - * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current - * aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value. - * Thus, {@code aggregate(Initializer, Aggregator, Serde, String)} can be used to compute aggregate functions like - * count (c.f. {@link #count(String)}). - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // some aggregation on value type double
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-key";
-     * Long aggForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * Therefore, the store name must be a valid Kafka topic name and cannot contain characters other than ASCII - * alphanumerics, '.', '_' and '-'. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param initializer an {@link Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@link Aggregator} that computes a new aggregate result - * @param aggValueSerde aggregate value serdes for materializing the aggregated table, - * if not specified the default serdes defined in the configs will be used - * @param queryableStoreName the name of the state store created from this operation; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then this will be equivalent to {@link KGroupedStream#aggregate(Initializer, Aggregator, Serde)}. - * @param the value type of the resulting {@link KTable} - * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @deprecated use {@link #aggregate(Initializer, Aggregator, Materialized) aggregate(initializer, aggregator, Materialized.as(queryableStoreName).withValueSerde(aggValueSerde))} - */ - @Deprecated - KTable aggregate(final Initializer initializer, - final Aggregator aggregator, - final Serde aggValueSerde, - final String queryableStoreName); - - /** - * Aggregate the values of records in this stream by the grouped key. - * Records with {@code null} key or value are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer) combining via reduce(...)} as it, for example, - * allows the result to have a different type than the input values. - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * that can be queried using the provided {@code queryableStoreName}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * The specified {@link Initializer} is applied once directly before the first input record is processed to - * provide an initial intermediate aggregation result that is used to process the first record. - * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current - * aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value. - * Thus, {@code aggregate(Initializer, Aggregator, Serde, String)} can be used to compute aggregate functions like - * count (c.f. {@link #count()}). - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // some aggregation on value type double
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-key";
-     * Long aggForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * Therefore, the store name must be a valid Kafka topic name and cannot contain characters other than ASCII - * alphanumerics, '.', '_' and '-'. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param initializer an {@link Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@link Aggregator} that computes a new aggregate result - * @param materialized an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}. - * @param the value type of the resulting {@link KTable} - * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - */ - KTable aggregate(final Initializer initializer, - final Aggregator aggregator, - final Materialized> materialized); - - - /** - * Aggregate the values of records in this stream by the grouped key. + * Aggregate the values of records in this stream by the grouped key. * Records with {@code null} key or value are ignored. * Aggregating is a generalization of {@link #reduce(Reducer) combining via reduce(...)} as it, for example, * allows the result to have a different type than the input values. @@ -1134,451 +295,6 @@ KTable aggregate(final Initializer initializer, KTable aggregate(final Initializer initializer, final Aggregator aggregator); - /** - * Aggregate the values of records in this stream by the grouped key. - * Records with {@code null} key or value are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer) combining via reduce(...)} as it, for example, - * allows the result to have a different type than the input values. - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * that can be queried using the provided {@code queryableStoreName}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * The specified {@link Initializer} is applied once directly before the first input record is processed to - * provide an initial intermediate aggregation result that is used to process the first record. - * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current - * aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value. - * Thus, {@code aggregate(Initializer, Aggregator, Serde, String)} can be used to compute aggregate functions like - * count (c.f. {@link #count()}). - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name - * and "-changelog" is a fixed suffix. - * Note that the internal store name may not be queriable through Interactive Queries. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param initializer an {@link Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@link Aggregator} that computes a new aggregate result - * @param aggValueSerde aggregate value serdes for materializing the aggregated table, - * if not specified the default serdes defined in the configs will be used - * @param the value type of the resulting {@link KTable} - * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @deprecated use {@link #aggregate(Initializer, Aggregator, Materialized) aggregate(initializer, aggregator, Materialized.with(null, aggValueSerde))} - */ - @Deprecated - KTable aggregate(final Initializer initializer, - final Aggregator aggregator, - final Serde aggValueSerde); - - /** - * Aggregate the values of records in this stream by the grouped key. - * Records with {@code null} key or value are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer, org.apache.kafka.streams.processor.StateStoreSupplier) - * combining via reduce(...)} as it, for example, allows the result to have a different type than the input values. - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * provided by the given {@code storeSupplier}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * The specified {@link Initializer} is applied once directly before the first input record is processed to - * provide an initial intermediate aggregation result that is used to process the first record. - * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current - * aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value. - * Thus, {@code aggregate(Initializer, Aggregator, org.apache.kafka.streams.processor.StateStoreSupplier)} can be - * used to compute aggregate functions like count (c.f. {@link #count()}). - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * Use {@link org.apache.kafka.streams.processor.StateStoreSupplier#name()} to get the store name: - * 

{@code
-     * KafkaStreams streams = ... // some aggregation on value type double
-     * Sting queryableStoreName = storeSupplier.name();
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-key";
-     * Long aggForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - * - * @param initializer an {@link Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@link Aggregator} that computes a new aggregate result - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @param the value type of the resulting {@link KTable} - * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @deprecated use {@link #aggregate(Initializer, Aggregator, Materialized) aggregate(initializer, aggregator, Materialized.as(KeyValueByteStoreSupplier))} - */ - @Deprecated - KTable aggregate(final Initializer initializer, - final Aggregator aggregator, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - - /** - * Aggregate the values of records in this stream by the grouped key and defined windows. - * Records with {@code null} key or value are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer, Windows, String) combining via reduce(...)} as it, - * for example, allows the result to have a different type than the input values. - * The specified {@code windows} define either hopping time windows that can be overlapping or tumbling (c.f. - * {@link TimeWindows}) or they define landmark windows (c.f. {@link UnlimitedWindows}). - * The result is written into a local windowed {@link KeyValueStore} (which is basically an ever-updating - * materialized view) that can be queried using the provided {@code queryableStoreName}. - * Windows are retained until their retention time expires (c.f. {@link Windows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * The specified {@link Initializer} is applied once per window directly before the first input record is - * processed to provide an initial intermediate aggregation result that is used to process the first record. - * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current - * aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value. - * Thus, {@code aggregate(Initializer, Aggregator, Windows, Serde, String)} can be used to compute aggregate - * functions like count (c.f. {@link #count(Windows)}). - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local windowed {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // some windowed aggregation on value type double
-     * ReadOnlyWindowStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.windowStore());
-     * String key = "some-key";
-     * long fromTime = ...;
-     * long toTime = ...;
-     * WindowStoreIterator aggForKeyForWindows = localWindowStore.fetch(key, timeFrom, timeTo); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * Therefore, the store name must be a valid Kafka topic name and cannot contain characters other than ASCII - * alphanumerics, '.', '_' and '-'. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * - * @param initializer an {@link Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@link Aggregator} that computes a new aggregate result - * @param windows the specification of the aggregation {@link Windows} - * @param aggValueSerde aggregate value serdes for materializing the aggregated table, - * if not specified the default serdes defined in the configs will be used - * @param the value type of the resulting {@link KTable} - * @param queryableStoreName the name of the state store created from this operation; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then this will be equivalent to {@link KGroupedStream#aggregate(Initializer, Aggregator, Windows, Serde)}. - * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent - * the latest (rolling) aggregate for each key within a window - * @deprecated use {@link #windowedBy(Windows) windowedBy(windows)} followed by - * {@link TimeWindowedKStream#aggregate(Initializer, Aggregator, Materialized) aggregate(initializer, aggregator, Materialized.as(queryableStoreName).withValueSerde(aggValueSerde))} - */ - @Deprecated - KTable, VR> aggregate(final Initializer initializer, - final Aggregator aggregator, - final Windows windows, - final Serde aggValueSerde, - final String queryableStoreName); - - /** - * Aggregate the values of records in this stream by the grouped key and defined windows. - * Records with {@code null} key or value are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer, Windows, String) combining via reduce(...)} as it, - * for example, allows the result to have a different type than the input values. - * The specified {@code windows} define either hopping time windows that can be overlapping or tumbling (c.f. - * {@link TimeWindows}) or they define landmark windows (c.f. {@link UnlimitedWindows}). - * The result is written into a local windowed {@link KeyValueStore} (which is basically an ever-updating - * materialized view) that can be queried using the provided {@code queryableStoreName}. - * Windows are retained until their retention time expires (c.f. {@link Windows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * The specified {@link Initializer} is applied once per window directly before the first input record is - * processed to provide an initial intermediate aggregation result that is used to process the first record. - * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current - * aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value. - * Thus, {@code aggregate(Initializer, Aggregator, Windows, Serde, String)} can be used to compute aggregate - * functions like count (c.f. {@link #count(Windows)}). - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name - * and "-changelog" is a fixed suffix. - * Note that the internal store name may not be queriable through Interactive Queries. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param initializer an {@link Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@link Aggregator} that computes a new aggregate result - * @param windows the specification of the aggregation {@link Windows} - * @param aggValueSerde aggregate value serdes for materializing the aggregated table, - * if not specified the default serdes defined in the configs will be used - * @param the value type of the resulting {@link KTable} - * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent - * the latest (rolling) aggregate for each key within a window - * @deprecated use {@link #windowedBy(Windows) windowedBy(windows)} followed by - * {@link TimeWindowedKStream#aggregate(Initializer, Aggregator, Materialized)} aggregate(initializer, aggregator, Materialized.with(null, aggValueSerde))} - */ - @Deprecated - KTable, VR> aggregate(final Initializer initializer, - final Aggregator aggregator, - final Windows windows, - final Serde aggValueSerde); - - /** - * Aggregate the values of records in this stream by the grouped key and defined windows. - * Records with {@code null} key or value are ignored. - * Aggregating is a generalization of - * {@link #reduce(Reducer, Windows, org.apache.kafka.streams.processor.StateStoreSupplier) combining via reduce(...)} - * as it, for example, allows the result to have a different type than the input values. - * The specified {@code windows} define either hopping time windows that can be overlapping or tumbling (c.f. - * {@link TimeWindows}) or they define landmark windows (c.f. {@link UnlimitedWindows}). - * The result is written into a local windowed {@link KeyValueStore} (which is basically an ever-updating - * materialized view) provided by the given {@code storeSupplier}. - * Windows are retained until their retention time expires (c.f. {@link Windows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * The specified {@link Initializer} is applied once per window directly before the first input record is - * processed to provide an initial intermediate aggregation result that is used to process the first record. - * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current - * aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value. - * Thus, {@code aggregate(Initializer, Aggregator, Windows, org.apache.kafka.streams.processor.StateStoreSupplier)} - * can be used to compute aggregate functions like count (c.f. {@link #count(Windows)}). - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local windowed {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * Use {@link org.apache.kafka.streams.processor.StateStoreSupplier#name()} to get the store name: - * 

{@code
-     * KafkaStreams streams = ... // some windowed aggregation on value type Long
-     * Sting queryableStoreName = storeSupplier.name();
-     * ReadOnlyWindowStore localWindowStore = streams.store(queryableStoreName, QueryableStoreTypes.windowStore());
-     * String key = "some-key";
-     * long fromTime = ...;
-     * long toTime = ...;
-     * WindowStoreIterator aggForKeyForWindows = localWindowStore.fetch(key, timeFrom, timeTo); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - * - * @param initializer an {@link Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@link Aggregator} that computes a new aggregate result - * @param windows the specification of the aggregation {@link Windows} - * @param the value type of the resulting {@link KTable} - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent - * the latest (rolling) aggregate for each key within a window - * @deprecated use {@link #windowedBy(Windows) windowedBy(windows)} followed by - * {@link TimeWindowedKStream#aggregate(Initializer, Aggregator, Materialized) aggregate(initializer, aggregator, Materialized.as(KeyValueByteStoreSupplier))} - */ - @Deprecated - KTable, VR> aggregate(final Initializer initializer, - final Aggregator aggregator, - final Windows windows, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - - /** - * Aggregate the values of records in this stream by the grouped key and defined {@link SessionWindows}. - * Records with {@code null} key or value are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer, SessionWindows, String) combining via - * reduce(...)} as it, for example, allows the result to have a different type than the input values. - * The result is written into a local {@link SessionStore} (which is basically an ever-updating - * materialized view) that can be queried using the provided {@code queryableStoreName}. - * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * The specified {@link Initializer} is applied once per session directly before the first input record is - * processed to provide an initial intermediate aggregation result that is used to process the first record. - * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current - * aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value. - * Thus, {@code aggregate(Initializer, Aggregator, Merger, SessionWindows, Serde, String)} can be used to compute - * aggregate functions like count (c.f. {@link #count(SessionWindows)}) - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link SessionStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * 

{@code
-     * KafkaStreams streams = ... // some windowed aggregation on value type double
-     * ReadOnlySessionStore sessionStore = streams.store(queryableStoreName, QueryableStoreTypes.sessionStore());
-     * String key = "some-key";
-     * KeyValueIterator, Long> aggForKeyForSession = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - * - * @param initializer the instance of {@link Initializer} - * @param aggregator the instance of {@link Aggregator} - * @param sessionMerger the instance of {@link Merger} - * @param sessionWindows the specification of the aggregation {@link SessionWindows} - * @param aggValueSerde aggregate value serdes for materializing the aggregated table, - * if not specified the default serdes defined in the configs will be used - * @param the value type of the resulting {@link KTable} - * @param queryableStoreName the name of the state store created from this operation; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} then this will be equivalent to {@link KGroupedStream#aggregate(Initializer, Aggregator, Merger, SessionWindows, Serde)}. - * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent - * the latest (rolling) aggregate for each key within a window - * @deprecated use {@link #windowedBy(SessionWindows) windowedBy(sessionWindows)} followed by - * {@link SessionWindowedKStream#aggregate(Initializer, Aggregator, Merger, Materialized) aggregate(initializer, aggregator, sessionMerger, Materialized.as(queryableStoreName).withValueSerde(aggValueSerde))} - */ - @Deprecated - KTable, T> aggregate(final Initializer initializer, - final Aggregator aggregator, - final Merger sessionMerger, - final SessionWindows sessionWindows, - final Serde aggValueSerde, - final String queryableStoreName); - - /** - * Aggregate the values of records in this stream by the grouped key and defined {@link SessionWindows}. - * Records with {@code null} key or value are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer, SessionWindows, String) combining via - * reduce(...)} as it, for example, allows the result to have a different type than the input values. - * The result is written into a local {@link SessionStore} (which is basically an ever-updating - * materialized view) that can be queried using the provided {@code queryableStoreName}. - * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * The specified {@link Initializer} is applied once per session directly before the first input record is - * processed to provide an initial intermediate aggregation result that is used to process the first record. - * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current - * aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value. - * Thus, {@code aggregate(Initializer, Aggregator, Merger, SessionWindows, Serde, String)} can be used to compute - * aggregate functions like count (c.f. {@link #count(SessionWindows)}) - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * @param initializer the instance of {@link Initializer} - * @param aggregator the instance of {@link Aggregator} - * @param sessionMerger the instance of {@link Merger} - * @param sessionWindows the specification of the aggregation {@link SessionWindows} - * @param aggValueSerde aggregate value serdes for materializing the aggregated table, - * if not specified the default serdes defined in the configs will be used - * @param the value type of the resulting {@link KTable} - * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent - * the latest (rolling) aggregate for each key within a window - * @deprecated use {@link #windowedBy(SessionWindows) windowedBy(sessionWindows)} followed by - * {@link SessionWindowedKStream#aggregate(Initializer, Aggregator, Merger, Materialized) aggregate(initializer, aggregator, sessionMerger, Materialized.with(null, aggValueSerde))} - */ - @Deprecated - KTable, T> aggregate(final Initializer initializer, - final Aggregator aggregator, - final Merger sessionMerger, - final SessionWindows sessionWindows, - final Serde aggValueSerde); - - /** - * Aggregate the values of records in this stream by the grouped key and defined {@link SessionWindows}. - * Records with {@code null} key or value are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer, SessionWindows, String) combining via - * reduce(...)} as it, for example, allows the result to have a different type than the input values. - * The result is written into a local {@link SessionStore} (which is basically an ever-updating materialized view) - * provided by the given {@code storeSupplier}. - * SessionWindows are retained until their retention time expires (c.f. {@link SessionWindows#until(long)}). - * Furthermore, updates to the store are sent downstream into a windowed {@link KTable} changelog stream, where - * "windowed" implies that the {@link KTable} key is a combined key of the original record key and a window ID. - *

- * The specified {@link Initializer} is applied once per session directly before the first input record is - * processed to provide an initial intermediate aggregation result that is used to process the first record. - * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current - * aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value. - * Thus, {@code #aggregate(Initializer, Aggregator, Merger, SessionWindows, Serde, org.apache.kafka.streams.processor.StateStoreSupplier)} - * can be used to compute aggregate functions like count (c.f. {@link #count(SessionWindows)}). - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same window and key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link SessionStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. - * Use {@link org.apache.kafka.streams.processor.StateStoreSupplier#name()} to get the store name: - * 

{@code
-     * KafkaStreams streams = ... // some windowed aggregation on value type double
-     * Sting queryableStoreName = storeSupplier.name();
-     * ReadOnlySessionStore sessionStore = streams.store(queryableStoreName, QueryableStoreTypes.sessionStore());
-     * String key = "some-key";
-     * KeyValueIterator, Long> aggForKeyForSession = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - * - * - * @param initializer the instance of {@link Initializer} - * @param aggregator the instance of {@link Aggregator} - * @param sessionMerger the instance of {@link Merger} - * @param sessionWindows the specification of the aggregation {@link SessionWindows} - * @param aggValueSerde aggregate value serdes for materializing the aggregated table, - * if not specified the default serdes defined in the configs will be used - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @param the value type of the resulting {@link KTable} - * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent - * the latest (rolling) aggregate for each key within a window - * @deprecated use {@link #windowedBy(SessionWindows) windowedBy(sessionWindows)} followed by - * {@link SessionWindowedKStream#aggregate(Initializer, Aggregator, Merger, Materialized) aggregate(initializer, aggregator, sessionMerger, Materialized.as(KeyValueByteStoreSupplier).withValueSerde(aggValueSerde))} - */ - @Deprecated - KTable, T> aggregate(final Initializer initializer, - final Aggregator aggregator, - final Merger sessionMerger, - final SessionWindows sessionWindows, - final Serde aggValueSerde, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - /** * Create a new {@link TimeWindowedKStream} instance that can be used to perform windowed aggregations. * @param windows the specification of the aggregation {@link Windows} diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java index 1916f16e2c907..210336f26b965 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java @@ -41,50 +41,6 @@ @InterfaceStability.Evolving public interface KGroupedTable { - /** - * Count number of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) mapped} to - * the same key into a new instance of {@link KTable}. - * Records with {@code null} key are ignored. - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * that can be queried using the provided {@code queryableStoreName}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // counting words
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-word";
-     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * The store name must be a valid Kafka topic name and cannot contain characters other than ASCII alphanumerics, - * '.', '_' and '-'. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param queryableStoreName the name of the underlying {@link KTable} state store; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} this is the equivalent of {@link KGroupedTable#count()}. - * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that - * represent the latest (rolling) count (i.e., number of records) for each key - * @deprecated use {@link #count(Materialized) count(Materialized.as(queryableStoreName))} - */ - @Deprecated - KTable count(final String queryableStoreName); - /** * Count number of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) mapped} to * the same key into a new instance of {@link KTable}. @@ -154,122 +110,6 @@ public interface KGroupedTable { */ KTable count(); - /** - * Count number of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) mapped} to - * the same key into a new instance of {@link KTable}. - * Records with {@code null} key are ignored. - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * provided by the given {@code storeSupplier}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // counting words
-     * String queryableStoreName = storeSupplier.name();
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-word";
-     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that - * represent the latest (rolling) count (i.e., number of records) for each key - * @deprecated use {@link #count(Materialized) count(Materialized.as(KeyValueByteStoreSupplier)} - */ - @Deprecated - KTable count(final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - - /** - * Combine the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) - * mapped} to the same key into a new instance of {@link KTable}. - * Records with {@code null} key are ignored. - * Combining implies that the type of the aggregate result is the same as the type of the input value - * (c.f. {@link #aggregate(Initializer, Aggregator, Aggregator, Serde, String)}). - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * that can be queried using the provided {@code queryableStoreName}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * Each update to the original {@link KTable} results in a two step update of the result {@link KTable}. - * The specified {@link Reducer adder} is applied for each update record and computes a new aggregate using the - * current aggregate (first argument) and the record's value (second argument) by adding the new record to the - * aggregate. - * The specified {@link Reducer subtractor} is applied for each "replaced" record of the original {@link KTable} - * and computes a new aggregate using the current aggregate (first argument) and the record's value (second - * argument) by "removing" the "replaced" record from the aggregate. - * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's - * value as-is. - * Thus, {@code reduce(Reducer, Reducer, String)} can be used to compute aggregate functions like sum. - * For sum, the adder and subtractor would work as follows: - * 

{@code
-     * public class SumAdder implements Reducer {
-     *   public Integer apply(Integer currentAgg, Integer newValue) {
-     *     return currentAgg + newValue;
-     *   }
-     * }
-     *
-     * public class SumSubtractor implements Reducer {
-     *   public Integer apply(Integer currentAgg, Integer oldValue) {
-     *     return currentAgg - oldValue;
-     *   }
-     * }
-     * }
- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // counting words
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-word";
-     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * The store name must be a valid Kafka topic name and cannot contain characters other than ASCII alphanumerics, - * '.', '_' and '-'. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param adder a {@link Reducer} that adds a new value to the aggregate result - * @param subtractor a {@link Reducer} that removed an old value from the aggregate result - * @param queryableStoreName the name of the underlying {@link KTable} state store; valid characters are ASCII alphanumerics, - * '.', '_' and '-'. If {@code null} this is the equivalent of {@link KGroupedTable#reduce(Reducer, Reducer)} ()}. - * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @deprecated use {@link #reduce(Reducer, Reducer, Materialized) reduce(adder, subtractor, Materialized.as(queryableStoreName))} - */ - @Deprecated - KTable reduce(final Reducer adder, - final Reducer subtractor, - final String queryableStoreName); - /** * Combine the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) * mapped} to the same key into a new instance of {@link KTable}. @@ -396,164 +236,6 @@ KTable reduce(final Reducer adder, KTable reduce(final Reducer adder, final Reducer subtractor); - /** - * Combine the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) - * mapped} to the same key into a new instance of {@link KTable}. - * Records with {@code null} key are ignored. - * Combining implies that the type of the aggregate result is the same as the type of the input value - * (c.f. {@link #aggregate(Initializer, Aggregator, Aggregator, Serde, String)}). - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * provided by the given {@code storeSupplier}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * Each update to the original {@link KTable} results in a two step update of the result {@link KTable}. - * The specified {@link Reducer adder} is applied for each update record and computes a new aggregate using the - * current aggregate (first argument) and the record's value (second argument) by adding the new record to the - * aggregate. - * The specified {@link Reducer subtractor} is applied for each "replaced" record of the original {@link KTable} - * and computes a new aggregate using the current aggregate (first argument) and the record's value (second - * argument) by "removing" the "replaced" record from the aggregate. - * If there is no current aggregate the {@link Reducer} is not applied and the new aggregate will be the record's - * value as-is. - * Thus, {@code reduce(Reducer, Reducer, String)} can be used to compute aggregate functions like sum. - * For sum, the adder and subtractor would work as follows: - * 

{@code
-     * public class SumAdder implements Reducer {
-     *   public Integer apply(Integer currentAgg, Integer newValue) {
-     *     return currentAgg + newValue;
-     *   }
-     * }
-     *
-     * public class SumSubtractor implements Reducer {
-     *   public Integer apply(Integer currentAgg, Integer oldValue) {
-     *     return currentAgg - oldValue;
-     *   }
-     * }
-     * }
- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // counting words
-     * String queryableStoreName = storeSupplier.name();
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-word";
-     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param adder a {@link Reducer} that adds a new value to the aggregate result - * @param subtractor a {@link Reducer} that removed an old value from the aggregate result - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @deprecated use {@link #reduce(Reducer, Reducer, Materialized) reduce(adder, subtractor, Materialized.as(KeyValueByteStoreSupplier))} - */ - @Deprecated - KTable reduce(final Reducer adder, - final Reducer subtractor, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - - /** - * Aggregate the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) - * mapped} to the same key into a new instance of {@link KTable} using default serializers and deserializers. - * Records with {@code null} key are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer, Reducer, String) combining via reduce(...)} as it, - * for example, allows the result to have a different type than the input values. - * If the result value type does not match the {@link StreamsConfig#DEFAULT_VALUE_SERDE_CLASS_CONFIG default value - * serde} you should use {@link KGroupedTable#aggregate(Initializer, Aggregator, Aggregator, Serde, String) - * aggregate(Initializer, Aggregator, Aggregator, Serde, String)}. - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * provided by the given {@code storeSupplier}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * The specified {@link Initializer} is applied once directly before the first input record is processed to - * provide an initial intermediate aggregation result that is used to process the first record. - * Each update to the original {@link KTable} results in a two step update of the result {@link KTable}. - * The specified {@link Aggregator adder} is applied for each update record and computes a new aggregate using the - * current aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value by adding the new record to the aggregate. - * The specified {@link Aggregator subtractor} is applied for each "replaced" record of the original {@link KTable} - * and computes a new aggregate using the current aggregate and the record's value by "removing" the "replaced" - * record from the aggregate. - * Thus, {@code aggregate(Initializer, Aggregator, Aggregator, String)} can be used to compute aggregate functions - * like sum. - * For sum, the initializer, adder, and subtractor would work as follows: - * 

{@code
-     * // in this example, LongSerde.class must be set as default value serde in StreamsConfig
-     * public class SumInitializer implements Initializer {
-     *   public Long apply() {
-     *     return 0L;
-     *   }
-     * }
-     *
-     * public class SumAdder implements Aggregator {
-     *   public Long apply(String key, Integer newValue, Long aggregate) {
-     *     return aggregate + newValue;
-     *   }
-     * }
-     *
-     * public class SumSubstractor implements Aggregator {
-     *   public Long apply(String key, Integer oldValue, Long aggregate) {
-     *     return aggregate - oldValue;
-     *   }
-     * }
-     * }
- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // counting words
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-word";
-     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param initializer a {@link Initializer} that provides an initial aggregate result value - * @param adder a {@link Aggregator} that adds a new record to the aggregate result - * @param subtractor a {@link Aggregator} that removed an old record from the aggregate result - * @param queryableStoreName the name of the underlying {@link KTable} state store. - * If {@code null} this is the equivalent of {@link KGroupedTable#aggregate(Initializer, Aggregator, Aggregator)} ()}. - * @param the value type of the aggregated {@link KTable} - * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @deprecated use {@link #aggregate(Initializer, Aggregator, Aggregator, Materialized) aggregate(initializer, adder, subtractor, Materialized.as(queryableStoreName))} - */ - @Deprecated - KTable aggregate(final Initializer initializer, - final Aggregator adder, - final Aggregator subtractor, - final String queryableStoreName); - /** * Aggregate the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) * mapped} to the same key into a new instance of {@link KTable} using default serializers and deserializers. @@ -704,246 +386,4 @@ KTable aggregate(final Initializer initializer, final Aggregator subtractor); - /** - * Aggregate the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) - * mapped} to the same key into a new instance of {@link KTable} using default serializers and deserializers. - * Records with {@code null} key are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer, Reducer, String) combining via reduce(...)} as it, - * for example, allows the result to have a different type than the input values. - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * that can be queried using the provided {@code queryableStoreName}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * The specified {@link Initializer} is applied once directly before the first input record is processed to - * provide an initial intermediate aggregation result that is used to process the first record. - * Each update to the original {@link KTable} results in a two step update of the result {@link KTable}. - * The specified {@link Aggregator adder} is applied for each update record and computes a new aggregate using the - * current aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value by adding the new record to the aggregate. - * The specified {@link Aggregator subtractor} is applied for each "replaced" record of the original {@link KTable} - * and computes a new aggregate using the current aggregate and the record's value by "removing" the "replaced" - * record from the aggregate. - * Thus, {@code aggregate(Initializer, Aggregator, Aggregator, String)} can be used to compute aggregate functions - * like sum. - * For sum, the initializer, adder, and subtractor would work as follows: - * 

{@code
-     * public class SumInitializer implements Initializer {
-     *   public Long apply() {
-     *     return 0L;
-     *   }
-     * }
-     *
-     * public class SumAdder implements Aggregator {
-     *   public Long apply(String key, Integer newValue, Long aggregate) {
-     *     return aggregate + newValue;
-     *   }
-     * }
-     *
-     * public class SumSubstractor implements Aggregator {
-     *   public Long apply(String key, Integer oldValue, Long aggregate) {
-     *     return aggregate - oldValue;
-     *   }
-     * }
-     * }
- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // counting words
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-word";
-     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * The store name must be a valid Kafka topic name and cannot contain characters other than ASCII alphanumerics, - * '.', '_' and '-'. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param initializer a {@link Initializer} that provides an initial aggregate result value - * @param adder a {@link Aggregator} that adds a new record to the aggregate result - * @param subtractor a {@link Aggregator} that removed an old record from the aggregate result - * @param aggValueSerde aggregate value serdes for materializing the aggregated table, - * if not specified the default serdes defined in the configs will be used - * @param queryableStoreName the name of the underlying {@link KTable} state store; valid characters are ASCII - * alphanumerics, '.', '_' and '-'. If {@code null} this is the equivalent of {@link KGroupedTable#aggregate(Initializer, Aggregator, Aggregator, Serde)} ()}. - * @param the value type of the aggregated {@link KTable} - * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @deprecated use {@link #aggregate(Initializer, Aggregator, Aggregator, Materialized) aggregate(initializer, adder, subtractor, Materialized.as(queryableStoreName).withValueSerde(aggValueSerde))} - */ - @Deprecated - KTable aggregate(final Initializer initializer, - final Aggregator adder, - final Aggregator subtractor, - final Serde aggValueSerde, - final String queryableStoreName); - - /** - * Aggregate the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) - * mapped} to the same key into a new instance of {@link KTable} using default serializers and deserializers. - * Records with {@code null} key are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer, Reducer, Materialized) combining via reduce(...)} as it, - * for example, allows the result to have a different type than the input values. - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * that can be queried using the provided {@code queryableStoreName}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * The specified {@link Initializer} is applied once directly before the first input record is processed to - * provide an initial intermediate aggregation result that is used to process the first record. - * Each update to the original {@link KTable} results in a two step update of the result {@link KTable}. - * The specified {@link Aggregator adder} is applied for each update record and computes a new aggregate using the - * current aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value by adding the new record to the aggregate. - * The specified {@link Aggregator subtractor} is applied for each "replaced" record of the original {@link KTable} - * and computes a new aggregate using the current aggregate and the record's value by "removing" the "replaced" - * record from the aggregate. - * Thus, {@code aggregate(Initializer, Aggregator, Aggregator, String)} can be used to compute aggregate functions - * like sum. - * For sum, the initializer, adder, and subtractor would work as follows: - * 

{@code
-     * public class SumInitializer implements Initializer {
-     *   public Long apply() {
-     *     return 0L;
-     *   }
-     * }
-     *
-     * public class SumAdder implements Aggregator {
-     *   public Long apply(String key, Integer newValue, Long aggregate) {
-     *     return aggregate + newValue;
-     *   }
-     * }
-     *
-     * public class SumSubstractor implements Aggregator {
-     *   public Long apply(String key, Integer oldValue, Long aggregate) {
-     *     return aggregate - oldValue;
-     *   }
-     * }
-     * }
- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name - * and "-changelog" is a fixed suffix. - * Note that the internal store name may not be queriable through Interactive Queries. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param initializer a {@link Initializer} that provides an initial aggregate result value - * @param adder a {@link Aggregator} that adds a new record to the aggregate result - * @param subtractor a {@link Aggregator} that removed an old record from the aggregate result - * @param aggValueSerde aggregate value serdes for materializing the aggregated table, - * if not specified the default serdes defined in the configs will be used - * @param the value type of the aggregated {@link KTable} - * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @deprecated use {@link #aggregate(Initializer, Aggregator, Aggregator, Materialized) aggregate(initializer, adder, subtractor, Materialized.with(null, aggValueSerde))} - */ - @Deprecated - KTable aggregate(final Initializer initializer, - final Aggregator adder, - final Aggregator subtractor, - final Serde aggValueSerde); - - - /** - * Aggregate the value of records of the original {@link KTable} that got {@link KTable#groupBy(KeyValueMapper) - * mapped} to the same key into a new instance of {@link KTable} using default serializers and deserializers. - * Records with {@code null} key are ignored. - * Aggregating is a generalization of {@link #reduce(Reducer, Reducer, String) combining via reduce(...)} as it, - * for example, allows the result to have a different type than the input values. - * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * provided by the given {@code storeSupplier}. - * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. - *

- * The specified {@link Initializer} is applied once directly before the first input record is processed to - * provide an initial intermediate aggregation result that is used to process the first record. - * Each update to the original {@link KTable} results in a two step update of the result {@link KTable}. - * The specified {@link Aggregator adder} is applied for each update record and computes a new aggregate using the - * current aggregate (or for the very first record using the intermediate aggregation result provided via the - * {@link Initializer}) and the record's value by adding the new record to the aggregate. - * The specified {@link Aggregator subtractor} is applied for each "replaced" record of the original {@link KTable} - * and computes a new aggregate using the current aggregate and the record's value by "removing" the "replaced" - * record from the aggregate. - * Thus, {@code aggregate(Initializer, Aggregator, Aggregator, String)} can be used to compute aggregate functions - * like sum. - * For sum, the initializer, adder, and subtractor would work as follows: - * 

{@code
-     * public class SumInitializer implements Initializer {
-     *   public Long apply() {
-     *     return 0L;
-     *   }
-     * }
-     *
-     * public class SumAdder implements Aggregator {
-     *   public Long apply(String key, Integer newValue, Long aggregate) {
-     *     return aggregate + newValue;
-     *   }
-     * }
-     *
-     * public class SumSubstractor implements Aggregator {
-     *   public Long apply(String key, Integer oldValue, Long aggregate) {
-     *     return aggregate - oldValue;
-     *   }
-     * }
-     * }
- * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to - * the same key. - * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of - * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for - * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and - * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}. - *

- * To query the local {@link KeyValueStore} it must be obtained via - * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}: - * 

{@code
-     * KafkaStreams streams = ... // counting words
-     * String queryableStoreName = storeSupplier.name();
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-word";
-     * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }
- * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to - * query the value of the key on a parallel running instance of your Kafka Streams application. - *

- * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is - * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. - * - * @param initializer a {@link Initializer} that provides an initial aggregate result value - * @param adder a {@link Aggregator} that adds a new record to the aggregate result - * @param subtractor a {@link Aggregator} that removed an old record from the aggregate result - * @param storeSupplier user defined state store supplier. Cannot be {@code null}. - * @param the value type of the aggregated {@link KTable} - * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @deprecated use {@link #aggregate(Initializer, Aggregator, Aggregator, Materialized) aggregate(initializer, adder, subtractor, Materialized.as(KeyValueByteStoreSupplier))} - */ - @Deprecated - KTable aggregate(final Initializer initializer, - final Aggregator adder, - final Aggregator subtractor, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier); - } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java index 82e2823c3ed02..e9aeae54fa1a7 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java @@ -78,7 +78,6 @@ private void determineIsQueryable(final String queryableStoreName) { } @SuppressWarnings("deprecation") - @Override public KTable reduce(final Reducer reducer, final String queryableStoreName) { determineIsQueryable(queryableStoreName); @@ -92,7 +91,6 @@ public KTable reduce(final Reducer reducer) { } @SuppressWarnings("deprecation") - @Override public KTable reduce(final Reducer reducer, final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { Objects.requireNonNull(reducer, "reducer can't be null"); @@ -116,24 +114,7 @@ public KTable reduce(final Reducer reducer, materializedInternal); } - @SuppressWarnings("deprecation") - @Override - public KTable, V> reduce(final Reducer reducer, - final Windows windows, - final String queryableStoreName) { - determineIsQueryable(queryableStoreName); - return reduce(reducer, windows, windowedStore(keySerde, valSerde, windows, getOrCreateName(queryableStoreName, REDUCE_NAME))); - } - - @SuppressWarnings("deprecation") - @Override - public KTable, V> reduce(final Reducer reducer, - final Windows windows) { - return windowedBy(windows).reduce(reducer); - } - @SuppressWarnings({"unchecked", "deprecation"}) - @Override public KTable, V> reduce(final Reducer reducer, final Windows windows, final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { @@ -148,7 +129,6 @@ public KTable, V> reduce(final Reducer reducer } @SuppressWarnings("deprecation") - @Override public KTable aggregate(final Initializer initializer, final Aggregator aggregator, final Serde aggValueSerde, @@ -194,15 +174,6 @@ public KTable aggregate(final Initializer initializer, } @SuppressWarnings("deprecation") - @Override - public KTable aggregate(final Initializer initializer, - final Aggregator aggregator, - final Serde aggValueSerde) { - return aggregate(initializer, aggregator, aggValueSerde, null); - } - - @SuppressWarnings("deprecation") - @Override public KTable aggregate(final Initializer initializer, final Aggregator aggregator, final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { @@ -215,31 +186,7 @@ public KTable aggregate(final Initializer initializer, storeSupplier); } - @SuppressWarnings("deprecation") - @Override - public KTable, T> aggregate(final Initializer initializer, - final Aggregator aggregator, - final Windows windows, - final Serde aggValueSerde, - final String queryableStoreName) { - determineIsQueryable(queryableStoreName); - return aggregate(initializer, aggregator, windows, windowedStore(keySerde, aggValueSerde, windows, getOrCreateName(queryableStoreName, AGGREGATE_NAME))); - } - - @SuppressWarnings("deprecation") - @Override - public KTable, T> aggregate(final Initializer initializer, - final Aggregator aggregator, - final Windows windows, - final Serde aggValueSerde) { - return windowedBy(windows).aggregate(initializer, aggregator, - Materialized.>as(builder.newStoreName(AGGREGATE_NAME)) - .withKeySerde(keySerde) - .withValueSerde(aggValueSerde)); - } - @SuppressWarnings({"unchecked", "deprecation"}) - @Override public KTable, T> aggregate(final Initializer initializer, final Aggregator aggregator, final Windows windows, @@ -256,7 +203,6 @@ public KTable, T> aggregate(final Initializer< } @SuppressWarnings("deprecation") - @Override public KTable count(final String queryableStoreName) { determineIsQueryable(queryableStoreName); return count(keyValueStore(keySerde, Serdes.Long(), getOrCreateName(queryableStoreName, AGGREGATE_NAME))); @@ -268,7 +214,6 @@ public KTable count() { } @SuppressWarnings("deprecation") - @Override public KTable count(final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { return aggregate(aggregateBuilder.countInitializer, aggregateBuilder.countAggregator, storeSupplier); } @@ -285,21 +230,6 @@ public KTable count(final Materialized KTable, Long> count(final Windows windows, - final String queryableStoreName) { - determineIsQueryable(queryableStoreName); - return count(windows, windowedStore(keySerde, Serdes.Long(), windows, getOrCreateName(queryableStoreName, AGGREGATE_NAME))); - } - - @SuppressWarnings("deprecation") - @Override - public KTable, Long> count(final Windows windows) { - return windowedBy(windows).count(); - } - - @SuppressWarnings("deprecation") - @Override public KTable, Long> count(final Windows windows, final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { return aggregate( @@ -310,42 +240,6 @@ public KTable, Long> count(final Windows windo } @SuppressWarnings({"unchecked", "deprecation"}) - @Override - public KTable, T> aggregate(final Initializer initializer, - final Aggregator aggregator, - final Merger sessionMerger, - final SessionWindows sessionWindows, - final Serde aggValueSerde, - final String queryableStoreName) { - determineIsQueryable(queryableStoreName); - return aggregate(initializer, - aggregator, - sessionMerger, - sessionWindows, - aggValueSerde, - storeFactory(keySerde, aggValueSerde, getOrCreateName(queryableStoreName, AGGREGATE_NAME)) - .sessionWindowed(sessionWindows.maintainMs()).build()); - - - } - - @SuppressWarnings("deprecation") - @Override - public KTable, T> aggregate(final Initializer initializer, - final Aggregator aggregator, - final Merger sessionMerger, - final SessionWindows sessionWindows, - final Serde aggValueSerde) { - return windowedBy(sessionWindows).aggregate(initializer, - aggregator, - sessionMerger, - Materialized.>as(builder.newStoreName(AGGREGATE_NAME)) - .withKeySerde(keySerde) - .withValueSerde(aggValueSerde)); - } - - @SuppressWarnings({"unchecked", "deprecation"}) - @Override public KTable, T> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, @@ -387,63 +281,8 @@ public SessionWindowedKStream windowedBy(final SessionWindows windows) { aggregateBuilder); } - @SuppressWarnings({"unchecked", "deprecation"}) - public KTable, Long> count(final SessionWindows sessionWindows, final String queryableStoreName) { - Materialized> materialized = Materialized.>as(getOrCreateName(queryableStoreName, AGGREGATE_NAME)) - .withKeySerde(keySerde) - .withValueSerde(Serdes.Long()); - return windowedBy(sessionWindows).count(materialized); - } - - @SuppressWarnings("deprecation") - public KTable, Long> count(final SessionWindows sessionWindows) { - return windowedBy(sessionWindows).count(); - } - - @SuppressWarnings("deprecation") - @Override - public KTable, Long> count(final SessionWindows sessionWindows, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(sessionWindows, "sessionWindows can't be null"); - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - final Merger sessionMerger = new Merger() { - @Override - public Long apply(final K aggKey, final Long aggOne, final Long aggTwo) { - return aggOne + aggTwo; - } - }; - - return aggregate(aggregateBuilder.countInitializer, - aggregateBuilder.countAggregator, - sessionMerger, - sessionWindows, - Serdes.Long(), - storeSupplier); - } - - - @SuppressWarnings({"unchecked", "deprecation"}) - @Override - public KTable, V> reduce(final Reducer reducer, - final SessionWindows sessionWindows, - final String queryableStoreName) { - determineIsQueryable(queryableStoreName); - - return reduce(reducer, sessionWindows, - storeFactory(keySerde, valSerde, getOrCreateName(queryableStoreName, AGGREGATE_NAME)) - .sessionWindowed(sessionWindows.maintainMs()).build()); - } - - @SuppressWarnings("deprecation") - @Override - public KTable, V> reduce(final Reducer reducer, - final SessionWindows sessionWindows) { - - return windowedBy(sessionWindows).reduce(reducer); - } @SuppressWarnings("deprecation") - @Override public KTable, V> reduce(final Reducer reducer, final SessionWindows sessionWindows, final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java index 6e33251d371db..3139d941b2573 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java @@ -18,7 +18,6 @@ import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.kstream.Aggregator; @@ -48,7 +47,6 @@ public class KGroupedTableImpl extends AbstractStream implements KGroup protected final Serde keySerde; protected final Serde valSerde; - private boolean isQueryable; private final Initializer countInitializer = new Initializer() { @Override public Long apply() { @@ -78,83 +76,6 @@ public Long apply(K aggKey, V value, Long aggregate) { super(builder, name, Collections.singleton(sourceName)); this.keySerde = keySerde; this.valSerde = valSerde; - this.isQueryable = true; - } - - private void determineIsQueryable(final String queryableStoreName) { - if (queryableStoreName == null) { - isQueryable = false; - } // no need for else {} since isQueryable is true by default - } - - @SuppressWarnings("deprecation") - @Override - public KTable aggregate(final Initializer initializer, - final Aggregator adder, - final Aggregator subtractor, - final Serde aggValueSerde, - final String queryableStoreName) { - determineIsQueryable(queryableStoreName); - return aggregate(initializer, adder, subtractor, keyValueStore(keySerde, aggValueSerde, getOrCreateName(queryableStoreName, AGGREGATE_NAME))); - } - - @SuppressWarnings("deprecation") - @Override - public KTable aggregate(final Initializer initializer, - final Aggregator adder, - final Aggregator subtractor, - final Serde aggValueSerde) { - return aggregate(initializer, adder, subtractor, aggValueSerde, null); - } - - @SuppressWarnings("deprecation") - @Override - public KTable aggregate(final Initializer initializer, - final Aggregator adder, - final Aggregator subtractor, - final String queryableStoreName) { - determineIsQueryable(queryableStoreName); - return aggregate(initializer, adder, subtractor, null, getOrCreateName(queryableStoreName, AGGREGATE_NAME)); - } - - @Override - public KTable aggregate(final Initializer initializer, - final Aggregator adder, - final Aggregator subtractor) { - return aggregate(initializer, adder, subtractor, (String) null); - } - - @SuppressWarnings("deprecation") - @Override - public KTable aggregate(final Initializer initializer, - final Aggregator adder, - final Aggregator subtractor, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(initializer, "initializer can't be null"); - Objects.requireNonNull(adder, "adder can't be null"); - Objects.requireNonNull(subtractor, "subtractor can't be null"); - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - ProcessorSupplier> aggregateSupplier = new KTableAggregate<>(storeSupplier.name(), initializer, adder, subtractor); - return doAggregate(aggregateSupplier, AGGREGATE_NAME, storeSupplier); - } - - @SuppressWarnings("deprecation") - private KTable doAggregate(final ProcessorSupplier> aggregateSupplier, - final String functionName, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - final String sinkName = builder.newProcessorName(KStreamImpl.SINK_NAME); - final String sourceName = builder.newProcessorName(KStreamImpl.SOURCE_NAME); - final String funcName = builder.newProcessorName(functionName); - - buildAggregate(aggregateSupplier, - storeSupplier.name() + KStreamImpl.REPARTITION_TOPIC_SUFFIX, - funcName, - sourceName, - sinkName); - builder.internalTopologyBuilder.addStateStore(storeSupplier, funcName); - - // return the KTable representation with the intermediate topic as the sources - return new KTableImpl<>(builder, funcName, aggregateSupplier, Collections.singleton(sourceName), storeSupplier.name(), isQueryable); } private void buildAggregate(final ProcessorSupplier> aggregateSupplier, @@ -196,16 +117,7 @@ private KTable doAggregate(final ProcessorSupplier> aggre .materialize(), funcName); // return the KTable representation with the intermediate topic as the sources - return new KTableImpl<>(builder, funcName, aggregateSupplier, Collections.singleton(sourceName), materialized.storeName(), isQueryable); - } - - @SuppressWarnings("deprecation") - @Override - public KTable reduce(final Reducer adder, - final Reducer subtractor, - final String queryableStoreName) { - determineIsQueryable(queryableStoreName); - return reduce(adder, subtractor, keyValueStore(keySerde, valSerde, getOrCreateName(queryableStoreName, REDUCE_NAME))); + return new KTableImpl<>(builder, funcName, aggregateSupplier, Collections.singleton(sourceName), materialized.storeName(), materialized.isQueryable()); } @Override @@ -226,26 +138,7 @@ public KTable reduce(final Reducer adder, @Override public KTable reduce(final Reducer adder, final Reducer subtractor) { - return reduce(adder, subtractor, (String) null); - } - - @SuppressWarnings("deprecation") - @Override - public KTable reduce(final Reducer adder, - final Reducer subtractor, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(adder, "adder can't be null"); - Objects.requireNonNull(subtractor, "subtractor can't be null"); - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - ProcessorSupplier> aggregateSupplier = new KTableReduce<>(storeSupplier.name(), adder, subtractor); - return doAggregate(aggregateSupplier, REDUCE_NAME, storeSupplier); - } - - @SuppressWarnings("deprecation") - @Override - public KTable count(final String queryableStoreName) { - determineIsQueryable(queryableStoreName); - return count(keyValueStore(keySerde, Serdes.Long(), getOrCreateName(queryableStoreName, AGGREGATE_NAME))); + return reduce(adder, subtractor, Materialized.>as(getOrCreateName(null, AGGREGATE_NAME))); } @Override @@ -256,7 +149,12 @@ public KTable count(final Materialized count() { + return count(Materialized.>as(getOrCreateName(null, AGGREGATE_NAME))); + } + + @Override public KTable aggregate(final Initializer initializer, final Aggregator adder, @@ -266,6 +164,7 @@ public KTable aggregate(final Initializer initializer, Objects.requireNonNull(adder, "adder can't be null"); Objects.requireNonNull(subtractor, "subtractor can't be null"); Objects.requireNonNull(materialized, "materialized can't be null"); + final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); if (materializedInternal.keySerde() == null) { @@ -279,18 +178,10 @@ public KTable aggregate(final Initializer initializer, } @Override - public KTable count() { - return count((String) null); - } - - @SuppressWarnings("deprecation") - @Override - public KTable count(final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - return this.aggregate( - countInitializer, - countAdder, - countSubtractor, - storeSupplier); + public KTable aggregate(final Initializer initializer, + final Aggregator adder, + final Aggregator subtractor) { + return aggregate(initializer, adder, subtractor, Materialized.>as(getOrCreateName(null, AGGREGATE_NAME))); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java index e19eed057b334..052cbdf3082ef 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.Consumed; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyValue; @@ -34,11 +35,14 @@ import org.apache.kafka.streams.kstream.KGroupedStream; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.Reducer; import org.apache.kafka.streams.kstream.Serialized; import org.apache.kafka.streams.kstream.TimeWindows; import org.apache.kafka.streams.kstream.Windowed; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.WindowStore; import org.apache.kafka.test.IntegrationTest; import org.apache.kafka.test.MockMapper; import org.apache.kafka.test.TestUtils; @@ -59,7 +63,7 @@ import static org.hamcrest.core.Is.is; /** - * Similar to KStreamAggregationIntegrationTest but with dedupping enabled + * Similar to KStreamAggregationIntegrationTest but with de dupping enabled * by virtue of having a large commit interval */ @Category({IntegrationTest.class}) @@ -100,7 +104,7 @@ public void before() throws InterruptedException { streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 10 * 1024 * 1024L); streamsConfiguration.put(IntegrationTestUtils.INTERNAL_LEAVE_GROUP_ON_CLOSE, true); - KeyValueMapper mapper = MockMapper.selectValueMapper(); + KeyValueMapper mapper = MockMapper.selectValueMapper(); stream = builder.stream(streamOneInput, Consumed.with(Serdes.Integer(), Serdes.String())); groupedStream = stream .groupBy( @@ -128,8 +132,9 @@ public void whenShuttingDown() throws IOException { public void shouldReduce() throws Exception { produceMessages(System.currentTimeMillis()); groupedStream - .reduce(reducer, "reduce-by-key") - .to(Serdes.String(), Serdes.String(), outputTopic); + .reduce(reducer, Materialized.>as("reduce-by-key")) + .toStream() + .to(outputTopic, Produced.with(Serdes.String(), Serdes.String())); startStreams(); @@ -174,7 +179,8 @@ public void shouldReduceWindowed() throws Exception { produceMessages(secondBatchTimestamp); groupedStream - .reduce(reducer, TimeWindows.of(500L), "reduce-time-windows") + .windowedBy(TimeWindows.of(500L)) + .reduce(reducer, Materialized.>as("reduce-time-windows")) .toStream(new KeyValueMapper, String, String>() { @Override public String apply(Windowed windowedKey, String value) { @@ -227,7 +233,8 @@ public void shouldGroupByKey() throws Exception { produceMessages(timestamp); stream.groupByKey(Serialized.with(Serdes.Integer(), Serdes.String())) - .count(TimeWindows.of(500L), "count-windows") + .windowedBy(TimeWindows.of(500L)) + .count(Materialized.>as("count-windows")) .toStream(new KeyValueMapper, Long, String>() { @Override public String apply(final Windowed windowedKey, final Long value) { diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java index 52b9ee80915a4..bb1745b17d0bd 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java @@ -54,8 +54,10 @@ import org.apache.kafka.streams.kstream.internals.SessionWindow; import org.apache.kafka.streams.kstream.internals.TimeWindow; import org.apache.kafka.streams.state.KeyValueIterator; +import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.QueryableStoreTypes; import org.apache.kafka.streams.state.ReadOnlySessionStore; +import org.apache.kafka.streams.state.SessionStore; import org.apache.kafka.streams.state.WindowStore; import org.apache.kafka.test.IntegrationTest; import org.apache.kafka.test.MockMapper; @@ -167,7 +169,7 @@ public void whenShuttingDown() throws IOException { public void shouldReduce() throws Exception { produceMessages(mockTime.milliseconds()); groupedStream - .reduce(reducer, "reduce-by-key") + .reduce(reducer, Materialized.>as("reduce-by-key")) .to(Serdes.String(), Serdes.String(), outputTopic); startStreams(); @@ -285,15 +287,14 @@ record = "KeyValue(" + record + ")"; } } - @SuppressWarnings("deprecation") @Test public void shouldAggregate() throws Exception { produceMessages(mockTime.milliseconds()); groupedStream.aggregate( initializer, aggregator, - Serdes.Integer(), - "aggregate-by-selected-key") + Materialized.>as("aggregate-by-selected-key") + .withValueSerde(Serdes.Integer())) .to(Serdes.String(), Serdes.Integer(), outputTopic); startStreams(); @@ -439,12 +440,11 @@ public int compare(final KeyValue o1, final KeyValue ))); } - @SuppressWarnings("deprecation") @Test public void shouldCount() throws Exception { produceMessages(mockTime.milliseconds()); - groupedStream.count("count-by-key") + groupedStream.count(Materialized.>as("count-by-key")) .to(Serdes.String(), Serdes.Long(), outputTopic); shouldCountHelper(); @@ -573,7 +573,8 @@ public void shouldCountSessionWindows() throws Exception { builder.stream(userSessionsStream, Consumed.with(Serdes.String(), Serdes.String())) .groupByKey(Serialized.with(Serdes.String(), Serdes.String())) - .count(SessionWindows.with(sessionGap).until(maintainMillis)) + .windowedBy(SessionWindows.with(sessionGap).until(maintainMillis)) + .count() .toStream() .foreach(new ForeachAction, Long>() { @Override @@ -661,12 +662,13 @@ public void shouldReduceSessionWindows() throws Exception { final String userSessionsStore = "UserSessionsStore"; builder.stream(userSessionsStream, Consumed.with(Serdes.String(), Serdes.String())) .groupByKey(Serialized.with(Serdes.String(), Serdes.String())) + .windowedBy(SessionWindows.with(sessionGap).until(maintainMillis)) .reduce(new Reducer() { @Override public String apply(final String value1, final String value2) { return value1 + ":" + value2; } - }, SessionWindows.with(sessionGap).until(maintainMillis), userSessionsStore) + }, Materialized.>as(userSessionsStore)) .foreach(new ForeachAction, String>() { @Override public void apply(final Windowed key, final String value) { diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/KStreamBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/KStreamBuilderTest.java index f9949d3c5b86e..72cd36d8c17c8 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/KStreamBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/KStreamBuilderTest.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.streams.errors.TopologyBuilderException; import org.apache.kafka.streams.kstream.internals.KStreamImpl; @@ -26,6 +27,7 @@ import org.apache.kafka.streams.processor.TopologyBuilder; import org.apache.kafka.streams.processor.internals.ProcessorTopology; import org.apache.kafka.streams.processor.internals.SourceNode; +import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.test.KStreamTestDriver; import org.apache.kafka.test.MockMapper; import org.apache.kafka.test.MockProcessorSupplier; @@ -186,7 +188,7 @@ public boolean test(final String key, final String value) { }); final KStream merged = processedSource1.merge(processedSource2).merge(source3); - merged.groupByKey().count("my-table"); + merged.groupByKey().count(Materialized.>as("my-table")); final Map> actual = builder.stateStoreNameToSourceTopics(); assertEquals(Utils.mkList("topic-1", "topic-2", "topic-3"), actual.get("my-table")); } @@ -301,7 +303,7 @@ public void shouldMapStateStoresToCorrectSourceTopics() { assertEquals(Collections.singletonList("table-topic"), builder.stateStoreNameToSourceTopics().get("table-store")); final KStream mapped = playEvents.map(MockMapper.selectValueKeyValueMapper()); - mapped.leftJoin(table, MockValueJoiner.TOSTRING_JOINER).groupByKey().count("count"); + mapped.leftJoin(table, MockValueJoiner.TOSTRING_JOINER).groupByKey().count(Materialized.>as("my-table")); assertEquals(Collections.singletonList("table-topic"), builder.stateStoreNameToSourceTopics().get("table-store")); assertEquals(Collections.singletonList(APP_ID + "-KSTREAM-MAP-0000000003-repartition"), builder.stateStoreNameToSourceTopics().get("count")); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilderTest.java index b9ba60894973e..c3c96426aa2fd 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilderTest.java @@ -132,7 +132,7 @@ public boolean test(final String key, final String value) { }); final KStream merged = processedSource1.merge(processedSource2).merge(source3); - merged.groupByKey().count("my-table"); + merged.groupByKey().count(Materialized.>as("my-table")); final Map> actual = builder.internalTopologyBuilder.stateStoreNameToSourceTopics(); assertEquals(Utils.mkList("topic-1", "topic-2", "topic-3"), actual.get("my-table")); } @@ -278,7 +278,7 @@ public void shouldMapStateStoresToCorrectSourceTopics() throws Exception { assertEquals(Collections.singletonList("table-topic"), builder.internalTopologyBuilder.stateStoreNameToSourceTopics().get("table-store")); final KStream mapped = playEvents.map(MockMapper.selectValueKeyValueMapper()); - mapped.leftJoin(table, MockValueJoiner.TOSTRING_JOINER).groupByKey().count("count"); + mapped.leftJoin(table, MockValueJoiner.TOSTRING_JOINER).groupByKey().count(Materialized.>as("count")); assertEquals(Collections.singletonList("table-topic"), builder.internalTopologyBuilder.stateStoreNameToSourceTopics().get("table-store")); assertEquals(Collections.singletonList(APP_ID + "-KSTREAM-MAP-0000000003-repartition"), builder.internalTopologyBuilder.stateStoreNameToSourceTopics().get("count")); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java index c74d0dce44955..2a740c1a21a1f 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java @@ -81,125 +81,84 @@ public void before() { groupedStream = stream.groupByKey(Serialized.with(Serdes.String(), Serdes.String())); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotHaveNullReducerOnReduce() { - groupedStream.reduce(null, "store"); + groupedStream.reduce(null); } - @SuppressWarnings("deprecation") @Test public void shouldAllowNullStoreNameOnReduce() { - groupedStream.reduce(MockReducer.STRING_ADDER, (String) null); + groupedStream.reduce(MockReducer.STRING_ADDER, Materialized.>as(null)); } - @SuppressWarnings("deprecation") @Test(expected = InvalidTopicException.class) public void shouldNotHaveInvalidStoreNameOnReduce() { - groupedStream.reduce(MockReducer.STRING_ADDER, INVALID_STORE_NAME); + groupedStream.reduce(MockReducer.STRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); } - @SuppressWarnings("deprecation") - @Test(expected = NullPointerException.class) - public void shouldNotHaveNullStoreSupplierOnReduce() { - groupedStream.reduce(MockReducer.STRING_ADDER, (StateStoreSupplier) null); - } - - @SuppressWarnings("deprecation") - @Test(expected = NullPointerException.class) - public void shouldNotHaveNullStoreSupplierOnCount() { - groupedStream.count((StateStoreSupplier) null); - } - - - @SuppressWarnings("deprecation") - @Test(expected = NullPointerException.class) - public void shouldNotHaveNullStoreSupplierOnWindowedCount() { - groupedStream.count(TimeWindows.of(10), (StateStoreSupplier) null); - } - - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotHaveNullReducerWithWindowedReduce() { - groupedStream.reduce(null, TimeWindows.of(10), "store"); + groupedStream.windowedBy(TimeWindows.of(10)).reduce(null, Materialized.>as("store")); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotHaveNullWindowsWithWindowedReduce() { - groupedStream.reduce(MockReducer.STRING_ADDER, (Windows) null, "store"); + groupedStream.windowedBy((Windows) null); } - @SuppressWarnings("deprecation") @Test public void shouldAllowNullStoreNameWithWindowedReduce() { - groupedStream.reduce(MockReducer.STRING_ADDER, TimeWindows.of(10), (String) null); + groupedStream.windowedBy(TimeWindows.of(10)).reduce(MockReducer.STRING_ADDER, Materialized.>as(null)); } - @SuppressWarnings("deprecation") @Test(expected = InvalidTopicException.class) public void shouldNotHaveInvalidStoreNameWithWindowedReduce() { - groupedStream.reduce(MockReducer.STRING_ADDER, TimeWindows.of(10), INVALID_STORE_NAME); + groupedStream.windowedBy(TimeWindows.of(10)).reduce(MockReducer.STRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotHaveNullInitializerOnAggregate() { - groupedStream.aggregate(null, MockAggregator.TOSTRING_ADDER, Serdes.String(), "store"); + groupedStream.aggregate(null, MockAggregator.TOSTRING_ADDER, Materialized.>as("store")); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotHaveNullAdderOnAggregate() { - groupedStream.aggregate(MockInitializer.STRING_INIT, null, Serdes.String(), "store"); + groupedStream.aggregate(MockInitializer.STRING_INIT, null, Materialized.>as("store")); } - @SuppressWarnings("deprecation") @Test public void shouldAllowNullStoreNameOnAggregate() { - groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Serdes.String(), null); + groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as(null)); } - @SuppressWarnings("deprecation") @Test(expected = InvalidTopicException.class) public void shouldNotHaveInvalidStoreNameOnAggregate() { - groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Serdes.String(), INVALID_STORE_NAME); + groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotHaveNullInitializerOnWindowedAggregate() { - groupedStream.aggregate(null, MockAggregator.TOSTRING_ADDER, TimeWindows.of(10), Serdes.String(), "store"); + groupedStream.windowedBy(TimeWindows.of(10)).aggregate(null, MockAggregator.TOSTRING_ADDER, Materialized.>as("store")); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotHaveNullAdderOnWindowedAggregate() { - groupedStream.aggregate(MockInitializer.STRING_INIT, null, TimeWindows.of(10), Serdes.String(), "store"); + groupedStream.windowedBy(TimeWindows.of(10)).aggregate(MockInitializer.STRING_INIT, null, Materialized.>as("store")); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotHaveNullWindowsOnWindowedAggregate() { - groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, null, Serdes.String(), "store"); + groupedStream.windowedBy((Windows) null); } - @SuppressWarnings("deprecation") @Test public void shouldAllowNullStoreNameOnWindowedAggregate() { - groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, TimeWindows.of(10), Serdes.String(), null); + groupedStream.windowedBy(TimeWindows.of(10)).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as(null)); } - @SuppressWarnings("deprecation") @Test(expected = InvalidTopicException.class) public void shouldNotHaveInvalidStoreNameOnWindowedAggregate() { - groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, TimeWindows.of(10), Serdes.String(), INVALID_STORE_NAME); - } - - @SuppressWarnings("deprecation") - @Test(expected = NullPointerException.class) - public void shouldNotHaveNullStoreSupplierOnWindowedAggregate() { - groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, TimeWindows.of(10), (StateStoreSupplier) null); + groupedStream.windowedBy(TimeWindows.of(10)).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); } private void doAggregateSessionWindows(final Map, Integer> results) { @@ -222,11 +181,10 @@ private void doAggregateSessionWindows(final Map, Integer> resu assertEquals(Integer.valueOf(3), results.get(new Windowed<>("1", new SessionWindow(70, 100)))); } - @SuppressWarnings("deprecation") @Test public void shouldAggregateSessionWindows() { final Map, Integer> results = new HashMap<>(); - final KTable, Integer> table = groupedStream.aggregate(new Initializer() { + final KTable, Integer> table = groupedStream.windowedBy(SessionWindows.with(30)).aggregate(new Initializer() { @Override public Integer apply() { return 0; @@ -241,7 +199,7 @@ public Integer apply(final String aggKey, final String value, final Integer aggr public Integer apply(final String aggKey, final Integer aggOne, final Integer aggTwo) { return aggOne + aggTwo; } - }, SessionWindows.with(30), Serdes.Integer(), "session-store"); + }, Materialized.>as("session-store").withValueSerde(Serdes.Integer())); table.toStream().foreach(new ForeachAction, Integer>() { @Override public void apply(final Windowed key, final Integer value) { @@ -253,11 +211,10 @@ public void apply(final Windowed key, final Integer value) { assertEquals(table.queryableStoreName(), "session-store"); } - @SuppressWarnings("deprecation") @Test public void shouldAggregateSessionWindowsWithInternalStoreName() { final Map, Integer> results = new HashMap<>(); - final KTable, Integer> table = groupedStream.aggregate(new Initializer() { + final KTable, Integer> table = groupedStream.windowedBy(SessionWindows.with(30)).aggregate(new Initializer() { @Override public Integer apply() { return 0; @@ -272,7 +229,7 @@ public Integer apply(final String aggKey, final String value, final Integer aggr public Integer apply(final String aggKey, final Integer aggOne, final Integer aggTwo) { return aggOne + aggTwo; } - }, SessionWindows.with(30), Serdes.Integer()); + }, Materialized.>with(null, Serdes.Integer())); table.toStream().foreach(new ForeachAction, Integer>() { @Override public void apply(final Windowed key, final Integer value) { @@ -303,11 +260,11 @@ private void doCountSessionWindows(final Map, Long> results) { assertEquals(Long.valueOf(3), results.get(new Windowed<>("1", new SessionWindow(70, 100)))); } - @SuppressWarnings("deprecation") @Test public void shouldCountSessionWindows() { final Map, Long> results = new HashMap<>(); - final KTable, Long> table = groupedStream.count(SessionWindows.with(30), "session-store"); + final KTable, Long> table = groupedStream.windowedBy(SessionWindows.with(30)) + .count(Materialized.>as("session-store")); table.toStream().foreach(new ForeachAction, Long>() { @Override public void apply(final Windowed key, final Long value) { @@ -318,11 +275,10 @@ public void apply(final Windowed key, final Long value) { assertEquals(table.queryableStoreName(), "session-store"); } - @SuppressWarnings("deprecation") @Test public void shouldCountSessionWindowsWithInternalStoreName() { final Map, Long> results = new HashMap<>(); - final KTable, Long> table = groupedStream.count(SessionWindows.with(30)); + final KTable, Long> table = groupedStream.windowedBy(SessionWindows.with(30)).count(); table.toStream().foreach(new ForeachAction, Long>() { @Override public void apply(final Windowed key, final Long value) { @@ -353,20 +309,16 @@ private void doReduceSessionWindows(final Map, String> results) assertEquals("A:B:C", results.get(new Windowed<>("1", new SessionWindow(70, 100)))); } - @SuppressWarnings("deprecation") @Test public void shouldReduceSessionWindows() { final Map, String> results = new HashMap<>(); - final KTable, String> table = groupedStream.reduce( - new Reducer() { - @Override - public String apply(final String value1, final String value2) { - return value1 + ":" + value2; - } - }, - SessionWindows.with(30), - "session-store" - ); + final KTable, String> table = groupedStream.windowedBy(SessionWindows.with(30)) + .reduce(new Reducer() { + @Override + public String apply(final String value1, final String value2) { + return value1 + ":" + value2; + } + }, Materialized.>as("session-store")); table.toStream().foreach(new ForeachAction, String>() { @Override public void apply(final Windowed key, final String value) { @@ -377,19 +329,16 @@ public void apply(final Windowed key, final String value) { assertEquals(table.queryableStoreName(), "session-store"); } - @SuppressWarnings("deprecation") @Test public void shouldReduceSessionWindowsWithInternalStoreName() { final Map, String> results = new HashMap<>(); - final KTable, String> table = groupedStream.reduce( - new Reducer() { - @Override - public String apply(final String value1, final String value2) { - return value1 + ":" + value2; - } - }, - SessionWindows.with(30) - ); + final KTable, String> table = groupedStream.windowedBy(SessionWindows.with(30)) + .reduce(new Reducer() { + @Override + public String apply(final String value1, final String value2) { + return value1 + ":" + value2; + } + }); table.toStream().foreach(new ForeachAction, String>() { @Override public void apply(final Windowed key, final String value) { @@ -400,136 +349,78 @@ public void apply(final Windowed key, final String value) { assertNull(table.queryableStoreName()); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotAcceptNullReducerWhenReducingSessionWindows() { - groupedStream.reduce(null, SessionWindows.with(10), "store"); + groupedStream.windowedBy(SessionWindows.with(30)).reduce(null, Materialized.>as("store")); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotAcceptNullSessionWindowsReducingSessionWindows() { - groupedStream.reduce(MockReducer.STRING_ADDER, (SessionWindows) null, "store"); - } - - @SuppressWarnings("deprecation") - @Test - public void shouldAcceptNullStoreNameWhenReducingSessionWindows() { - groupedStream.reduce(MockReducer.STRING_ADDER, SessionWindows.with(10), (String) null); + groupedStream.windowedBy((SessionWindows) null); } - @SuppressWarnings("deprecation") @Test(expected = InvalidTopicException.class) public void shouldNotAcceptInvalidStoreNameWhenReducingSessionWindows() { - groupedStream.reduce(MockReducer.STRING_ADDER, SessionWindows.with(10), INVALID_STORE_NAME); + groupedStream.windowedBy(SessionWindows.with(30)).reduce(MockReducer.STRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotAcceptNullStateStoreSupplierWhenReducingSessionWindows() { - groupedStream.reduce(MockReducer.STRING_ADDER, SessionWindows.with(10), (StateStoreSupplier) null); + groupedStream.windowedBy(SessionWindows.with(30)).reduce(null, Materialized.>as(null)); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotAcceptNullInitializerWhenAggregatingSessionWindows() { - groupedStream.aggregate(null, MockAggregator.TOSTRING_ADDER, new Merger() { + groupedStream.windowedBy(SessionWindows.with(30)).aggregate(null, MockAggregator.TOSTRING_ADDER, new Merger() { @Override public String apply(final String aggKey, final String aggOne, final String aggTwo) { return null; } - }, SessionWindows.with(10), Serdes.String(), "storeName"); + }, Materialized.>as("storeName")); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotAcceptNullAggregatorWhenAggregatingSessionWindows() { - groupedStream.aggregate(MockInitializer.STRING_INIT, null, new Merger() { + groupedStream.windowedBy(SessionWindows.with(30)).aggregate(MockInitializer.STRING_INIT, null, new Merger() { @Override public String apply(final String aggKey, final String aggOne, final String aggTwo) { return null; } - }, SessionWindows.with(10), Serdes.String(), "storeName"); + }, Materialized.>as("storeName")); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotAcceptNullSessionMergerWhenAggregatingSessionWindows() { - groupedStream.aggregate( - MockInitializer.STRING_INIT, - MockAggregator.TOSTRING_ADDER, - null, - SessionWindows.with(10), - Serdes.String(), - "storeName"); + groupedStream.windowedBy(SessionWindows.with(30)).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, + null, + Materialized.>as("storeName")); } - @SuppressWarnings("deprecation") @Test(expected = NullPointerException.class) public void shouldNotAcceptNullSessionWindowsWhenAggregatingSessionWindows() { - groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, new Merger() { - @Override - public String apply(final String aggKey, final String aggOne, final String aggTwo) { - return null; - } - }, null, Serdes.String(), "storeName"); + groupedStream.windowedBy((SessionWindows) null); } - @SuppressWarnings("deprecation") @Test public void shouldAcceptNullStoreNameWhenAggregatingSessionWindows() { - groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, new Merger() { - @Override - public String apply(final String aggKey, final String aggOne, final String aggTwo) { - return null; - } - }, SessionWindows.with(10), Serdes.String(), (String) null); + groupedStream.windowedBy(SessionWindows.with(10)) + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, new Merger() { + @Override + public String apply(final String aggKey, final String aggOne, final String aggTwo) { + return null; + } + }, Materialized.>with(Serdes.String(), Serdes.String())); } - @SuppressWarnings("deprecation") @Test(expected = InvalidTopicException.class) public void shouldNotAcceptInvalidStoreNameWhenAggregatingSessionWindows() { - groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, new Merger() { - @Override - public String apply(final String aggKey, final String aggOne, final String aggTwo) { - return null; - } - }, SessionWindows.with(10), Serdes.String(), INVALID_STORE_NAME); - } - - @SuppressWarnings("deprecation") - @Test(expected = NullPointerException.class) - public void shouldNotAcceptNullStateStoreSupplierNameWhenAggregatingSessionWindows() { - groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, new Merger() { - @Override - public String apply(final String aggKey, final String aggOne, final String aggTwo) { - return null; - } - }, SessionWindows.with(10), Serdes.String(), (StateStoreSupplier) null); - } - - @SuppressWarnings("deprecation") - @Test(expected = NullPointerException.class) - public void shouldNotAcceptNullSessionWindowsWhenCountingSessionWindows() { - groupedStream.count((SessionWindows) null, "store"); - } - - @SuppressWarnings("deprecation") - @Test - public void shouldAcceptNullStoreNameWhenCountingSessionWindows() { - groupedStream.count(SessionWindows.with(90), (String) null); - } - - @SuppressWarnings("deprecation") - @Test(expected = InvalidTopicException.class) - public void shouldNotAcceptInvalidStoreNameWhenCountingSessionWindows() { - groupedStream.count(SessionWindows.with(90), INVALID_STORE_NAME); - } - - @SuppressWarnings("deprecation") - @Test(expected = NullPointerException.class) - public void shouldNotAcceptNullStateStoreSupplierWhenCountingSessionWindows() { - groupedStream.count(SessionWindows.with(90), (StateStoreSupplier) null); + groupedStream.windowedBy(SessionWindows.with(10)) + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, new Merger() { + @Override + public String apply(final String aggKey, final String aggOne, final String aggTwo) { + return null; + } + }, Materialized.>as(INVALID_STORE_NAME)); } @SuppressWarnings("unchecked") @@ -692,13 +583,10 @@ private void doCountWindowed(final List, Long>> result ))); } - @SuppressWarnings("deprecation") @Test public void shouldCountWindowed() { final List, Long>> results = new ArrayList<>(); - groupedStream.count( - TimeWindows.of(500L), - "aggregate-by-key-windowed") + groupedStream.windowedBy(TimeWindows.of(500L)).count(Materialized.>as("aggregate-by-key-windowed")) .toStream() .foreach(new ForeachAction, Long>() { @Override @@ -710,12 +598,10 @@ public void apply(final Windowed key, final Long value) { doCountWindowed(results); } - @SuppressWarnings("deprecation") @Test public void shouldCountWindowedWithInternalStoreName() { final List, Long>> results = new ArrayList<>(); - groupedStream.count( - TimeWindows.of(500L)) + groupedStream.windowedBy(TimeWindows.of(500L)).count() .toStream() .foreach(new ForeachAction, Long>() { @Override diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java index 3bbd7e2fa5009..a906677d3184a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java @@ -28,7 +28,6 @@ import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Serialized; -import org.apache.kafka.streams.processor.StateStoreSupplier; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.test.KStreamTestDriver; import org.apache.kafka.test.MockAggregator; @@ -49,7 +48,6 @@ import static org.junit.Assert.assertNull; -@SuppressWarnings("deprecation") public class KGroupedTableImplTest { private final StreamsBuilder builder = new StreamsBuilder(); @@ -67,62 +65,57 @@ public void before() { @Test public void shouldAllowNullStoreNameOnCount() { - groupedTable.count((String) null); + groupedTable.count(Materialized.>as(null)); } @Test public void shouldAllowNullStoreNameOnAggregate() { - groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, (String) null); + groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as(null)); } @Test(expected = InvalidTopicException.class) public void shouldNotAllowInvalidStoreNameOnAggregate() { - groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, INVALID_STORE_NAME); + groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as(INVALID_STORE_NAME)); } @Test(expected = NullPointerException.class) public void shouldNotAllowNullInitializerOnAggregate() { - groupedTable.aggregate(null, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, "store"); + groupedTable.aggregate(null, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as("store")); } @Test(expected = NullPointerException.class) public void shouldNotAllowNullAdderOnAggregate() { - groupedTable.aggregate(MockInitializer.STRING_INIT, null, MockAggregator.TOSTRING_REMOVER, "store"); + groupedTable.aggregate(MockInitializer.STRING_INIT, null, MockAggregator.TOSTRING_REMOVER, Materialized.>as("store")); } @Test(expected = NullPointerException.class) public void shouldNotAllowNullSubtractorOnAggregate() { - groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, null, "store"); + groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, null, Materialized.>as("store")); } @Test(expected = NullPointerException.class) public void shouldNotAllowNullAdderOnReduce() { - groupedTable.reduce(null, MockReducer.STRING_REMOVER, "store"); + groupedTable.reduce(null, MockReducer.STRING_REMOVER, Materialized.>as("store")); } @Test(expected = NullPointerException.class) public void shouldNotAllowNullSubtractorOnReduce() { - groupedTable.reduce(MockReducer.STRING_ADDER, null, "store"); + groupedTable.reduce(MockReducer.STRING_ADDER, null, Materialized.>as("store")); } @Test public void shouldAllowNullStoreNameOnReduce() { - groupedTable.reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, (String) null); + groupedTable.reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as(null)); } @Test(expected = InvalidTopicException.class) public void shouldNotAllowInvalidStoreNameOnReduce() { - groupedTable.reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, INVALID_STORE_NAME); - } - - @Test(expected = NullPointerException.class) - public void shouldNotAllowNullStoreSupplierOnReduce() { - groupedTable.reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, (StateStoreSupplier) null); + groupedTable.reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as(INVALID_STORE_NAME)); } private void doShouldReduce(final KTable reduced, final String topic) { final Map results = new HashMap<>(); - reduced.foreach(new ForeachAction() { + reduced.toStream().foreach(new ForeachAction() { @Override public void apply(final String key, final Integer value) { results.put(key, value); @@ -164,7 +157,7 @@ public KeyValue apply(String key, Number value) { .withKeySerde(Serdes.String()) .withValueSerde(Serdes.Double())) .groupBy(intProjection) - .reduce(MockReducer.INTEGER_ADDER, MockReducer.INTEGER_SUBTRACTOR, "reduced"); + .reduce(MockReducer.INTEGER_ADDER, MockReducer.INTEGER_SUBTRACTOR, Materialized.>as("reduced")); doShouldReduce(reduced, topic); assertEquals(reduced.queryableStoreName(), "reduced"); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java index 7082251888946..424b077889ed9 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java @@ -69,7 +69,8 @@ public void testAggBasic() { final KTable, String> table2 = builder .stream(topic1, Consumed.with(strSerde, strSerde)) .groupByKey(Serialized.with(strSerde, strSerde)) - .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, TimeWindows.of(10).advanceBy(5), strSerde, "topic1-Canonized"); + .windowedBy(TimeWindows.of(10).advanceBy(5)) + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic1-Canonized")); final MockProcessorSupplier, String> proc2 = new MockProcessorSupplier<>(); table2.toStream().process(proc2); @@ -162,14 +163,17 @@ public void testJoin() { final KTable, String> table1 = builder .stream(topic1, Consumed.with(strSerde, strSerde)) .groupByKey(Serialized.with(strSerde, strSerde)) - .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, TimeWindows.of(10).advanceBy(5), strSerde, "topic1-Canonized"); + .windowedBy(TimeWindows.of(10).advanceBy(5)) + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic1-Canonized")); final MockProcessorSupplier, String> proc1 = new MockProcessorSupplier<>(); table1.toStream().process(proc1); final KTable, String> table2 = builder - .stream(topic2, Consumed.with(strSerde, strSerde)).groupByKey(Serialized.with(strSerde, strSerde)) - .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, TimeWindows.of(10).advanceBy(5), strSerde, "topic2-Canonized"); + .stream(topic2, Consumed.with(strSerde, strSerde)) + .groupByKey(Serialized.with(strSerde, strSerde)) + .windowedBy(TimeWindows.of(10).advanceBy(5)) + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic2-Canonized")); final MockProcessorSupplier, String> proc2 = new MockProcessorSupplier<>(); table2.toStream().process(proc2); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java index df8d2923a0453..9308601e1411c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.streams.Consumed; import org.apache.kafka.streams.KeyValue; @@ -28,10 +29,12 @@ import org.apache.kafka.streams.kstream.Initializer; import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Reducer; import org.apache.kafka.streams.kstream.Serialized; import org.apache.kafka.streams.kstream.ValueJoiner; import org.apache.kafka.streams.kstream.ValueMapper; +import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.test.KStreamTestDriver; import org.apache.kafka.test.MockAggregator; import org.apache.kafka.test.MockInitializer; @@ -78,8 +81,7 @@ public void testAggBasic() { ).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, - stringSerde, - "topic1-Canonized"); + Materialized.>as("topic1-Canonized").withValueSerde(stringSerde)); table2.toStream().process(proc); @@ -126,8 +128,7 @@ public void testAggCoalesced() { ).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, - stringSerde, - "topic1-Canonized"); + Materialized.>as("topic1-Canonized").withValueSerde(stringSerde)); table2.toStream().process(proc); @@ -167,8 +168,7 @@ public KeyValue apply(String key, String value) { .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, - stringSerde, - "topic1-Canonized"); + Materialized.>as("topic1-Canonized").withValueSerde(stringSerde)); table2.toStream().process(proc); @@ -236,7 +236,7 @@ public void testCount() { builder.table(input, consumed) .groupBy(MockMapper.selectValueKeyValueMapper(), stringSerialzied) - .count("count") + .count(Materialized.>as("count")) .toStream() .process(proc); @@ -266,7 +266,7 @@ public void testCountCoalesced() { builder.table(input, consumed) .groupBy(MockMapper.selectValueKeyValueMapper(), stringSerialzied) - .count("count") + .count(Materialized.>as("count")) .toStream() .process(proc); @@ -319,7 +319,7 @@ public String apply(String aggKey, String value, String aggregate) { public String apply(String key, String value, String aggregate) { return aggregate.replaceAll(value, ""); } - }, Serdes.String(), "someStore") + }, Materialized.>as("someStore").withValueSerde(Serdes.String())) .toStream() .process(proc); @@ -370,7 +370,7 @@ public Long apply(final Long value1, final Long value2) { public Long apply(final Long value1, final Long value2) { return value1 - value2; } - }, "reducer-store"); + }, Materialized.>as("reducer-store")); reduce.toStream().foreach(new ForeachAction() { @Override diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java index a0ca8274782b3..3c4ad0f6afaf6 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java @@ -457,7 +457,7 @@ public boolean test(String key, String value) { return value.equalsIgnoreCase("accept"); } }).groupBy(MockMapper.noOpKeyValueMapper()) - .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, "mock-result"); + .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as("mock-result")); doTestSkipNullOnMaterialization(builder, table1, table2, topic1); } @@ -479,7 +479,7 @@ public boolean test(String key, String value) { return value.equalsIgnoreCase("accept"); } }, "anyStoreNameFilter").groupBy(MockMapper.noOpKeyValueMapper()) - .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, "mock-result"); + .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as("mock-result")); doTestSkipNullOnMaterialization(builder, table1, table2, topic1); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java index 8646380d80c64..aebb6d17d9871 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java @@ -343,11 +343,11 @@ public void testRepartition() throws NoSuchFieldException, IllegalAccessExceptio ); table1.groupBy(MockMapper.noOpKeyValueMapper()) - .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, "mock-result1"); + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as("mock-result1")); table1.groupBy(MockMapper.noOpKeyValueMapper()) - .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, "mock-result2"); + .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as("mock-result2")); driver.setUp(builder, stateDir, stringSerde, stringSerde); driver.setTime(0L); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java index 6331b5731d8a0..9b4c2722e4a4c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java @@ -18,17 +18,20 @@ import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.Consumed; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsBuilderTest; import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Serialized; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.processor.MockProcessorContext; import org.apache.kafka.streams.processor.Processor; import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender; +import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.test.KStreamTestDriver; import org.apache.kafka.test.MockProcessorSupplier; import org.apache.kafka.test.MockReducer; @@ -358,7 +361,7 @@ public KeyValue apply(final Long key, final String value) { }, Serialized.with(Serdes.Long(), Serdes.String()) ) - .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_ADDER, "agg-store"); + .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_ADDER, Materialized.>as("agg-store")); final KTable one = builder.table(tableOne, consumed); final KTable two = builder.table(tableTwo, consumed); diff --git a/streams/src/test/java/org/apache/kafka/streams/perf/YahooBenchmark.java b/streams/src/test/java/org/apache/kafka/streams/perf/YahooBenchmark.java index e9785322e017b..294bb2628d0bc 100644 --- a/streams/src/test/java/org/apache/kafka/streams/perf/YahooBenchmark.java +++ b/streams/src/test/java/org/apache/kafka/streams/perf/YahooBenchmark.java @@ -27,6 +27,7 @@ import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.Consumed; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsBuilder; @@ -36,11 +37,13 @@ import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Predicate; import org.apache.kafka.streams.kstream.Serialized; import org.apache.kafka.streams.kstream.TimeWindows; import org.apache.kafka.streams.kstream.ValueJoiner; import org.apache.kafka.streams.kstream.ValueMapper; +import org.apache.kafka.streams.state.WindowStore; import java.util.ArrayList; import java.util.HashMap; @@ -337,7 +340,8 @@ public String apply(String key, String value) { // calculate windowed counts keyedByCampaign .groupByKey(Serialized.with(Serdes.String(), Serdes.String())) - .count(TimeWindows.of(10 * 1000), "time-windows"); + .windowedBy(TimeWindows.of(10 * 1000)) + .count(Materialized.>as("time-windows")); return new KafkaStreams(builder.build(), streamConfig); } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetadataStateTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetadataStateTest.java index 8e5d90dc9acba..a123fcf0e4c99 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetadataStateTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetadataStateTest.java @@ -71,19 +71,19 @@ public class StreamsMetadataStateTest { private StreamPartitioner partitioner; @Before - public void before() throws Exception { + public void before() { builder = new StreamsBuilder(); final KStream one = builder.stream("topic-one"); - one.groupByKey().count("table-one"); + one.groupByKey().count(Materialized.>as("table-one")); final KStream two = builder.stream("topic-two"); - two.groupByKey().count("table-two"); + two.groupByKey().count(Materialized.>as("table-two")); builder.stream("topic-three") .groupByKey() - .count("table-three"); + .count(Materialized.>as("table-three")); - one.merge(two).groupByKey().count("merged-table"); + one.merge(two).groupByKey().count(Materialized.>as("merged-table")); builder.stream("topic-four").mapValues(new ValueMapper() { @Override From 0bd43c3d829ea5e38f3fbefcbbd77537111b97b8 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 23 Apr 2018 23:35:04 -0700 Subject: [PATCH 03/13] continue working on KStreamImpl and KTableImpl --- .../kstream/internals/KStreamImpl.java | 38 +-------------- .../streams/kstream/internals/KTableImpl.java | 47 ++----------------- 2 files changed, 4 insertions(+), 81 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java index 9a6e5d577dc7b..2ddd5ff9518cf 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamImpl.java @@ -104,8 +104,6 @@ public class KStreamImpl extends AbstractStream implements KStream defaultKeyValueMapper; - private final boolean repartitionRequired; public KStreamImpl(final InternalStreamsBuilder builder, @@ -114,12 +112,6 @@ public KStreamImpl(final InternalStreamsBuilder builder, final boolean repartitionRequired) { super(builder, name, sourceNodes); this.repartitionRequired = repartitionRequired; - this.defaultKeyValueMapper = new KeyValueMapper() { - @Override - public String apply(K key, V value) { - return String.format("%s, %s", key, value); - } - }; } @Override @@ -191,16 +183,6 @@ public KStream mapValues(final ValueMapperWithKey(builder, name, sourceNodes, this.repartitionRequired); } - @SuppressWarnings("deprecation") - public void print(final KeyValueMapper mapper, - final Serde keySerde, - final Serde valSerde, - final String label) { - Objects.requireNonNull(mapper, "mapper can't be null"); - Objects.requireNonNull(label, "label can't be null"); - print(Printed.toSysOut().withLabel(label).withKeyValueMapper(mapper)); - } - @Override public void print(final Printed printed) { Objects.requireNonNull(printed, "printed can't be null"); @@ -209,17 +191,6 @@ public void print(final Printed printed) { builder.internalTopologyBuilder.addProcessor(name, printedInternal.build(this.name), this.name); } - @SuppressWarnings("deprecation") - public void writeAsText(final String filePath, - final String label, - final Serde keySerde, - final Serde valSerde, KeyValueMapper mapper) { - Objects.requireNonNull(filePath, "filePath can't be null"); - Objects.requireNonNull(label, "label can't be null"); - Objects.requireNonNull(mapper, "mapper can't be null"); - print(Printed.toFile(filePath).withKeyValueMapper(mapper).withLabel(label)); - } - @Override public KStream flatMap(final KeyValueMapper>> mapper) { Objects.requireNonNull(mapper, "mapper can't be null"); @@ -295,13 +266,6 @@ private KStream merge(final InternalStreamsBuilder builder, return new KStreamImpl<>(builder, name, allSourceNodes, requireRepartitioning); } - @SuppressWarnings("deprecation") - public KStream through(final Serde keySerde, - final Serde valSerde, - final StreamPartitioner partitioner, String topic) { - return through(topic, Produced.with(keySerde, valSerde, partitioner)); - } - @Override public KStream through(final String topic, final Produced produced) { final ProducedInternal producedInternal = new ProducedInternal<>(produced); @@ -333,7 +297,7 @@ public KStream peek(final ForeachAction action) { @Override public KStream through(final String topic) { - return through(null, null, null, topic); + return through(topic, Produced.with(null, null, null)); } @Override diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java index 0c53dbbc9a0ea..8a174e19b0615 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java @@ -136,26 +136,6 @@ public String queryableStoreName() { return this.queryableStoreName; } - @SuppressWarnings("deprecation") - private KTable doFilter(final Predicate predicate, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier, - final boolean isFilterNot) { - Objects.requireNonNull(predicate, "predicate can't be null"); - String name = builder.newProcessorName(FILTER_NAME); - String internalStoreName = null; - if (storeSupplier != null) { - internalStoreName = storeSupplier.name(); - } - KTableProcessorSupplier processorSupplier = new KTableFilter<>(this, predicate, isFilterNot, internalStoreName); - builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); - if (storeSupplier != null) { - builder.internalTopologyBuilder.addStateStore(storeSupplier, name); - return new KTableImpl<>(builder, name, processorSupplier, this.keySerde, this.valSerde, sourceNodes, internalStoreName, true); - } else { - return new KTableImpl<>(builder, name, processorSupplier, sourceNodes, this.queryableStoreName, false); - } - } - private KTable doFilter(final Predicate predicate, final MaterializedInternal> materialized, final boolean filterNot) { @@ -177,12 +157,12 @@ private KTable doFilter(final Predicate predicate, this.valSerde, sourceNodes, builder.name(), - true); + materialized.isQueryable()); } @Override public KTable filter(final Predicate predicate) { - return filter(predicate, (String) null); + return filter(predicate, Materialized.>with(null, null)); } @Override @@ -193,19 +173,9 @@ public KTable filter(final Predicate predicate, return doFilter(predicate, new MaterializedInternal<>(materialized, builder, FILTER_NAME), false); } - @SuppressWarnings("deprecation") - public KTable filter(final Predicate predicate, - final String queryableStoreName) { - org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier = null; - if (queryableStoreName != null) { - storeSupplier = keyValueStore(this.keySerde, this.valSerde, queryableStoreName); - } - return doFilter(predicate, storeSupplier, false); - } - @Override public KTable filterNot(final Predicate predicate) { - return filterNot(predicate, (String) null); + return filterNot(predicate, Materialized.>with(null, null)); } @Override @@ -215,17 +185,6 @@ public KTable filterNot(final Predicate predicate, Objects.requireNonNull(materialized, "materialized can't be null"); return doFilter(predicate, new MaterializedInternal<>(materialized, builder, FILTER_NAME), true); } - - @SuppressWarnings("deprecation") - public KTable filterNot(final Predicate predicate, - final String queryableStoreName) { - org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier = null; - if (queryableStoreName != null) { - storeSupplier = keyValueStore(this.keySerde, this.valSerde, queryableStoreName); - } - return doFilter(predicate, storeSupplier, true); - } - @SuppressWarnings("deprecation") private KTable doMapValues(final ValueMapperWithKey mapper, final Serde valueSerde, From c302aba8ece8976896153e501ca7e8f224a83f5c Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 24 Apr 2018 15:05:56 -0700 Subject: [PATCH 04/13] continue working on KGroupedStreamImpl --- .../kstream/internals/KGroupedStreamImpl.java | 189 +----------------- 1 file changed, 2 insertions(+), 187 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java index e9aeae54fa1a7..8ecfb5c2b6a33 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java @@ -24,20 +24,15 @@ import org.apache.kafka.streams.kstream.KGroupedStream; import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.Materialized; -import org.apache.kafka.streams.kstream.Merger; import org.apache.kafka.streams.kstream.Reducer; import org.apache.kafka.streams.kstream.SessionWindowedKStream; import org.apache.kafka.streams.kstream.SessionWindows; import org.apache.kafka.streams.kstream.TimeWindowedKStream; import org.apache.kafka.streams.kstream.Window; -import org.apache.kafka.streams.kstream.Windowed; import org.apache.kafka.streams.kstream.Windows; import org.apache.kafka.streams.state.KeyValueStore; -import org.apache.kafka.streams.state.SessionStore; import org.apache.kafka.streams.state.StoreBuilder; -import org.apache.kafka.streams.state.WindowStore; -import java.util.Collections; import java.util.Objects; import java.util.Set; @@ -50,7 +45,6 @@ class KGroupedStreamImpl extends AbstractStream implements KGroupedStre private final Serde valSerde; private final boolean repartitionRequired; private final GroupedStreamAggregateBuilder aggregateBuilder; - private boolean isQueryable = true; KGroupedStreamImpl(final InternalStreamsBuilder builder, final String name, @@ -68,37 +62,11 @@ class KGroupedStreamImpl extends AbstractStream implements KGroupedStre this.keySerde = keySerde; this.valSerde = valSerde; this.repartitionRequired = repartitionRequired; - this.isQueryable = true; - } - - private void determineIsQueryable(final String queryableStoreName) { - if (queryableStoreName == null) { - isQueryable = false; - } // no need for else {} since isQueryable is true by default - } - - @SuppressWarnings("deprecation") - public KTable reduce(final Reducer reducer, - final String queryableStoreName) { - determineIsQueryable(queryableStoreName); - return reduce(reducer, keyValueStore(keySerde, valSerde, getOrCreateName(queryableStoreName, REDUCE_NAME))); } @Override public KTable reduce(final Reducer reducer) { - determineIsQueryable(null); - return reduce(reducer, (String) null); - } - - @SuppressWarnings("deprecation") - public KTable reduce(final Reducer reducer, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(reducer, "reducer can't be null"); - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - return doAggregate( - new KStreamReduce(storeSupplier.name(), reducer), - REDUCE_NAME, - storeSupplier); + return reduce(reducer, Materialized.>with(null, null)); } @Override @@ -114,29 +82,6 @@ public KTable reduce(final Reducer reducer, materializedInternal); } - @SuppressWarnings({"unchecked", "deprecation"}) - public KTable, V> reduce(final Reducer reducer, - final Windows windows, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(reducer, "reducer can't be null"); - Objects.requireNonNull(windows, "windows can't be null"); - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - return (KTable, V>) doAggregate( - new KStreamWindowReduce(windows, storeSupplier.name(), reducer), - REDUCE_NAME, - storeSupplier - ); - } - - @SuppressWarnings("deprecation") - public KTable aggregate(final Initializer initializer, - final Aggregator aggregator, - final Serde aggValueSerde, - final String queryableStoreName) { - determineIsQueryable(queryableStoreName); - return aggregate(initializer, aggregator, keyValueStore(keySerde, aggValueSerde, getOrCreateName(queryableStoreName, AGGREGATE_NAME))); - } - @Override public KTable aggregate(final Initializer initializer, final Aggregator aggregator, @@ -173,49 +118,9 @@ public KTable aggregate(final Initializer initializer, } - @SuppressWarnings("deprecation") - public KTable aggregate(final Initializer initializer, - final Aggregator aggregator, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(initializer, "initializer can't be null"); - Objects.requireNonNull(aggregator, "aggregator can't be null"); - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - return doAggregate( - new KStreamAggregate<>(storeSupplier.name(), initializer, aggregator), - AGGREGATE_NAME, - storeSupplier); - } - - @SuppressWarnings({"unchecked", "deprecation"}) - public KTable, T> aggregate(final Initializer initializer, - final Aggregator aggregator, - final Windows windows, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(initializer, "initializer can't be null"); - Objects.requireNonNull(aggregator, "aggregator can't be null"); - Objects.requireNonNull(windows, "windows can't be null"); - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - return (KTable, T>) doAggregate( - new KStreamWindowAggregate<>(windows, storeSupplier.name(), initializer, aggregator), - AGGREGATE_NAME, - storeSupplier - ); - } - - @SuppressWarnings("deprecation") - public KTable count(final String queryableStoreName) { - determineIsQueryable(queryableStoreName); - return count(keyValueStore(keySerde, Serdes.Long(), getOrCreateName(queryableStoreName, AGGREGATE_NAME))); - } - @Override public KTable count() { - return count((String) null); - } - - @SuppressWarnings("deprecation") - public KTable count(final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - return aggregate(aggregateBuilder.countInitializer, aggregateBuilder.countAggregator, storeSupplier); + return count(Materialized.>with(null, null)); } @Override @@ -229,36 +134,6 @@ public KTable count(final Materialized KTable, Long> count(final Windows windows, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - return aggregate( - aggregateBuilder.countInitializer, - aggregateBuilder.countAggregator, - windows, - storeSupplier); - } - - @SuppressWarnings({"unchecked", "deprecation"}) - public KTable, T> aggregate(final Initializer initializer, - final Aggregator aggregator, - final Merger sessionMerger, - final SessionWindows sessionWindows, - final Serde aggValueSerde, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(initializer, "initializer can't be null"); - Objects.requireNonNull(aggregator, "aggregator can't be null"); - Objects.requireNonNull(sessionWindows, "sessionWindows can't be null"); - Objects.requireNonNull(sessionMerger, "sessionMerger can't be null"); - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - - return (KTable, T>) doAggregate( - new KStreamSessionWindowAggregate<>(sessionWindows, storeSupplier.name(), initializer, aggregator, sessionMerger), - AGGREGATE_NAME, - storeSupplier); - - } - @Override public TimeWindowedKStream windowedBy(final Windows windows) { return new TimeWindowedKStreamImpl<>(windows, @@ -281,43 +156,6 @@ public SessionWindowedKStream windowedBy(final SessionWindows windows) { aggregateBuilder); } - - @SuppressWarnings("deprecation") - public KTable, V> reduce(final Reducer reducer, - final SessionWindows sessionWindows, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(reducer, "reducer can't be null"); - Objects.requireNonNull(sessionWindows, "sessionWindows can't be null"); - Objects.requireNonNull(storeSupplier, "storeSupplier can't be null"); - - final Initializer initializer = new Initializer() { - @Override - public V apply() { - return null; - } - }; - - final Aggregator aggregator = new Aggregator() { - @Override - public V apply(final K aggKey, final V value, final V aggregate) { - if (aggregate == null) { - return value; - } - return reducer.apply(aggregate, value); - } - }; - - final Merger sessionMerger = new Merger() { - @Override - public V apply(final K aggKey, final V aggOne, final V aggTwo) { - return aggregator.apply(aggKey, aggTwo, aggOne); - } - }; - - return aggregate(initializer, aggregator, sessionMerger, sessionWindows, valSerde, storeSupplier); - } - - private KTable doAggregate(final KStreamAggProcessorSupplier aggregateSupplier, final String functionName, final MaterializedInternal> materializedInternal) { @@ -328,29 +166,6 @@ private KTable doAggregate(final KStreamAggProcessorSupplier KTable doAggregate( - final KStreamAggProcessorSupplier aggregateSupplier, - final String functionName, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - - final String aggFunctionName = builder.newProcessorName(functionName); - - final String sourceName = repartitionIfRequired(storeSupplier.name()); - - builder.internalTopologyBuilder.addProcessor(aggFunctionName, aggregateSupplier, sourceName); - builder.internalTopologyBuilder.addStateStore(storeSupplier, aggFunctionName); - - return new KTableImpl<>( - builder, - aggFunctionName, - aggregateSupplier, - sourceName.equals(this.name) ? sourceNodes - : Collections.singleton(sourceName), - storeSupplier.name(), - isQueryable); - } - /** * @return the new sourceName if repartitioned. Otherwise the name of this stream */ From 81c05cadee0ab2aa3d075e416c0eb494c8a05bf0 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 24 Apr 2018 19:14:39 -0700 Subject: [PATCH 05/13] fix unit tests --- .../kafka/streams/kstream/KGroupedStream.java | 3 - .../kafka/streams/kstream/KGroupedTable.java | 1 - .../apache/kafka/streams/kstream/KStream.java | 1 - .../kstream/internals/KGroupedStreamImpl.java | 44 ++++------- .../kstream/internals/KGroupedTableImpl.java | 6 +- .../streams/kstream/internals/KTableImpl.java | 72 ++--------------- .../internals/SessionWindowedKStreamImpl.java | 66 +++++----------- .../KStreamAggregationIntegrationTest.java | 13 +--- .../streams/kstream/KStreamBuilderTest.java | 2 +- .../internals/KGroupedStreamImplTest.java | 77 +++++++------------ .../internals/KGroupedTableImplTest.java | 22 +++--- .../kstream/internals/KStreamImplTest.java | 1 - .../internals/KStreamWindowAggregateTest.java | 6 +- .../internals/KTableAggregateTest.java | 14 ++-- .../kstream/internals/KTableFilterTest.java | 12 +-- .../kstream/internals/KTableImplTest.java | 9 +-- .../internals/KTableKTableLeftJoinTest.java | 2 +- .../kafka/streams/perf/YahooBenchmark.java | 2 +- .../internals/StreamsMetadataStateTest.java | 8 +- 19 files changed, 115 insertions(+), 246 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java index f7069e9074dfe..a0cab6302c19e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java @@ -17,7 +17,6 @@ package org.apache.kafka.streams.kstream; import org.apache.kafka.common.annotation.InterfaceStability; -import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.KafkaStreams; @@ -25,8 +24,6 @@ import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.QueryableStoreType; -import org.apache.kafka.streams.state.SessionStore; -import org.apache.kafka.streams.state.WindowStore; /** * {@code KGroupedStream} is an abstraction of a grouped record stream of {@link KeyValue} pairs. diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java index 210336f26b965..1d1300e5bff5f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java @@ -17,7 +17,6 @@ package org.apache.kafka.streams.kstream; import org.apache.kafka.common.annotation.InterfaceStability; -import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsConfig; diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java index 853145384b70e..88b856576298f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java @@ -25,7 +25,6 @@ import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.Topology; -import org.apache.kafka.streams.kstream.internals.WindowedStreamPartitioner; import org.apache.kafka.streams.processor.Processor; import org.apache.kafka.streams.processor.ProcessorContext; import org.apache.kafka.streams.processor.ProcessorSupplier; diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java index 8ecfb5c2b6a33..1a4478c57fa25 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java @@ -66,7 +66,7 @@ class KGroupedStreamImpl extends AbstractStream implements KGroupedStre @Override public KTable reduce(final Reducer reducer) { - return reduce(reducer, Materialized.>with(null, null)); + return reduce(reducer, Materialized.>with(keySerde, valSerde)); } @Override @@ -76,6 +76,13 @@ public KTable reduce(final Reducer reducer, Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized, builder, REDUCE_NAME); + if (materializedInternal.keySerde() == null) { + materializedInternal.withKeySerde(keySerde); + } + if (materializedInternal.valueSerde() == null) { + materializedInternal.withValueSerde(valSerde); + } + return doAggregate( new KStreamReduce(materializedInternal.storeName(), reducer), REDUCE_NAME, @@ -89,14 +96,13 @@ public KTable aggregate(final Initializer initializer, Objects.requireNonNull(initializer, "initializer can't be null"); Objects.requireNonNull(aggregator, "aggregator can't be null"); Objects.requireNonNull(materialized, "materialized can't be null"); - return aggregateMaterialized(initializer, aggregator, materialized); - } - private KTable aggregateMaterialized(final Initializer initializer, - final Aggregator aggregator, - final Materialized> materialized) { final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); + if (materializedInternal.keySerde() == null) { + materializedInternal.withKeySerde(keySerde); + } + return doAggregate( new KStreamAggregate<>(materializedInternal.storeName(), initializer, aggregator), AGGREGATE_NAME, @@ -106,21 +112,12 @@ private KTable aggregateMaterialized(final Initializer initializ @Override public KTable aggregate(final Initializer initializer, final Aggregator aggregator) { - Objects.requireNonNull(initializer, "initializer can't be null"); - Objects.requireNonNull(aggregator, "aggregator can't be null"); - MaterializedInternal> materializedInternal = - new MaterializedInternal<>(Materialized.>with(keySerde, null), - builder, - AGGREGATE_NAME); - return doAggregate(new KStreamAggregate<>(materializedInternal.storeName(), initializer, aggregator), - AGGREGATE_NAME, - materializedInternal); - + return aggregate(initializer, aggregator, Materialized.>with(keySerde, null)); } @Override public KTable count() { - return count(Materialized.>with(null, null)); + return count(Materialized.>with(keySerde, Serdes.Long())); } @Override @@ -128,6 +125,9 @@ public KTable count(final Materialized> materializedInternal = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); + if (materializedInternal.keySerde() == null) { + materialized.withKeySerde(keySerde); + } if (materializedInternal.valueSerde() == null) { materialized.withValueSerde(Serdes.Long()); } @@ -165,14 +165,4 @@ private KTable doAggregate(final KStreamAggProcessorSupplier reduce(final Reducer adder, @Override public KTable reduce(final Reducer adder, final Reducer subtractor) { - return reduce(adder, subtractor, Materialized.>as(getOrCreateName(null, AGGREGATE_NAME))); + return reduce(adder, subtractor, Materialized.>as(getOrCreateName(null, AGGREGATE_NAME))); } @Override @@ -151,7 +151,7 @@ public KTable count(final Materialized count() { - return count(Materialized.>as(getOrCreateName(null, AGGREGATE_NAME))); + return count(Materialized.>as(getOrCreateName(null, AGGREGATE_NAME))); } @@ -181,7 +181,7 @@ public KTable aggregate(final Initializer initializer, public KTable aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor) { - return aggregate(initializer, adder, subtractor, Materialized.>as(getOrCreateName(null, AGGREGATE_NAME))); + return aggregate(initializer, adder, subtractor, Materialized.>as(getOrCreateName(null, AGGREGATE_NAME))); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java index 8a174e19b0615..b059670455b2c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java @@ -28,6 +28,7 @@ import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.Serialized; import org.apache.kafka.streams.kstream.ValueJoiner; import org.apache.kafka.streams.kstream.ValueMapper; @@ -185,34 +186,15 @@ public KTable filterNot(final Predicate predicate, Objects.requireNonNull(materialized, "materialized can't be null"); return doFilter(predicate, new MaterializedInternal<>(materialized, builder, FILTER_NAME), true); } - @SuppressWarnings("deprecation") - private KTable doMapValues(final ValueMapperWithKey mapper, - final Serde valueSerde, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(mapper); - String name = builder.newProcessorName(MAPVALUES_NAME); - String internalStoreName = null; - if (storeSupplier != null) { - internalStoreName = storeSupplier.name(); - } - KTableProcessorSupplier processorSupplier = new KTableMapValues<>(this, mapper, internalStoreName); - builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); - if (storeSupplier != null) { - builder.internalTopologyBuilder.addStateStore(storeSupplier, name); - return new KTableImpl<>(builder, name, processorSupplier, this.keySerde, valueSerde, sourceNodes, internalStoreName, true); - } else { - return new KTableImpl<>(builder, name, processorSupplier, sourceNodes, this.queryableStoreName, false); - } - } @Override public KTable mapValues(final ValueMapper mapper) { - return doMapValues(withKey(mapper), null, null); + return mapValues(withKey(mapper), Materialized.>with(null, null)); } @Override public KTable mapValues(final ValueMapperWithKey mapper) { - return doMapValues(mapper, null, null); + return mapValues(mapper, Materialized.>with(null, null)); } @@ -447,14 +429,14 @@ public KTable through(final String topic) { @SuppressWarnings("deprecation") @Override public void to(final String topic) { - //to(null, null, null, topic); + to(null, null, null, topic); } @SuppressWarnings("deprecation") @Override public void to(final StreamPartitioner partitioner, final String topic) { - //to(null, null, partitioner, topic); + to(null, null, partitioner, topic); } @SuppressWarnings("deprecation") @@ -462,7 +444,7 @@ public void to(final StreamPartitioner partitioner, public void to(final Serde keySerde, final Serde valSerde, final String topic) { - //this.toStream().to(keySerde, valSerde, null, topic); + this.toStream().to(topic, Produced.with(keySerde, valSerde)); } @SuppressWarnings("deprecation") @@ -471,7 +453,7 @@ public void to(final Serde keySerde, final Serde valSerde, final StreamPartitioner partitioner, final String topic) { - //this.toStream().to(keySerde, valSerde, partitioner, topic); + this.toStream().to(topic, Produced.with(keySerde, valSerde).withStreamPartitioner(partitioner)); } @Override @@ -539,46 +521,6 @@ public KTable leftJoin(final KTable other, false); } - @SuppressWarnings({"unchecked", "deprecation"}) - private KTable doJoin(final KTable other, - final ValueJoiner joiner, - final boolean leftOuter, - final boolean rightOuter, - final Serde joinSerde, - final String queryableStoreName) { - Objects.requireNonNull(other, "other can't be null"); - Objects.requireNonNull(joiner, "joiner can't be null"); - - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier - = queryableStoreName == null ? null : keyValueStore(this.keySerde, joinSerde, queryableStoreName); - - return doJoin(other, joiner, leftOuter, rightOuter, storeSupplier); - } - - @SuppressWarnings({"unchecked", "deprecation"}) - private KTable doJoin(final KTable other, - final ValueJoiner joiner, - final boolean leftOuter, - final boolean rightOuter, - final org.apache.kafka.streams.processor.StateStoreSupplier storeSupplier) { - Objects.requireNonNull(other, "other can't be null"); - Objects.requireNonNull(joiner, "joiner can't be null"); - final String joinMergeName = builder.newProcessorName(MERGE_NAME); - final String internalQueryableName = storeSupplier == null ? null : storeSupplier.name(); - final KTable result = buildJoin((AbstractStream) other, - joiner, - leftOuter, - rightOuter, - joinMergeName, - internalQueryableName); - - if (internalQueryableName != null) { - builder.internalTopologyBuilder.addStateStore(storeSupplier, joinMergeName); - } - - return result; - } - @SuppressWarnings("unchecked") private KTable doJoin(final KTable other, final ValueJoiner joiner, diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java index 34e5bd7b1bafe..bcbe71da1f3ea 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java @@ -65,6 +65,7 @@ public V apply() { final Serde valSerde, final GroupedStreamAggregateBuilder aggregateBuilder) { super(builder, name, sourceNodes); + Objects.requireNonNull(windows, "windows can't be null"); this.windows = windows; this.keySerde = keySerde; this.valSerde = valSerde; @@ -73,10 +74,7 @@ public V apply() { @Override public KTable, Long> count() { - return doAggregate(aggregateBuilder.countInitializer, - aggregateBuilder.countAggregator, - countMerger, - Serdes.Long()); + return count(Materialized.>with(null, Serdes.Long())); } @Override @@ -93,15 +91,25 @@ public KTable, Long> count(final Materialized, V> reduce(final Reducer reducer) { + return reduce(reducer, Materialized.>with(null, null)); + } + + @Override + public KTable, V> reduce(final Reducer reducer, + final Materialized> materialized) { + Objects.requireNonNull(reducer, "reducer can't be null"); + Objects.requireNonNull(materialized, "materialized can't be null"); + final Aggregator reduceAggregator = aggregatorForReducer(reducer); + return aggregate(reduceInitializer, reduceAggregator, mergerForAggregator(reduceAggregator), materialized); + } + @Override public KTable, T> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger) { - Objects.requireNonNull(initializer, "initializer can't be null"); - Objects.requireNonNull(aggregator, "aggregator can't be null"); - Objects.requireNonNull(sessionMerger, "sessionMerger can't be null"); - return doAggregate(initializer, aggregator, sessionMerger, (Serde) valSerde); + return aggregate(initializer, aggregator, sessionMerger, Materialized.>with(null, null)); } @SuppressWarnings("unchecked") @@ -123,25 +131,9 @@ public KTable, VR> aggregate(final Initializer initializer, new KStreamSessionWindowAggregate<>(windows, materializedInternal.storeName(), initializer, aggregator, sessionMerger), AGGREGATE_NAME, materialize(materializedInternal), - true); - } - - @Override - public KTable, V> reduce(final Reducer reducer) { - Objects.requireNonNull(reducer, "reducer can't be null"); - return doAggregate(reduceInitializer, aggregatorForReducer(reducer), mergerForAggregator(aggregatorForReducer(reducer)), valSerde); + materializedInternal.isQueryable()); } - @Override - public KTable, V> reduce(final Reducer reducer, - final Materialized> materialized) { - Objects.requireNonNull(reducer, "reducer can't be null"); - Objects.requireNonNull(materialized, "materialized can't be null"); - final Aggregator reduceAggregator = aggregatorForReducer(reducer); - return aggregate(reduceInitializer, reduceAggregator, mergerForAggregator(reduceAggregator), materialized); - } - - private StoreBuilder> materialize(final MaterializedInternal> materialized) { SessionBytesStoreSupplier supplier = (SessionBytesStoreSupplier) materialized.storeSupplier(); if (supplier == null) { @@ -184,26 +176,4 @@ public V apply(final K aggKey, final V value, final V aggregate) { } }; } - - private StoreBuilder> storeBuilder(final String storeName, final Serde aggValueSerde) { - return Stores.sessionStoreBuilder( - Stores.persistentSessionStore( - storeName, - windows.maintainMs()), - keySerde, - aggValueSerde).withCachingEnabled(); - } - - - @SuppressWarnings("unchecked") - private KTable, VR> doAggregate(final Initializer initializer, - final Aggregator aggregator, - final Merger merger, - final Serde serde) { - final String storeName = builder.newStoreName(AGGREGATE_NAME); - return (KTable, VR>) aggregateBuilder.build(new KStreamSessionWindowAggregate<>(windows, storeName, initializer, aggregator, merger), - AGGREGATE_NAME, - storeBuilder(storeName, serde), - false); - } } diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java index bb1745b17d0bd..11c2dbf0a980f 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java @@ -164,12 +164,11 @@ public void whenShuttingDown() throws IOException { IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration); } - @SuppressWarnings("deprecation") @Test public void shouldReduce() throws Exception { produceMessages(mockTime.milliseconds()); groupedStream - .reduce(reducer, Materialized.>as("reduce-by-key")) + .reduce(reducer, Materialized.>as("reduce-by-key")) .to(Serdes.String(), Serdes.String(), outputTopic); startStreams(); @@ -293,8 +292,7 @@ public void shouldAggregate() throws Exception { groupedStream.aggregate( initializer, aggregator, - Materialized.>as("aggregate-by-selected-key") - .withValueSerde(Serdes.Integer())) + Materialized.>as("aggregate-by-selected-key")) .to(Serdes.String(), Serdes.Integer(), outputTopic); startStreams(); @@ -444,13 +442,12 @@ public int compare(final KeyValue o1, final KeyValue public void shouldCount() throws Exception { produceMessages(mockTime.milliseconds()); - groupedStream.count(Materialized.>as("count-by-key")) + groupedStream.count(Materialized.>as("count-by-key")) .to(Serdes.String(), Serdes.Long(), outputTopic); shouldCountHelper(); } - @SuppressWarnings("deprecation") @Test public void shouldCountWithInternalStore() throws Exception { produceMessages(mockTime.milliseconds()); @@ -506,7 +503,6 @@ public int compare(final KeyValue o1, final KeyValue } - @SuppressWarnings("deprecation") @Test public void shouldCountSessionWindows() throws Exception { final long sessionGap = 5 * 60 * 1000L; @@ -595,7 +591,6 @@ public void apply(final Windowed key, final Long value) { assertThat(results.get(new Windowed<>("penny", new SessionWindow(t3, t3))), equalTo(1L)); } - @SuppressWarnings("deprecation") @Test public void shouldReduceSessionWindows() throws Exception { final long sessionGap = 1000L; // something to do with time @@ -668,7 +663,7 @@ public void shouldReduceSessionWindows() throws Exception { public String apply(final String value1, final String value2) { return value1 + ":" + value2; } - }, Materialized.>as(userSessionsStore)) + }, Materialized.>as(userSessionsStore).withValueSerde(Serdes.String())) .foreach(new ForeachAction, String>() { @Override public void apply(final Windowed key, final String value) { diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/KStreamBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/KStreamBuilderTest.java index 72cd36d8c17c8..529d7c154929a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/KStreamBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/KStreamBuilderTest.java @@ -303,7 +303,7 @@ public void shouldMapStateStoresToCorrectSourceTopics() { assertEquals(Collections.singletonList("table-topic"), builder.stateStoreNameToSourceTopics().get("table-store")); final KStream mapped = playEvents.map(MockMapper.selectValueKeyValueMapper()); - mapped.leftJoin(table, MockValueJoiner.TOSTRING_JOINER).groupByKey().count(Materialized.>as("my-table")); + mapped.leftJoin(table, MockValueJoiner.TOSTRING_JOINER).groupByKey().count(Materialized.>as("count")); assertEquals(Collections.singletonList("table-topic"), builder.stateStoreNameToSourceTopics().get("table-store")); assertEquals(Collections.singletonList(APP_ID + "-KSTREAM-MAP-0000000003-repartition"), builder.stateStoreNameToSourceTopics().get("count")); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java index 2a740c1a21a1f..e87990ccc4c14 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java @@ -38,7 +38,6 @@ import org.apache.kafka.streams.kstream.TimeWindows; import org.apache.kafka.streams.kstream.Windowed; import org.apache.kafka.streams.kstream.Windows; -import org.apache.kafka.streams.processor.StateStoreSupplier; import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.SessionStore; @@ -86,19 +85,14 @@ public void shouldNotHaveNullReducerOnReduce() { groupedStream.reduce(null); } - @Test - public void shouldAllowNullStoreNameOnReduce() { - groupedStream.reduce(MockReducer.STRING_ADDER, Materialized.>as(null)); - } - @Test(expected = InvalidTopicException.class) public void shouldNotHaveInvalidStoreNameOnReduce() { - groupedStream.reduce(MockReducer.STRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); + groupedStream.reduce(MockReducer.STRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); } @Test(expected = NullPointerException.class) public void shouldNotHaveNullReducerWithWindowedReduce() { - groupedStream.windowedBy(TimeWindows.of(10)).reduce(null, Materialized.>as("store")); + groupedStream.windowedBy(TimeWindows.of(10)).reduce(null, Materialized.>as("store")); } @Test(expected = NullPointerException.class) @@ -106,44 +100,34 @@ public void shouldNotHaveNullWindowsWithWindowedReduce() { groupedStream.windowedBy((Windows) null); } - @Test - public void shouldAllowNullStoreNameWithWindowedReduce() { - groupedStream.windowedBy(TimeWindows.of(10)).reduce(MockReducer.STRING_ADDER, Materialized.>as(null)); - } - @Test(expected = InvalidTopicException.class) public void shouldNotHaveInvalidStoreNameWithWindowedReduce() { - groupedStream.windowedBy(TimeWindows.of(10)).reduce(MockReducer.STRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); + groupedStream.windowedBy(TimeWindows.of(10)).reduce(MockReducer.STRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); } @Test(expected = NullPointerException.class) public void shouldNotHaveNullInitializerOnAggregate() { - groupedStream.aggregate(null, MockAggregator.TOSTRING_ADDER, Materialized.>as("store")); + groupedStream.aggregate(null, MockAggregator.TOSTRING_ADDER, Materialized.>as("store")); } @Test(expected = NullPointerException.class) public void shouldNotHaveNullAdderOnAggregate() { - groupedStream.aggregate(MockInitializer.STRING_INIT, null, Materialized.>as("store")); - } - - @Test - public void shouldAllowNullStoreNameOnAggregate() { - groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as(null)); + groupedStream.aggregate(MockInitializer.STRING_INIT, null, Materialized.>as("store")); } @Test(expected = InvalidTopicException.class) public void shouldNotHaveInvalidStoreNameOnAggregate() { - groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); + groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); } @Test(expected = NullPointerException.class) public void shouldNotHaveNullInitializerOnWindowedAggregate() { - groupedStream.windowedBy(TimeWindows.of(10)).aggregate(null, MockAggregator.TOSTRING_ADDER, Materialized.>as("store")); + groupedStream.windowedBy(TimeWindows.of(10)).aggregate(null, MockAggregator.TOSTRING_ADDER, Materialized.>as("store")); } @Test(expected = NullPointerException.class) public void shouldNotHaveNullAdderOnWindowedAggregate() { - groupedStream.windowedBy(TimeWindows.of(10)).aggregate(MockInitializer.STRING_INIT, null, Materialized.>as("store")); + groupedStream.windowedBy(TimeWindows.of(10)).aggregate(MockInitializer.STRING_INIT, null, Materialized.>as("store")); } @Test(expected = NullPointerException.class) @@ -151,14 +135,9 @@ public void shouldNotHaveNullWindowsOnWindowedAggregate() { groupedStream.windowedBy((Windows) null); } - @Test - public void shouldAllowNullStoreNameOnWindowedAggregate() { - groupedStream.windowedBy(TimeWindows.of(10)).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as(null)); - } - @Test(expected = InvalidTopicException.class) public void shouldNotHaveInvalidStoreNameOnWindowedAggregate() { - groupedStream.windowedBy(TimeWindows.of(10)).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); + groupedStream.windowedBy(TimeWindows.of(10)).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); } private void doAggregateSessionWindows(final Map, Integer> results) { @@ -199,7 +178,7 @@ public Integer apply(final String aggKey, final String value, final Integer aggr public Integer apply(final String aggKey, final Integer aggOne, final Integer aggTwo) { return aggOne + aggTwo; } - }, Materialized.>as("session-store").withValueSerde(Serdes.Integer())); + }, Materialized.>as("session-store").withValueSerde(Serdes.Integer())); table.toStream().foreach(new ForeachAction, Integer>() { @Override public void apply(final Windowed key, final Integer value) { @@ -229,7 +208,7 @@ public Integer apply(final String aggKey, final String value, final Integer aggr public Integer apply(final String aggKey, final Integer aggOne, final Integer aggTwo) { return aggOne + aggTwo; } - }, Materialized.>with(null, Serdes.Integer())); + }, Materialized.>with(null, Serdes.Integer())); table.toStream().foreach(new ForeachAction, Integer>() { @Override public void apply(final Windowed key, final Integer value) { @@ -314,11 +293,11 @@ public void shouldReduceSessionWindows() { final Map, String> results = new HashMap<>(); final KTable, String> table = groupedStream.windowedBy(SessionWindows.with(30)) .reduce(new Reducer() { - @Override - public String apply(final String value1, final String value2) { - return value1 + ":" + value2; - } - }, Materialized.>as("session-store")); + @Override + public String apply(final String value1, final String value2) { + return value1 + ":" + value2; + } + }, Materialized.>as("session-store").withValueSerde(Serdes.String())); table.toStream().foreach(new ForeachAction, String>() { @Override public void apply(final Windowed key, final String value) { @@ -334,11 +313,11 @@ public void shouldReduceSessionWindowsWithInternalStoreName() { final Map, String> results = new HashMap<>(); final KTable, String> table = groupedStream.windowedBy(SessionWindows.with(30)) .reduce(new Reducer() { - @Override - public String apply(final String value1, final String value2) { - return value1 + ":" + value2; - } - }); + @Override + public String apply(final String value1, final String value2) { + return value1 + ":" + value2; + } + }, Materialized.>with(Serdes.String(), Serdes.String())); table.toStream().foreach(new ForeachAction, String>() { @Override public void apply(final Windowed key, final String value) { @@ -351,7 +330,7 @@ public void apply(final Windowed key, final String value) { @Test(expected = NullPointerException.class) public void shouldNotAcceptNullReducerWhenReducingSessionWindows() { - groupedStream.windowedBy(SessionWindows.with(30)).reduce(null, Materialized.>as("store")); + groupedStream.windowedBy(SessionWindows.with(30)).reduce(null, Materialized.>as("store")); } @Test(expected = NullPointerException.class) @@ -361,12 +340,12 @@ public void shouldNotAcceptNullSessionWindowsReducingSessionWindows() { @Test(expected = InvalidTopicException.class) public void shouldNotAcceptInvalidStoreNameWhenReducingSessionWindows() { - groupedStream.windowedBy(SessionWindows.with(30)).reduce(MockReducer.STRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); + groupedStream.windowedBy(SessionWindows.with(30)).reduce(MockReducer.STRING_ADDER, Materialized.>as(INVALID_STORE_NAME)); } @Test(expected = NullPointerException.class) public void shouldNotAcceptNullStateStoreSupplierWhenReducingSessionWindows() { - groupedStream.windowedBy(SessionWindows.with(30)).reduce(null, Materialized.>as(null)); + groupedStream.windowedBy(SessionWindows.with(30)).reduce(null, Materialized.>as(null)); } @Test(expected = NullPointerException.class) @@ -376,7 +355,7 @@ public void shouldNotAcceptNullInitializerWhenAggregatingSessionWindows() { public String apply(final String aggKey, final String aggOne, final String aggTwo) { return null; } - }, Materialized.>as("storeName")); + }, Materialized.>as("storeName")); } @Test(expected = NullPointerException.class) @@ -386,14 +365,14 @@ public void shouldNotAcceptNullAggregatorWhenAggregatingSessionWindows() { public String apply(final String aggKey, final String aggOne, final String aggTwo) { return null; } - }, Materialized.>as("storeName")); + }, Materialized.>as("storeName")); } @Test(expected = NullPointerException.class) public void shouldNotAcceptNullSessionMergerWhenAggregatingSessionWindows() { groupedStream.windowedBy(SessionWindows.with(30)).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, null, - Materialized.>as("storeName")); + Materialized.>as("storeName")); } @Test(expected = NullPointerException.class) @@ -586,7 +565,7 @@ private void doCountWindowed(final List, Long>> result @Test public void shouldCountWindowed() { final List, Long>> results = new ArrayList<>(); - groupedStream.windowedBy(TimeWindows.of(500L)).count(Materialized.>as("aggregate-by-key-windowed")) + groupedStream.windowedBy(TimeWindows.of(500L)).count(Materialized.>as("aggregate-by-key-windowed")) .toStream() .foreach(new ForeachAction, Long>() { @Override diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java index a906677d3184a..a32058c829ee3 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java @@ -65,52 +65,52 @@ public void before() { @Test public void shouldAllowNullStoreNameOnCount() { - groupedTable.count(Materialized.>as(null)); + groupedTable.count(Materialized.>as(null)); } @Test public void shouldAllowNullStoreNameOnAggregate() { - groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as(null)); + groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as(null)); } @Test(expected = InvalidTopicException.class) public void shouldNotAllowInvalidStoreNameOnAggregate() { - groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as(INVALID_STORE_NAME)); + groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as(INVALID_STORE_NAME)); } @Test(expected = NullPointerException.class) public void shouldNotAllowNullInitializerOnAggregate() { - groupedTable.aggregate(null, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as("store")); + groupedTable.aggregate(null, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as("store")); } @Test(expected = NullPointerException.class) public void shouldNotAllowNullAdderOnAggregate() { - groupedTable.aggregate(MockInitializer.STRING_INIT, null, MockAggregator.TOSTRING_REMOVER, Materialized.>as("store")); + groupedTable.aggregate(MockInitializer.STRING_INIT, null, MockAggregator.TOSTRING_REMOVER, Materialized.>as("store")); } @Test(expected = NullPointerException.class) public void shouldNotAllowNullSubtractorOnAggregate() { - groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, null, Materialized.>as("store")); + groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, null, Materialized.>as("store")); } @Test(expected = NullPointerException.class) public void shouldNotAllowNullAdderOnReduce() { - groupedTable.reduce(null, MockReducer.STRING_REMOVER, Materialized.>as("store")); + groupedTable.reduce(null, MockReducer.STRING_REMOVER, Materialized.>as("store")); } @Test(expected = NullPointerException.class) public void shouldNotAllowNullSubtractorOnReduce() { - groupedTable.reduce(MockReducer.STRING_ADDER, null, Materialized.>as("store")); + groupedTable.reduce(MockReducer.STRING_ADDER, null, Materialized.>as("store")); } @Test public void shouldAllowNullStoreNameOnReduce() { - groupedTable.reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as(null)); + groupedTable.reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as(null)); } @Test(expected = InvalidTopicException.class) public void shouldNotAllowInvalidStoreNameOnReduce() { - groupedTable.reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as(INVALID_STORE_NAME)); + groupedTable.reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as(INVALID_STORE_NAME)); } private void doShouldReduce(final KTable reduced, final String topic) { @@ -157,7 +157,7 @@ public KeyValue apply(String key, Number value) { .withKeySerde(Serdes.String()) .withValueSerde(Serdes.Double())) .groupBy(intProjection) - .reduce(MockReducer.INTEGER_ADDER, MockReducer.INTEGER_SUBTRACTOR, Materialized.>as("reduced")); + .reduce(MockReducer.INTEGER_ADDER, MockReducer.INTEGER_SUBTRACTOR, Materialized.>as("reduced")); doShouldReduce(reduced, topic); assertEquals(reduced.queryableStoreName(), "reduced"); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java index 98d10010712cd..f25607259697a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamImplTest.java @@ -23,7 +23,6 @@ import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsBuilderTest; -import org.apache.kafka.streams.errors.TopologyException; import org.apache.kafka.streams.kstream.GlobalKTable; import org.apache.kafka.streams.kstream.JoinWindows; import org.apache.kafka.streams.kstream.Joined; diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java index 424b077889ed9..2a4c776a0da1c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java @@ -70,7 +70,7 @@ public void testAggBasic() { .stream(topic1, Consumed.with(strSerde, strSerde)) .groupByKey(Serialized.with(strSerde, strSerde)) .windowedBy(TimeWindows.of(10).advanceBy(5)) - .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic1-Canonized")); + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic1-Canonized")); final MockProcessorSupplier, String> proc2 = new MockProcessorSupplier<>(); table2.toStream().process(proc2); @@ -164,7 +164,7 @@ public void testJoin() { .stream(topic1, Consumed.with(strSerde, strSerde)) .groupByKey(Serialized.with(strSerde, strSerde)) .windowedBy(TimeWindows.of(10).advanceBy(5)) - .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic1-Canonized")); + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic1-Canonized")); final MockProcessorSupplier, String> proc1 = new MockProcessorSupplier<>(); table1.toStream().process(proc1); @@ -173,7 +173,7 @@ public void testJoin() { .stream(topic2, Consumed.with(strSerde, strSerde)) .groupByKey(Serialized.with(strSerde, strSerde)) .windowedBy(TimeWindows.of(10).advanceBy(5)) - .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic2-Canonized")); + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic2-Canonized")); final MockProcessorSupplier, String> proc2 = new MockProcessorSupplier<>(); table2.toStream().process(proc2); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java index 9308601e1411c..073beab7a6c68 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableAggregateTest.java @@ -81,7 +81,7 @@ public void testAggBasic() { ).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, - Materialized.>as("topic1-Canonized").withValueSerde(stringSerde)); + Materialized.>as("topic1-Canonized").withValueSerde(stringSerde)); table2.toStream().process(proc); @@ -128,7 +128,7 @@ public void testAggCoalesced() { ).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, - Materialized.>as("topic1-Canonized").withValueSerde(stringSerde)); + Materialized.>as("topic1-Canonized").withValueSerde(stringSerde)); table2.toStream().process(proc); @@ -168,7 +168,7 @@ public KeyValue apply(String key, String value) { .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, - Materialized.>as("topic1-Canonized").withValueSerde(stringSerde)); + Materialized.>as("topic1-Canonized").withValueSerde(stringSerde)); table2.toStream().process(proc); @@ -236,7 +236,7 @@ public void testCount() { builder.table(input, consumed) .groupBy(MockMapper.selectValueKeyValueMapper(), stringSerialzied) - .count(Materialized.>as("count")) + .count(Materialized.>as("count")) .toStream() .process(proc); @@ -266,7 +266,7 @@ public void testCountCoalesced() { builder.table(input, consumed) .groupBy(MockMapper.selectValueKeyValueMapper(), stringSerialzied) - .count(Materialized.>as("count")) + .count(Materialized.>as("count")) .toStream() .process(proc); @@ -319,7 +319,7 @@ public String apply(String aggKey, String value, String aggregate) { public String apply(String key, String value, String aggregate) { return aggregate.replaceAll(value, ""); } - }, Materialized.>as("someStore").withValueSerde(Serdes.String())) + }, Materialized.>as("someStore").withValueSerde(Serdes.String())) .toStream() .process(proc); @@ -370,7 +370,7 @@ public Long apply(final Long value1, final Long value2) { public Long apply(final Long value1, final Long value2) { return value1 - value2; } - }, Materialized.>as("reducer-store")); + }, Materialized.>as("reducer-store")); reduce.toStream().foreach(new ForeachAction() { @Override diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java index 3c4ad0f6afaf6..4963be37f7553 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java @@ -253,7 +253,7 @@ public void testQueryableValueGetter() { public boolean test(String key, Integer value) { return (value % 2) == 0; } - }, "anyStoreNameFilter"); + }, Materialized.>as("anyStoreNameFilter")); KTableImpl table3 = (KTableImpl) table1.filterNot( new Predicate() { @Override @@ -337,7 +337,7 @@ public void testQueryableNotSendingOldValue() { public boolean test(String key, Integer value) { return (value % 2) == 0; } - }, "anyStoreNameFilter"); + }, Materialized.>as("anyStoreNameFilter")); doTestNotSendingOldValue(builder, table1, table2, topic1); } @@ -415,7 +415,7 @@ public void testQueryableSendingOldValue() { public boolean test(String key, Integer value) { return (value % 2) == 0; } - }, "anyStoreNameFilter"); + }, Materialized.>as("anyStoreNameFilter")); doTestSendingOldValue(builder, table1, table2, topic1); } @@ -457,7 +457,7 @@ public boolean test(String key, String value) { return value.equalsIgnoreCase("accept"); } }).groupBy(MockMapper.noOpKeyValueMapper()) - .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as("mock-result")); + .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as("mock-result")); doTestSkipNullOnMaterialization(builder, table1, table2, topic1); } @@ -478,8 +478,8 @@ public void testQueryableSkipNullOnMaterialization() { public boolean test(String key, String value) { return value.equalsIgnoreCase("accept"); } - }, "anyStoreNameFilter").groupBy(MockMapper.noOpKeyValueMapper()) - .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as("mock-result")); + }, Materialized.>as("anyStoreNameFilter")).groupBy(MockMapper.noOpKeyValueMapper()) + .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as("mock-result")); doTestSkipNullOnMaterialization(builder, table1, table2, topic1); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java index aebb6d17d9871..5e7cef2c2785c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableImplTest.java @@ -29,7 +29,6 @@ import org.apache.kafka.streams.kstream.ValueJoiner; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; -import org.apache.kafka.streams.processor.StateStoreSupplier; import org.apache.kafka.streams.processor.internals.SinkNode; import org.apache.kafka.streams.processor.internals.SourceNode; import org.apache.kafka.streams.state.KeyValueStore; @@ -343,11 +342,11 @@ public void testRepartition() throws NoSuchFieldException, IllegalAccessExceptio ); table1.groupBy(MockMapper.noOpKeyValueMapper()) - .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as("mock-result1")); + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as("mock-result1")); table1.groupBy(MockMapper.noOpKeyValueMapper()) - .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as("mock-result2")); + .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as("mock-result2")); driver.setUp(builder, stateDir, stringSerde, stringSerde); driver.setTime(0L); @@ -477,7 +476,7 @@ public void shouldThrowNullPointerOnFilterWhenMaterializedIsNull() { public boolean test(final String key, final String value) { return false; } - }, (Materialized>) null); + }, (Materialized) null); } @Test(expected = NullPointerException.class) @@ -487,7 +486,7 @@ public void shouldThrowNullPointerOnFilterNotWhenMaterializedIsNull() { public boolean test(final String key, final String value) { return false; } - }, (Materialized>) null); + }, (Materialized) null); } @Test(expected = NullPointerException.class) diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java index 9b4c2722e4a4c..dce320168735e 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableKTableLeftJoinTest.java @@ -361,7 +361,7 @@ public KeyValue apply(final Long key, final String value) { }, Serialized.with(Serdes.Long(), Serdes.String()) ) - .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_ADDER, Materialized.>as("agg-store")); + .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_ADDER, Materialized.>as("agg-store")); final KTable one = builder.table(tableOne, consumed); final KTable two = builder.table(tableTwo, consumed); diff --git a/streams/src/test/java/org/apache/kafka/streams/perf/YahooBenchmark.java b/streams/src/test/java/org/apache/kafka/streams/perf/YahooBenchmark.java index 294bb2628d0bc..2f3a0067ac53f 100644 --- a/streams/src/test/java/org/apache/kafka/streams/perf/YahooBenchmark.java +++ b/streams/src/test/java/org/apache/kafka/streams/perf/YahooBenchmark.java @@ -341,7 +341,7 @@ public String apply(String key, String value) { keyedByCampaign .groupByKey(Serialized.with(Serdes.String(), Serdes.String())) .windowedBy(TimeWindows.of(10 * 1000)) - .count(Materialized.>as("time-windows")); + .count(Materialized.>as("time-windows")); return new KafkaStreams(builder.build(), streamConfig); } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetadataStateTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetadataStateTest.java index a123fcf0e4c99..e9bb2a396ea3d 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetadataStateTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetadataStateTest.java @@ -74,16 +74,16 @@ public class StreamsMetadataStateTest { public void before() { builder = new StreamsBuilder(); final KStream one = builder.stream("topic-one"); - one.groupByKey().count(Materialized.>as("table-one")); + one.groupByKey().count(Materialized.>as("table-one")); final KStream two = builder.stream("topic-two"); - two.groupByKey().count(Materialized.>as("table-two")); + two.groupByKey().count(Materialized.>as("table-two")); builder.stream("topic-three") .groupByKey() - .count(Materialized.>as("table-three")); + .count(Materialized.>as("table-three")); - one.merge(two).groupByKey().count(Materialized.>as("merged-table")); + one.merge(two).groupByKey().count(Materialized.>as("merged-table")); builder.stream("topic-four").mapValues(new ValueMapper() { @Override From 9592fc1e90a30a605e725e96a9d196f0f19903f0 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 24 Apr 2018 21:31:02 -0700 Subject: [PATCH 06/13] fix two critical bugs --- .../kstream/internals/AbstractStream.java | 6 -- .../GroupedStreamAggregateBuilder.java | 8 ++ .../kstream/internals/KGroupedStreamImpl.java | 8 +- .../kstream/internals/KGroupedTableImpl.java | 35 ++++++--- .../streams/kstream/internals/KTableImpl.java | 74 +++++++++++++------ .../internals/SessionWindowedKStreamImpl.java | 46 ++++++++---- .../internals/TimeWindowedKStreamImpl.java | 74 ++++++------------- .../PurgeRepartitionTopicIntegrationTest.java | 2 + .../internals/KGroupedTableImplTest.java | 15 ---- .../internals/KStreamWindowAggregateTest.java | 6 +- .../SessionWindowedKStreamImplTest.java | 5 +- 11 files changed, 152 insertions(+), 127 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/AbstractStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/AbstractStream.java index 7410a0a06eef0..3c6539902e8e8 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/AbstractStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/AbstractStream.java @@ -72,12 +72,6 @@ Set ensureJoinableWith(final AbstractStream other) { return allSourceNodes; } - String getOrCreateName(final String queryableStoreName, final String prefix) { - final String returnName = queryableStoreName != null ? queryableStoreName : builder.newStoreName(prefix); - Topic.validate(returnName); - return returnName; - } - static ValueJoiner reverseJoiner(final ValueJoiner joiner) { return new ValueJoiner() { @Override diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java index 24ed8a0c49f38..715c29115f195 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/GroupedStreamAggregateBuilder.java @@ -39,6 +39,7 @@ public Long apply() { return 0L; } }; + final Aggregator countAggregator = new Aggregator() { @Override public Long apply(K aggKey, V value, Long aggregate) { @@ -46,6 +47,13 @@ public Long apply(K aggKey, V value, Long aggregate) { } }; + final Initializer reduceInitializer = new Initializer() { + @Override + public V apply() { + return null; + } + }; + GroupedStreamAggregateBuilder(final InternalStreamsBuilder builder, final Serde keySerde, final Serde valueSerde, diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java index 1a4478c57fa25..ed0823270fd93 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java @@ -75,7 +75,7 @@ public KTable reduce(final Reducer reducer, Objects.requireNonNull(reducer, "reducer can't be null"); Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal - = new MaterializedInternal<>(materialized, builder, REDUCE_NAME); + = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); if (materializedInternal.keySerde() == null) { materializedInternal.withKeySerde(keySerde); } @@ -131,7 +131,11 @@ public KTable count(final Materialized(materializedInternal.storeName(), aggregateBuilder.countInitializer, aggregateBuilder.countAggregator), + AGGREGATE_NAME, + materializedInternal); } @Override diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java index 5d8a801cbf985..df50ef624d6dc 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.kstream.Aggregator; @@ -128,7 +129,13 @@ public KTable reduce(final Reducer adder, Objects.requireNonNull(subtractor, "subtractor can't be null"); Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal - = new MaterializedInternal<>(materialized, builder, REDUCE_NAME); + = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); + if (materializedInternal.keySerde() == null) { + materializedInternal.withKeySerde(keySerde); + } + if (materializedInternal.valueSerde() == null) { + materializedInternal.withValueSerde(valSerde); + } final ProcessorSupplier> aggregateSupplier = new KTableReduce<>(materializedInternal.storeName(), adder, subtractor); @@ -138,23 +145,33 @@ public KTable reduce(final Reducer adder, @Override public KTable reduce(final Reducer adder, final Reducer subtractor) { - return reduce(adder, subtractor, Materialized.>as(getOrCreateName(null, AGGREGATE_NAME))); + return reduce(adder, subtractor, Materialized.>with(keySerde, valSerde)); } @Override public KTable count(final Materialized> materialized) { - return aggregate(countInitializer, - countAdder, - countSubtractor, - materialized); + final MaterializedInternal> materializedInternal + = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); + if (materializedInternal.keySerde() == null) { + materialized.withKeySerde(keySerde); + } + if (materializedInternal.valueSerde() == null) { + materialized.withValueSerde(Serdes.Long()); + } + + final ProcessorSupplier> aggregateSupplier = new KTableAggregate<>(materializedInternal.storeName(), + countInitializer, + countAdder, + countSubtractor); + + return doAggregate(aggregateSupplier, AGGREGATE_NAME, materializedInternal); } @Override public KTable count() { - return count(Materialized.>as(getOrCreateName(null, AGGREGATE_NAME))); + return count(Materialized.>with(keySerde, Serdes.Long())); } - @Override public KTable aggregate(final Initializer initializer, final Aggregator adder, @@ -181,7 +198,7 @@ public KTable aggregate(final Initializer initializer, public KTable aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor) { - return aggregate(initializer, adder, subtractor, Materialized.>as(getOrCreateName(null, AGGREGATE_NAME))); + return aggregate(initializer, adder, subtractor, Materialized.>with(keySerde, null)); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java index b059670455b2c..ac94565384347 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java @@ -133,37 +133,49 @@ public String apply(K key, V value) { public String queryableStoreName() { if (!isQueryable) { return null; + } else { + return this.queryableStoreName; } - return this.queryableStoreName; } private KTable doFilter(final Predicate predicate, final MaterializedInternal> materialized, final boolean filterNot) { - String name = builder.newProcessorName(FILTER_NAME); + final String name = builder.newProcessorName(FILTER_NAME); - KTableProcessorSupplier processorSupplier = new KTableFilter<>(this, - predicate, - filterNot, - materialized.storeName()); - builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); + // only materialize if the state store is queryable + KTableProcessorSupplier processorSupplier; - final StoreBuilder builder = new KeyValueStoreMaterializer<>(materialized).materialize(); - this.builder.internalTopologyBuilder.addStateStore(builder, name); + if (materialized.isQueryable()) { + processorSupplier = new KTableFilter<>(this, + predicate, + filterNot, + materialized.storeName()); + + builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); + this.builder.internalTopologyBuilder.addStateStore(new KeyValueStoreMaterializer<>(materialized).materialize(), name); + } else { + processorSupplier = new KTableFilter<>(this, + predicate, + filterNot, + null); - return new KTableImpl<>(this.builder, + builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); + } + + return new KTableImpl<>(builder, name, processorSupplier, this.keySerde, this.valSerde, sourceNodes, - builder.name(), + materialized.storeName(), materialized.isQueryable()); } @Override public KTable filter(final Predicate predicate) { - return filter(predicate, Materialized.>with(null, null)); + return filter(predicate, Materialized.>with(keySerde, valSerde)); } @Override @@ -176,7 +188,7 @@ public KTable filter(final Predicate predicate, @Override public KTable filterNot(final Predicate predicate) { - return filterNot(predicate, Materialized.>with(null, null)); + return filterNot(predicate, Materialized.>with(keySerde, valSerde)); } @Override @@ -189,12 +201,12 @@ public KTable filterNot(final Predicate predicate, @Override public KTable mapValues(final ValueMapper mapper) { - return mapValues(withKey(mapper), Materialized.>with(null, null)); + return mapValues(withKey(mapper), Materialized.>with(keySerde, null)); } @Override public KTable mapValues(final ValueMapperWithKey mapper) { - return mapValues(mapper, Materialized.>with(null, null)); + return mapValues(mapper, Materialized.>with(keySerde, null)); } @@ -209,18 +221,31 @@ public KTable mapValues(final ValueMapperWithKey> materialized) { Objects.requireNonNull(mapper, "mapper can't be null"); Objects.requireNonNull(materialized, "materialized can't be null"); + final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized, builder, MAPVALUES_NAME); final String name = builder.newProcessorName(MAPVALUES_NAME); - final KTableProcessorSupplier processorSupplier = new KTableMapValues<>( - this, - mapper, - materializedInternal.storeName()); - builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); - builder.internalTopologyBuilder.addStateStore( - new KeyValueStoreMaterializer<>(materializedInternal).materialize(), - name); - return new KTableImpl<>(builder, name, processorSupplier, sourceNodes, this.queryableStoreName, true); + + // only materialize if the state store is queryable + KTableProcessorSupplier processorSupplier; + + if (materializedInternal.isQueryable()) { + processorSupplier = new KTableMapValues<>( + this, + mapper, + materializedInternal.storeName()); + + builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); + this.builder.internalTopologyBuilder.addStateStore(new KeyValueStoreMaterializer<>(materializedInternal).materialize(), name); + } else { + processorSupplier = new KTableMapValues<>( + this, + mapper, + null); + builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); + } + + return new KTableImpl<>(builder, name, processorSupplier, sourceNodes, materializedInternal.storeName(), materializedInternal.isQueryable()); } @SuppressWarnings("deprecation") @@ -538,6 +563,7 @@ private KTable doJoin(final KTable other, joinMergeName, internalQueryableName); + // only materialize if specified in Materialized if (materialized != null) { final StoreBuilder> storeBuilder = new KeyValueStoreMaterializer<>(materialized).materialize(); diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java index bcbe71da1f3ea..c3b295b0cfb01 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java @@ -37,6 +37,7 @@ import java.util.Set; import static org.apache.kafka.streams.kstream.internals.KGroupedStreamImpl.AGGREGATE_NAME; +import static org.apache.kafka.streams.kstream.internals.KGroupedStreamImpl.REDUCE_NAME; public class SessionWindowedKStreamImpl extends AbstractStream implements SessionWindowedKStream { private final SessionWindows windows; @@ -49,13 +50,6 @@ public Long apply(final K aggKey, final Long aggOne, final Long aggTwo) { return aggOne + aggTwo; } }; - private final Initializer reduceInitializer = new Initializer() { - @Override - public V apply() { - return null; - } - }; - SessionWindowedKStreamImpl(final SessionWindows windows, final InternalStreamsBuilder builder, @@ -74,42 +68,62 @@ public V apply() { @Override public KTable, Long> count() { - return count(Materialized.>with(null, Serdes.Long())); + return count(Materialized.>with(keySerde, Serdes.Long())); } + @SuppressWarnings("unchecked") @Override public KTable, Long> count(final Materialized> materialized) { Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); + if (materializedInternal.keySerde() == null) { + materializedInternal.withKeySerde(keySerde); + } if (materializedInternal.valueSerde() == null) { - materialized.withValueSerde(Serdes.Long()); + materializedInternal.withValueSerde(Serdes.Long()); } - return aggregate(aggregateBuilder.countInitializer, - aggregateBuilder.countAggregator, - countMerger, - materialized); + + return (KTable, Long>) aggregateBuilder.build( + new KStreamSessionWindowAggregate<>(windows, materializedInternal.storeName(), aggregateBuilder.countInitializer, aggregateBuilder.countAggregator, countMerger), + AGGREGATE_NAME, + materialize(materializedInternal), + materializedInternal.isQueryable()); } @Override public KTable, V> reduce(final Reducer reducer) { - return reduce(reducer, Materialized.>with(null, null)); + return reduce(reducer, Materialized.>with(keySerde, valSerde)); } + @SuppressWarnings("unchecked") @Override public KTable, V> reduce(final Reducer reducer, final Materialized> materialized) { Objects.requireNonNull(reducer, "reducer can't be null"); Objects.requireNonNull(materialized, "materialized can't be null"); final Aggregator reduceAggregator = aggregatorForReducer(reducer); - return aggregate(reduceInitializer, reduceAggregator, mergerForAggregator(reduceAggregator), materialized); + final MaterializedInternal> materializedInternal + = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); + if (materializedInternal.keySerde() == null) { + materializedInternal.withKeySerde(keySerde); + } + if (materializedInternal.valueSerde() == null) { + materializedInternal.withValueSerde(valSerde); + } + + return (KTable, V>) aggregateBuilder.build( + new KStreamSessionWindowAggregate<>(windows, materializedInternal.storeName(), aggregateBuilder.reduceInitializer, reduceAggregator, mergerForAggregator(reduceAggregator)), + REDUCE_NAME, + materialize(materializedInternal), + materializedInternal.isQueryable()); } @Override public KTable, T> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger) { - return aggregate(initializer, aggregator, sessionMerger, Materialized.>with(null, null)); + return aggregate(initializer, aggregator, sessionMerger, Materialized.>with(keySerde, null)); } @SuppressWarnings("unchecked") diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImpl.java index 5e5477051e9a7..cee0192a08fab 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImpl.java @@ -63,42 +63,33 @@ public class TimeWindowedKStreamImpl extends AbstractStr @Override public KTable, Long> count() { - return doAggregate( - aggregateBuilder.countInitializer, - aggregateBuilder.countAggregator, - Serdes.Long()); + return count(Materialized.>with(keySerde, Serdes.Long())); } + @SuppressWarnings("unchecked") @Override public KTable, Long> count(final Materialized> materialized) { Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); + if (materializedInternal.keySerde() == null) { + materializedInternal.withKeySerde(keySerde); + } if (materializedInternal.valueSerde() == null) { - materialized.withValueSerde(Serdes.Long()); + materializedInternal.withValueSerde(Serdes.Long()); } - return aggregate(aggregateBuilder.countInitializer, aggregateBuilder.countAggregator, materialized); + + return (KTable, Long>) aggregateBuilder.build(new KStreamWindowAggregate<>(windows, materializedInternal.storeName(), aggregateBuilder.countInitializer, aggregateBuilder.countAggregator), + AGGREGATE_NAME, + materialize(materializedInternal), + materializedInternal.isQueryable()); } - @SuppressWarnings("unchecked") @Override public KTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator) { - Objects.requireNonNull(initializer, "initializer can't be null"); - Objects.requireNonNull(aggregator, "aggregator can't be null"); - return doAggregate(initializer, aggregator, (Serde) valSerde); - } - - @SuppressWarnings("unchecked") - private KTable, VR> doAggregate(final Initializer initializer, - final Aggregator aggregator, - final Serde serde) { - final String storeName = builder.newStoreName(AGGREGATE_NAME); - return (KTable, VR>) aggregateBuilder.build(new KStreamWindowAggregate<>(windows, storeName, initializer, aggregator), - AGGREGATE_NAME, - windowStoreBuilder(storeName, serde), - false); + return aggregate(initializer, aggregator, Materialized.>with(keySerde, null)); } @SuppressWarnings("unchecked") @@ -114,25 +105,15 @@ public KTable, VR> aggregate(final Initializer initializer, if (materializedInternal.keySerde() == null) { materializedInternal.withKeySerde(keySerde); } - return (KTable, VR>) aggregateBuilder.build(new KStreamWindowAggregate<>(windows, - materializedInternal.storeName(), - initializer, - aggregator), + return (KTable, VR>) aggregateBuilder.build(new KStreamWindowAggregate<>(windows, materializedInternal.storeName(), initializer, aggregator), AGGREGATE_NAME, materialize(materializedInternal), - true); + materializedInternal.isQueryable()); } - @SuppressWarnings("unchecked") @Override public KTable, V> reduce(final Reducer reducer) { - Objects.requireNonNull(reducer, "reducer can't be null"); - final String storeName = builder.newStoreName(REDUCE_NAME); - return (KTable, V>) aggregateBuilder.build(new KStreamWindowReduce(windows, storeName, reducer), - REDUCE_NAME, - windowStoreBuilder(storeName, valSerde), - true); - + return reduce(reducer, Materialized.>with(keySerde, valSerde)); } @SuppressWarnings("unchecked") @@ -140,13 +121,20 @@ public KTable, V> reduce(final Reducer reducer) { public KTable, V> reduce(final Reducer reducer, final Materialized> materialized) { Objects.requireNonNull(reducer, "reducer can't be null"); Objects.requireNonNull(materialized, "materialized can't be null"); + final MaterializedInternal> materializedInternal - = new MaterializedInternal<>(materialized, builder, REDUCE_NAME); + = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); + if (materializedInternal.keySerde() == null) { + materializedInternal.withKeySerde(keySerde); + } + if (materializedInternal.valueSerde() == null) { + materializedInternal.withValueSerde(valSerde); + } return (KTable, V>) aggregateBuilder.build(new KStreamWindowReduce(windows, materializedInternal.storeName(), reducer), REDUCE_NAME, materialize(materializedInternal), - false); + materializedInternal.isQueryable()); } private StoreBuilder> materialize(final MaterializedInternal> materialized) { @@ -173,18 +161,4 @@ private StoreBuilder> materialize(final MaterializedInte } return builder; } - - - private StoreBuilder> windowStoreBuilder(final String storeName, final Serde aggValueSerde) { - return Stores.windowStoreBuilder( - Stores.persistentWindowStore( - storeName, - windows.maintainMs(), - windows.segments, - windows.size(), - false), - keySerde, - aggValueSerde).withCachingEnabled(); - } - } diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/PurgeRepartitionTopicIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/PurgeRepartitionTopicIntegrationTest.java index 9dfb6dda37ec2..ac113ce64d6cf 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/PurgeRepartitionTopicIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/PurgeRepartitionTopicIntegrationTest.java @@ -166,6 +166,8 @@ public void setup() { .groupBy(MockMapper.selectKeyKeyValueMapper()) .count(); + System.out.print(builder.build().describe()); + kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration, time); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java index a32058c829ee3..ba4ee21d1f6ab 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImplTest.java @@ -63,16 +63,6 @@ public void before() { .groupBy(MockMapper.selectValueKeyValueMapper()); } - @Test - public void shouldAllowNullStoreNameOnCount() { - groupedTable.count(Materialized.>as(null)); - } - - @Test - public void shouldAllowNullStoreNameOnAggregate() { - groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as(null)); - } - @Test(expected = InvalidTopicException.class) public void shouldNotAllowInvalidStoreNameOnAggregate() { groupedTable.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.>as(INVALID_STORE_NAME)); @@ -103,11 +93,6 @@ public void shouldNotAllowNullSubtractorOnReduce() { groupedTable.reduce(MockReducer.STRING_ADDER, null, Materialized.>as("store")); } - @Test - public void shouldAllowNullStoreNameOnReduce() { - groupedTable.reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as(null)); - } - @Test(expected = InvalidTopicException.class) public void shouldNotAllowInvalidStoreNameOnReduce() { groupedTable.reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as(INVALID_STORE_NAME)); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java index 2a4c776a0da1c..a299298873113 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamWindowAggregateTest.java @@ -70,7 +70,7 @@ public void testAggBasic() { .stream(topic1, Consumed.with(strSerde, strSerde)) .groupByKey(Serialized.with(strSerde, strSerde)) .windowedBy(TimeWindows.of(10).advanceBy(5)) - .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic1-Canonized")); + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic1-Canonized").withValueSerde(strSerde)); final MockProcessorSupplier, String> proc2 = new MockProcessorSupplier<>(); table2.toStream().process(proc2); @@ -164,7 +164,7 @@ public void testJoin() { .stream(topic1, Consumed.with(strSerde, strSerde)) .groupByKey(Serialized.with(strSerde, strSerde)) .windowedBy(TimeWindows.of(10).advanceBy(5)) - .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic1-Canonized")); + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic1-Canonized").withValueSerde(strSerde)); final MockProcessorSupplier, String> proc1 = new MockProcessorSupplier<>(); table1.toStream().process(proc1); @@ -173,7 +173,7 @@ public void testJoin() { .stream(topic2, Consumed.with(strSerde, strSerde)) .groupByKey(Serialized.with(strSerde, strSerde)) .windowedBy(TimeWindows.of(10).advanceBy(5)) - .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic2-Canonized")); + .aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.>as("topic2-Canonized").withValueSerde(strSerde)); final MockProcessorSupplier, String> proc2 = new MockProcessorSupplier<>(); table2.toStream().process(proc2); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java index 619ee9667518f..dd09936d27393 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java @@ -92,7 +92,7 @@ public void apply(final Windowed key, final Long value) { @Test public void shouldReduceWindowed() { final Map, String> results = new HashMap<>(); - stream.reduce(MockReducer.STRING_ADDER) + stream.reduce(MockReducer.STRING_ADDER, Materialized.>with(Serdes.String(), Serdes.String())) .toStream() .foreach(new ForeachAction, String>() { @Override @@ -112,7 +112,8 @@ public void shouldAggregateSessionWindowed() { final Map, String> results = new HashMap<>(); stream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, - sessionMerger) + sessionMerger, + Materialized.>with(Serdes.String(), Serdes.String())) .toStream() .foreach(new ForeachAction, String>() { @Override From 628069b3dab572350e6ed4d2faf16f0c7dd3d639 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Wed, 25 Apr 2018 11:33:22 -0700 Subject: [PATCH 07/13] more fixes on unit tests --- .../kafka/streams/kstream/internals/KGroupedStreamImpl.java | 4 ++-- .../kafka/streams/kstream/internals/KGroupedTableImpl.java | 4 ++-- .../test/java/org/apache/kafka/test/KStreamTestDriver.java | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java index ed0823270fd93..35fda941e81a7 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java @@ -126,10 +126,10 @@ public KTable count(final Materialized> materializedInternal = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); if (materializedInternal.keySerde() == null) { - materialized.withKeySerde(keySerde); + materializedInternal.withKeySerde(keySerde); } if (materializedInternal.valueSerde() == null) { - materialized.withValueSerde(Serdes.Long()); + materializedInternal.withValueSerde(Serdes.Long()); } return doAggregate( diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java index df50ef624d6dc..db119f30fd57f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedTableImpl.java @@ -153,10 +153,10 @@ public KTable count(final Materialized> materializedInternal = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); if (materializedInternal.keySerde() == null) { - materialized.withKeySerde(keySerde); + materializedInternal.withKeySerde(keySerde); } if (materializedInternal.valueSerde() == null) { - materialized.withValueSerde(Serdes.Long()); + materializedInternal.withValueSerde(Serdes.Long()); } final ProcessorSupplier> aggregateSupplier = new KTableAggregate<>(materializedInternal.storeName(), diff --git a/streams/src/test/java/org/apache/kafka/test/KStreamTestDriver.java b/streams/src/test/java/org/apache/kafka/test/KStreamTestDriver.java index 7313414981bd3..5a3c402f096e5 100644 --- a/streams/src/test/java/org/apache/kafka/test/KStreamTestDriver.java +++ b/streams/src/test/java/org/apache/kafka/test/KStreamTestDriver.java @@ -122,6 +122,8 @@ public void setUp(final StreamsBuilder builder, final long cacheSize) { final InternalTopologyBuilder internalTopologyBuilder = StreamsBuilderTest.internalTopologyBuilder(builder); + System.out.println(builder.build().describe()); + internalTopologyBuilder.setApplicationId("TestDriver"); topology = internalTopologyBuilder.build(null); globalTopology = internalTopologyBuilder.buildGlobalStateTopology(); From a10e11cf6edf76054ebe3f0ad983cc75376c5110 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Wed, 25 Apr 2018 12:47:59 -0700 Subject: [PATCH 08/13] fixed all unit tests --- .../streams/kstream/internals/KTableImpl.java | 47 +++++++++++++------ .../TimeWindowedKStreamImplTest.java | 5 +- .../apache/kafka/test/KStreamTestDriver.java | 2 - 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java index ac94565384347..844786d107d9a 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java @@ -146,7 +146,7 @@ private KTable doFilter(final Predicate predicate, // only materialize if the state store is queryable KTableProcessorSupplier processorSupplier; - if (materialized.isQueryable()) { + if (materialized != null && materialized.isQueryable()) { processorSupplier = new KTableFilter<>(this, predicate, filterNot, @@ -154,6 +154,15 @@ private KTable doFilter(final Predicate predicate, builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); this.builder.internalTopologyBuilder.addStateStore(new KeyValueStoreMaterializer<>(materialized).materialize(), name); + + return new KTableImpl<>(builder, + name, + processorSupplier, + this.keySerde, + this.valSerde, + sourceNodes, + materialized.storeName(), + true); } else { processorSupplier = new KTableFilter<>(this, predicate, @@ -161,21 +170,22 @@ private KTable doFilter(final Predicate predicate, null); builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); - } - return new KTableImpl<>(builder, - name, - processorSupplier, - this.keySerde, - this.valSerde, - sourceNodes, - materialized.storeName(), - materialized.isQueryable()); + return new KTableImpl<>(builder, + name, + processorSupplier, + this.keySerde, + this.valSerde, + sourceNodes, + this.queryableStoreName, + false); + } } @Override public KTable filter(final Predicate predicate) { - return filter(predicate, Materialized.>with(keySerde, valSerde)); + Objects.requireNonNull(predicate, "predicate can't be null"); + return doFilter(predicate, null, false); } @Override @@ -188,7 +198,8 @@ public KTable filter(final Predicate predicate, @Override public KTable filterNot(final Predicate predicate) { - return filterNot(predicate, Materialized.>with(keySerde, valSerde)); + Objects.requireNonNull(predicate, "predicate can't be null"); + return doFilter(predicate, null, true); } @Override @@ -201,13 +212,21 @@ public KTable filterNot(final Predicate predicate, @Override public KTable mapValues(final ValueMapper mapper) { - return mapValues(withKey(mapper), Materialized.>with(keySerde, null)); + return mapValues(withKey(mapper)); } @Override public KTable mapValues(final ValueMapperWithKey mapper) { - return mapValues(mapper, Materialized.>with(keySerde, null)); + Objects.requireNonNull(mapper, "mapper can't be null"); + final String name = builder.newProcessorName(MAPVALUES_NAME); + + final KTableProcessorSupplier processorSupplier = new KTableMapValues<>( + this, + mapper, + null); + builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); + return new KTableImpl<>(builder, name, processorSupplier, sourceNodes, this.queryableStoreName, false); } @Override diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java index 286a82396e41d..c7a082ddb04f1 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java @@ -105,8 +105,9 @@ public void apply(final Windowed key, final String value) { public void shouldAggregateWindowed() { final Map, String> results = new HashMap<>(); windowedStream.aggregate(MockInitializer.STRING_INIT, - MockAggregator.TOSTRING_ADDER - ) + MockAggregator.TOSTRING_ADDER, + Materialized.>with(Serdes.String(), Serdes.String() + )) .toStream() .foreach(new ForeachAction, String>() { @Override diff --git a/streams/src/test/java/org/apache/kafka/test/KStreamTestDriver.java b/streams/src/test/java/org/apache/kafka/test/KStreamTestDriver.java index 5a3c402f096e5..7313414981bd3 100644 --- a/streams/src/test/java/org/apache/kafka/test/KStreamTestDriver.java +++ b/streams/src/test/java/org/apache/kafka/test/KStreamTestDriver.java @@ -122,8 +122,6 @@ public void setUp(final StreamsBuilder builder, final long cacheSize) { final InternalTopologyBuilder internalTopologyBuilder = StreamsBuilderTest.internalTopologyBuilder(builder); - System.out.println(builder.build().describe()); - internalTopologyBuilder.setApplicationId("TestDriver"); topology = internalTopologyBuilder.build(null); globalTopology = internalTopologyBuilder.buildGlobalStateTopology(); From 33fc44ba5abd5fdd8f88d9b86ed0378fa0904466 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Wed, 25 Apr 2018 13:41:35 -0700 Subject: [PATCH 09/13] a few minor cleanups --- .../KStreamAggregationIntegrationTest.java | 2 +- .../internals/KGroupedStreamImplTest.java | 4 +-- .../kstream/internals/KTableFilterTest.java | 33 ++++--------------- .../SessionWindowedKStreamImplTest.java | 26 ++------------- 4 files changed, 13 insertions(+), 52 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java index 11c2dbf0a980f..f5d002a3d8223 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationIntegrationTest.java @@ -663,7 +663,7 @@ public void shouldReduceSessionWindows() throws Exception { public String apply(final String value1, final String value2) { return value1 + ":" + value2; } - }, Materialized.>as(userSessionsStore).withValueSerde(Serdes.String())) + }, Materialized.>as(userSessionsStore)) .foreach(new ForeachAction, String>() { @Override public void apply(final Windowed key, final String value) { diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java index e87990ccc4c14..ba76c1d61c052 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImplTest.java @@ -297,7 +297,7 @@ public void shouldReduceSessionWindows() { public String apply(final String value1, final String value2) { return value1 + ":" + value2; } - }, Materialized.>as("session-store").withValueSerde(Serdes.String())); + }, Materialized.>as("session-store")); table.toStream().foreach(new ForeachAction, String>() { @Override public void apply(final Windowed key, final String value) { @@ -317,7 +317,7 @@ public void shouldReduceSessionWindowsWithInternalStoreName() { public String apply(final String value1, final String value2) { return value1 + ":" + value2; } - }, Materialized.>with(Serdes.String(), Serdes.String())); + }); table.toStream().foreach(new ForeachAction, String>() { @Override public void apply(final Windowed key, final String value) { diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java index 4963be37f7553..d8b6b1c20ca9f 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KTableFilterTest.java @@ -101,7 +101,6 @@ public boolean test(String key, Integer value) { doTestKTable(builder, table2, table3, topic1); } - @SuppressWarnings("deprecation") @Test public void testQueryableKTable() { final StreamsBuilder builder = new StreamsBuilder(); @@ -110,30 +109,6 @@ public void testQueryableKTable() { KTable table1 = builder.table(topic1, consumed); - KTable table2 = table1.filter(new Predicate() { - @Override - public boolean test(String key, Integer value) { - return (value % 2) == 0; - } - }); - KTable table3 = table1.filterNot(new Predicate() { - @Override - public boolean test(String key, Integer value) { - return (value % 2) == 0; - } - }); - - doTestKTable(builder, table2, table3, topic1); - } - - @Test - public void shouldAddQueryableStore() { - final StreamsBuilder builder = new StreamsBuilder(); - - final String topic1 = "topic1"; - - KTable table1 = builder.table(topic1, consumed); - KTable table2 = table1.filter(new Predicate() { @Override public boolean test(String key, Integer value) { @@ -147,6 +122,9 @@ public boolean test(String key, Integer value) { } }); + assertEquals("anyStoreNameFilter", table2.queryableStoreName()); + assertNull(table3.queryableStoreName()); + doTestKTable(builder, table2, table3, topic1); } @@ -262,6 +240,9 @@ public boolean test(String key, Integer value) { } }); + assertEquals("anyStoreNameFilter", table2.queryableStoreName()); + assertNull(table3.queryableStoreName()); + doTestValueGetter(builder, table2, table3, topic1); } @@ -457,7 +438,7 @@ public boolean test(String key, String value) { return value.equalsIgnoreCase("accept"); } }).groupBy(MockMapper.noOpKeyValueMapper()) - .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.>as("mock-result")); + .reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER); doTestSkipNullOnMaterialization(builder, table1, table2, topic1); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java index dd09936d27393..6a84c0368a5cb 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImplTest.java @@ -92,7 +92,7 @@ public void apply(final Windowed key, final Long value) { @Test public void shouldReduceWindowed() { final Map, String> results = new HashMap<>(); - stream.reduce(MockReducer.STRING_ADDER, Materialized.>with(Serdes.String(), Serdes.String())) + stream.reduce(MockReducer.STRING_ADDER) .toStream() .foreach(new ForeachAction, String>() { @Override @@ -130,21 +130,6 @@ public void apply(final Windowed key, final String value) { @SuppressWarnings("unchecked") @Test public void shouldMaterializeCount() { - stream.count(Materialized.>as("count-store") - .withKeySerde(Serdes.String())); - - processData(); - final SessionStore store = (SessionStore) driver.allStateStores().get("count-store"); - final List, Long>> data = StreamsTestUtils.toList(store.fetch("1", "2")); - assertThat(data, equalTo(Arrays.asList( - KeyValue.pair(new Windowed<>("1", new SessionWindow(10, 15)), 2L), - KeyValue.pair(new Windowed<>("1", new SessionWindow(600, 600)), 1L), - KeyValue.pair(new Windowed<>("2", new SessionWindow(600, 600)), 1L)))); - } - - @SuppressWarnings("unchecked") - @Test - public void shouldMaterializeWithoutSpecifyingSerdes() { stream.count(Materialized.>as("count-store")); processData(); @@ -159,10 +144,7 @@ public void shouldMaterializeWithoutSpecifyingSerdes() { @SuppressWarnings("unchecked") @Test public void shouldMaterializeReduced() { - stream.reduce(MockReducer.STRING_ADDER, - Materialized.>as("reduced") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String())); + stream.reduce(MockReducer.STRING_ADDER, Materialized.>as("reduced")); processData(); final SessionStore sessionStore = (SessionStore) driver.allStateStores().get("reduced"); @@ -180,9 +162,7 @@ public void shouldMaterializeAggregated() { stream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, sessionMerger, - Materialized.>as("aggregated") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String())); + Materialized.>as("aggregated").withValueSerde(Serdes.String())); processData(); final SessionStore sessionStore = (SessionStore) driver.allStateStores().get("aggregated"); From 0d5364cb9a49ab010c1588165803029ccc8be4ff Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 7 May 2018 10:19:39 -0700 Subject: [PATCH 10/13] fix findbugs --- .../streams/kstream/internals/TimeWindowedKStreamImplTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java index c7a082ddb04f1..6f9198de56d39 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImplTest.java @@ -106,7 +106,7 @@ public void shouldAggregateWindowed() { final Map, String> results = new HashMap<>(); windowedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, - Materialized.>with(Serdes.String(), Serdes.String() + Materialized.>with(Serdes.String(), Serdes.String() )) .toStream() .foreach(new ForeachAction, String>() { From 52d3f1fbb9d31eb6673326e09a1d17468dee7eaf Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 7 May 2018 15:40:34 -0700 Subject: [PATCH 11/13] github comments --- .../kafka/streams/kstream/KGroupedStream.java | 115 +++++++++++------- .../kafka/streams/kstream/KGroupedTable.java | 51 ++++---- .../apache/kafka/streams/kstream/KStream.java | 60 +++++---- .../apache/kafka/streams/kstream/KTable.java | 9 +- .../kstream/SessionWindowedKStream.java | 37 +++++- .../streams/kstream/TimeWindowedKStream.java | 47 ++++++- .../streams/kstream/internals/KTableImpl.java | 113 +++++++---------- .../apache/kafka/streams/TopologyWrapper.java | 34 ++++++ ...StreamAggregationDedupIntegrationTest.java | 2 +- .../PurgeRepartitionTopicIntegrationTest.java | 2 - 10 files changed, 300 insertions(+), 170 deletions(-) create mode 100644 streams/src/test/java/org/apache/kafka/streams/TopologyWrapper.java diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java index 355811b5fb702..53a2be79add13 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedStream.java @@ -21,6 +21,7 @@ import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.QueryableStoreType; @@ -61,7 +62,8 @@ public interface KGroupedStream { * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name * and "-changelog" is a fixed suffix. * Note that the internal store name may not be queriable through Interactive Queries. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. + * + * You can retrieve all generated internal topic names via {@link Topology#describe()}. * * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that * represent the latest (rolling) count (i.e., number of records) for each key @@ -72,7 +74,7 @@ public interface KGroupedStream { * Count the number of records in this stream by the grouped key. * Records with {@code null} key or value are ignored. * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view) - * provided by the given {@code materialized}. + * provided by the given store name in {@code materialized}. * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream. *

* Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to @@ -86,7 +88,7 @@ public interface KGroupedStream { * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}. * 

{@code
      * KafkaStreams streams = ... // counting words
-     * String queryableStoreName = "count-store"; // the queryableStoreName should be the name of the store as defined by the Materialized instance
+     * String queryableStoreName = "storeName"; // the store name should be the name of the store as defined by the Materialized instance
      * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
      * String key = "some-word";
      * Long countForWord = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
@@ -94,6 +96,17 @@ public interface KGroupedStream {
      * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
      * query the value of the key on a parallel running instance of your Kafka Streams application.
      *
+     * 

+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
      * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
      *                      Note: the valueSerde will be automatically set to {@link org.apache.kafka.common.serialization.Serdes#Long() Serdes#Long()}
      *                      if there is no valueSerde provided
@@ -108,9 +121,6 @@ public interface KGroupedStream {
      * Records with {@code null} key or value are ignored.
      * Combining implies that the type of the aggregate result is the same as the type of the input value
      * (c.f. {@link #aggregate(Initializer, Aggregator)}).
-     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
-     * that can be queried using the provided {@code queryableStoreName}.
-     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
      * 

      * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current
      * aggregate and the record's value.
@@ -124,13 +134,16 @@ public interface KGroupedStream {
      * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
      * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
      * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
+     *
      * 

      * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
      * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is
      * user-specified in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
      * and "-changelog" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     * Note that the internal store name may not be queriable through Interactive Queries.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param reducer   a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}.
      * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
@@ -145,7 +158,7 @@ public interface KGroupedStream {
      * Combining implies that the type of the aggregate result is the same as the type of the input value
      * (c.f. {@link #aggregate(Initializer, Aggregator, Materialized)}).
      * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
-     * provided by the given {@code materialized}.
+     * provided by the given store name in {@code materialized}.
      * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
      * 

      * The specified {@link Reducer} is applied for each input record and computes a new aggregate using the current
@@ -175,7 +188,7 @@ public interface KGroupedStream {
      * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}.
      * 
{@code
      * KafkaStreams streams = ... // compute sum
-     * String queryableStoreName = "storeName" // the queryableStoreName should be the name of the store as defined by the Materialized instance
+     * String queryableStoreName = "storeName" // the store name should be the name of the store as defined by the Materialized instance
      * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
      * String key = "some-key";
      * Long sumForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
@@ -183,6 +196,16 @@ public interface KGroupedStream {
      * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
      * query the value of the key on a parallel running instance of your Kafka Streams application.
      *
+     * 

+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
+     * and "-changelog" is a fixed suffix.
+     * Note that the internal store name may not be queriable through Interactive Queries.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
      * @param reducer       a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}.
      * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
      * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
@@ -191,62 +214,48 @@ public interface KGroupedStream {
     KTable reduce(final Reducer reducer,
                         final Materialized> materialized);
 
-
     /**
      * Aggregate the values of records in this stream by the grouped key.
      * Records with {@code null} key or value are ignored.
      * Aggregating is a generalization of {@link #reduce(Reducer) combining via reduce(...)} as it, for example,
      * allows the result to have a different type than the input values.
-     * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
-     * that can be queried using the provided {@code queryableStoreName}.
-     * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
      * 

      * The specified {@link Initializer} is applied once directly before the first input record is processed to
      * provide an initial intermediate aggregation result that is used to process the first record.
      * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current
      * aggregate (or for the very first record using the intermediate aggregation result provided via the
      * {@link Initializer}) and the record's value.
-     * Thus, {@code aggregate(Initializer, Aggregator, Serde, String)} can be used to compute aggregate functions like
+     * Thus, {@code aggregate(Initializer, Aggregator)} can be used to compute aggregate functions like
      * count (c.f. {@link #count()}).
      * 

+     * The default value serde from config will be used for serializing the result.
+     * If a different serde is required then you should use {@link #aggregate(Initializer, Aggregator, Materialized)}.
+     * 

      * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
      * the same key.
      * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
      * parallel running Kafka Streams instances, and the {@link StreamsConfig configuration} parameters for
      * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
      * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
-     * 

-     * To query the local {@link KeyValueStore} it must be obtained via
-     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
-     * 
{@code
-     * KafkaStreams streams = ... // some aggregation on value type double
-     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
-     * String key = "some-key";
-     * Long aggForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
-     * }

-     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
-     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     *
      * 

      * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
-     * Therefore, the store name must be a valid Kafka topic name and cannot contain characters other than ASCII
-     * alphanumerics, '.', '_' and '-'.
-     * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is
+     * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is
      * user-specified in {@link StreamsConfig} via parameter
-     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the
-     * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
+     * and "-changelog" is a fixed suffix.
+     * Note that the internal store name may not be queriable through Interactive Queries.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param initializer   an {@link Initializer} that computes an initial intermediate aggregation result
      * @param aggregator    an {@link Aggregator} that computes a new aggregate result
-     * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
      * @param           the value type of the resulting {@link KTable}
      * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
      * latest (rolling) aggregate for each key
      */
      KTable aggregate(final Initializer initializer,
-                                 final Aggregator aggregator,
-                                 final Materialized> materialized);
-
+                                 final Aggregator aggregator);
 
     /**
      * Aggregate the values of records in this stream by the grouped key.
@@ -254,7 +263,7 @@  KTable aggregate(final Initializer initializer,
      * Aggregating is a generalization of {@link #reduce(Reducer) combining via reduce(...)} as it, for example,
      * allows the result to have a different type than the input values.
      * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
-     * that can be queried using the provided {@code queryableStoreName}.
+     * that can be queried by the given store name in {@code materialized}.
      * Furthermore, updates to the store are sent downstream into a {@link KTable} changelog stream.
      * 

      * The specified {@link Initializer} is applied once directly before the first input record is processed to
@@ -262,12 +271,9 @@  KTable aggregate(final Initializer initializer,
      * The specified {@link Aggregator} is applied for each input record and computes a new aggregate using the current
      * aggregate (or for the very first record using the intermediate aggregation result provided via the
      * {@link Initializer}) and the record's value.
-     * Thus, {@code aggregate(Initializer, Aggregator)} can be used to compute aggregate functions like
+     * Thus, {@code aggregate(Initializer, Aggregator, Serde, String)} can be used to compute aggregate functions like
      * count (c.f. {@link #count()}).
      * 

-     * The default value serde from config will be used for serializing the result.
-     * If a different serde is required then you should use {@link #aggregate(Initializer, Aggregator, Materialized)}.
-     * 

      * Not all updates might get sent downstream, as an internal cache is used to deduplicate consecutive updates to
      * the same key.
      * The rate of propagated updates depends on your input data rate, the number of distinct keys, the number of
@@ -275,22 +281,39 @@  KTable aggregate(final Initializer initializer,
      * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG cache size}, and
      * {@link StreamsConfig#COMMIT_INTERVAL_MS_CONFIG commit intervall}.
      * 

+     * To query the local {@link KeyValueStore} it must be obtained via
+     * {@link KafkaStreams#store(String, QueryableStoreType) KafkaStreams#store(...)}:
+     * 
{@code
+     * KafkaStreams streams = ... // some aggregation on value type double
+     * String queryableStoreName = "storeName" // the store name should be the name of the store as defined by the Materialized instance
+     * ReadOnlyKeyValueStore localStore = streams.store(queryableStoreName, QueryableStoreTypes.keyValueStore());
+     * String key = "some-key";
+     * Long aggForKey = localStore.get(key); // key must be local (application state is shared over all running Kafka Streams instances)
+     * }

+     * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
+     * query the value of the key on a parallel running instance of your Kafka Streams application.
+     *
+     * 

      * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
-     * The changelog topic will be named "${applicationId}-${internalStoreName}-changelog", where "applicationId" is
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
      * user-specified in {@link StreamsConfig} via parameter
-     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
-     * and "-changelog" is a fixed suffix.
-     * Note that the internal store name may not be queriable through Interactive Queries.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param initializer   an {@link Initializer} that computes an initial intermediate aggregation result
      * @param aggregator    an {@link Aggregator} that computes a new aggregate result
+     * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
      * @param           the value type of the resulting {@link KTable}
      * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the
      * latest (rolling) aggregate for each key
      */
      KTable aggregate(final Initializer initializer,
-                                 final Aggregator aggregator);
+                                 final Aggregator aggregator,
+                                 final Materialized> materialized);
 
     /**
      * Create a new {@link TimeWindowedKStream} instance that can be used to perform windowed aggregations.
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java
index 1d1300e5bff5f..0e263362f5ced 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KGroupedTable.java
@@ -20,6 +20,7 @@
 import org.apache.kafka.common.utils.Bytes;
 import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.Topology;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.QueryableStoreType;
 
@@ -65,15 +66,17 @@ public interface KGroupedTable {
      * }

      * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
      * query the value of the key on a parallel running instance of your Kafka Streams application.
+     *
      * 

      * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
-     * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
      * user-specified in {@link StreamsConfig} via parameter
-     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the
-     * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix.
-     * The store name must be a valid Kafka topic name and cannot contain characters other than ASCII alphanumerics,
-     * '.', '_' and '-'.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param materialized the instance of {@link Materialized} used to materialize the state store. Cannot be {@code null}
      * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that
@@ -102,7 +105,8 @@ public interface KGroupedTable {
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
      * and "-changelog" is a fixed suffix.
      * Note that the internal store name may not be queriable through Interactive Queries.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that
      * represent the latest (rolling) count (i.e., number of records) for each key
@@ -162,13 +166,14 @@ public interface KGroupedTable {
      * query the value of the key on a parallel running instance of your Kafka Streams application.
      * 

      * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
-     * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
      * user-specified in {@link StreamsConfig} via parameter
-     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the
-     * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix.
-     * The store name must be a valid Kafka topic name and cannot contain characters other than ASCII alphanumerics,
-     * '.', '_' and '-'.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param adder         a {@link Reducer} that adds a new value to the aggregate result
      * @param subtractor    a {@link Reducer} that removed an old value from the aggregate result
@@ -225,7 +230,8 @@ KTable reduce(final Reducer adder,
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
      * and "-changelog" is a fixed suffix.
      * Note that the internal store name may not be queriable through Interactive Queries.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param adder      a {@link Reducer} that adds a new value to the aggregate result
      * @param subtractor a {@link Reducer} that removed an old value from the aggregate result
@@ -296,11 +302,14 @@ KTable reduce(final Reducer adder,
      * query the value of the key on a parallel running instance of your Kafka Streams application.
      * 

      * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
-     * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
      * user-specified in {@link StreamsConfig} via parameter
-     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the
-     * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param initializer   an {@link Initializer} that provides an initial aggregate result value
      * @param adder         an {@link Aggregator} that adds a new record to the aggregate result
@@ -321,7 +330,7 @@  KTable aggregate(final Initializer initializer,
      * Records with {@code null} key are ignored.
      * Aggregating is a generalization of {@link #reduce(Reducer, Reducer) combining via reduce(...)} as it,
      * for example, allows the result to have a different type than the input values.
-     * If the result value type does not match the {@link StreamsConfig#VALUE_SERDE_CLASS_CONFIG default value
+     * If the result value type does not match the {@link StreamsConfig#DEFAULT_VALUE_SERDE_CLASS_CONFIG default value
      * serde} you should use {@link #aggregate(Initializer, Aggregator, Aggregator, Materialized)}.
      * The result is written into a local {@link KeyValueStore} (which is basically an ever-updating materialized view)
      * provided by the given {@code storeSupplier}.
@@ -371,7 +380,8 @@  KTable aggregate(final Initializer initializer,
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name
      * and "-changelog" is a fixed suffix.
      * Note that the internal store name may not be queriable through Interactive Queries.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param initializer a {@link Initializer} that provides an initial aggregate result value
      * @param adder       a {@link Aggregator} that adds a new record to the aggregate result
@@ -384,5 +394,4 @@  KTable aggregate(final Initializer initializer,
                                  final Aggregator adder,
                                  final Aggregator subtractor);
 
-
 }
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
index 88b856576298f..09506ff522170 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KStream.java
@@ -20,7 +20,6 @@
 import org.apache.kafka.common.annotation.InterfaceStability;
 import org.apache.kafka.common.serialization.Serde;
 import org.apache.kafka.streams.Consumed;
-import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
@@ -753,7 +752,9 @@ void process(final ProcessorSupplier processorSupplier,
      * This topic will be named "${applicationId}-XXX-repartition", where "applicationId" is user-specified in
      * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is
      * an internally generated name, and "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
      * 

      * For this case, all data of this stream will be redistributed through the repartitioning topic by writing all
      * records to it, and rereading all records from it, such that the resulting {@link KGroupedStream} is partitioned
@@ -780,7 +781,9 @@ void process(final ProcessorSupplier processorSupplier,
      * This topic will be named "${applicationId}-XXX-repartition", where "applicationId" is user-specified in
      * {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is
      * an internally generated name, and "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
      * 

      * For this case, all data of this stream will be redistributed through the repartitioning topic by writing all
      * records to it, and rereading all records from it, such that the resulting {@link KGroupedStream} is partitioned
@@ -803,7 +806,9 @@ void process(final ProcessorSupplier processorSupplier,
      * This topic will be named "${applicationId}-XXX-repartition", where "applicationId" is user-specified in
      * {@link  StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is
      * an internally generated name, and "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
      * 

      * All data of this stream will be redistributed through the repartitioning topic by writing all records to it,
      * and rereading all records from it, such that the resulting {@link KGroupedStream} is partitioned on the new key.
@@ -829,7 +834,9 @@ void process(final ProcessorSupplier processorSupplier,
      * This topic will be named "${applicationId}-XXX-repartition", where "applicationId" is user-specified in
      * {@link  StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is
      * an internally generated name, and "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
      * 

      * All data of this stream will be redistributed through the repartitioning topic by writing all records to it,
      * and rereading all records from it, such that the resulting {@link KGroupedStream} is partitioned on the new key.
@@ -890,7 +897,7 @@  KGroupedStream groupBy(final KeyValueMapper
      * Repartitioning can happen for one or both of the joining {@code KStream}s.
      * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
@@ -903,7 +910,8 @@  KGroupedStream groupBy(final KeyValueMapper KStream join(final KStream otherStream,
      * user-specified in {@link  StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and
      * "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
      * 

      * Repartitioning can happen for one or both of the joining {@code KStream}s.
      * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
@@ -979,7 +986,8 @@  KStream join(final KStream otherStream,
      * in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is an
      * internally generated name, and "-changelog" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param otherStream the {@code KStream} to be joined with this stream
      * @param joiner      a {@link ValueJoiner} that computes the join result for a pair of matching records
@@ -1049,7 +1057,6 @@  KStream join(final KStream otherStream,
      * user-specified in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and
      * "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
      * 

      * Repartitioning can happen for one or both of the joining {@code KStream}s.
      * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
@@ -1061,7 +1068,8 @@  KStream join(final KStream otherStream,
      * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified
      * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
      * "storeName" is an internally generated name, and "-changelog" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param otherStream the {@code KStream} to be joined with this stream
      * @param joiner      a {@link ValueJoiner} that computes the join result for a pair of matching records
@@ -1129,7 +1137,6 @@  KStream leftJoin(final KStream otherStream,
      * user-specified in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and
      * "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
      * 

      * Repartitioning can happen for one or both of the joining {@code KStream}s.
      * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
@@ -1141,7 +1148,8 @@  KStream leftJoin(final KStream otherStream,
      * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified
      * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
      * "storeName" is an internally generated name, and "-changelog" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param otherStream the {@code KStream} to be joined with this stream
      * @param joiner      a {@link ValueJoiner} that computes the join result for a pair of matching records
@@ -1213,7 +1221,6 @@  KStream leftJoin(final KStream otherStream,
      * user-specified in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and
      * "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
      * 

      * Repartitioning can happen for one or both of the joining {@code KStream}s.
      * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
@@ -1225,7 +1232,8 @@  KStream leftJoin(final KStream otherStream,
      * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified
      * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
      * "storeName" is an internally generated name, and "-changelog" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param otherStream the {@code KStream} to be joined with this stream
      * @param joiner      a {@link ValueJoiner} that computes the join result for a pair of matching records
@@ -1256,7 +1264,7 @@  KStream outerJoin(final KStream otherStream,
      * a value (with arbitrary type) for the result record.
      * The key of the result record is the same as for both joining input records.
      * Furthermore, for each input record of both {@code KStream}s that does not satisfy the join predicate the provided
-     * {@link ValueJoiner} will be called with a {@code null} value for the this/other stream, respectively.
+     * {@link ValueJoiner} will be called with a {@code null} value for this/other stream, respectively.
      * If an input record key or value is {@code null} the record will not be included in the join operation and thus no
      * output record will be added to the resulting {@code KStream}.
      * 

@@ -1294,7 +1302,6 @@  KStream outerJoin(final KStream otherStream,
      * user-specified in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and
      * "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
      * 

      * Repartitioning can happen for one or both of the joining {@code KStream}s.
      * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
@@ -1306,7 +1313,8 @@  KStream outerJoin(final KStream otherStream,
      * The changelog topic will be named "${applicationId}-storeName-changelog", where "applicationId" is user-specified
      * in {@link StreamsConfig} via parameter {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG},
      * "storeName" is an internally generated name, and "-changelog" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
      *
      * @param otherStream the {@code KStream} to be joined with this stream
      * @param joiner      a {@link ValueJoiner} that computes the join result for a pair of matching records
@@ -1379,7 +1387,9 @@  KStream outerJoin(final KStream otherStream,
      * user-specified in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and
      * "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
      * 

      * Repartitioning can happen only for this {@code KStream} but not for the provided {@link KTable}.
      * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
@@ -1453,7 +1463,9 @@  KStream join(final KTable table,
      * user-specified in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and
      * "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
      * 

      * Repartitioning can happen only for this {@code KStream} but not for the provided {@link KTable}.
      * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
@@ -1533,7 +1545,9 @@  KStream join(final KTable table,
      * user-specified in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and
      * "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
      * 

      * Repartitioning can happen only for this {@code KStream} but not for the provided {@link KTable}.
      * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
@@ -1610,7 +1624,9 @@  KStream leftJoin(final KTable table,
      * user-specified in {@link StreamsConfig} via parameter
      * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "XXX" is an internally generated name, and
      * "-repartition" is a fixed suffix.
-     * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
      * 

      * Repartitioning can happen only for this {@code KStream} but not for the provided {@link KTable}.
      * For this case, all data of the stream will be redistributed through the repartitioning topic by writing all
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
index 19a403183ffb2..dbead79c1f6eb 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/KTable.java
@@ -23,6 +23,7 @@
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.Topology;
 import org.apache.kafka.streams.processor.StateStore;
 import org.apache.kafka.streams.state.KeyValueBytesStoreSupplier;
 import org.apache.kafka.streams.state.KeyValueStore;
@@ -395,7 +396,9 @@  KTable mapValues(final ValueMapperWithKey
      * All data of this {@code KTable} will be redistributed through the repartitioning topic by writing all update
      * records to and rereading all updated records from it, such that the resulting {@link KGroupedTable} is partitioned
@@ -425,7 +428,9 @@  KTable mapValues(final ValueMapperWithKey
      * All data of this {@code KTable} will be redistributed through the repartitioning topic by writing all update
      * records to and rereading all updated records from it, such that the resulting {@link KGroupedTable} is partitioned
diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindowedKStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindowedKStream.java
index efc60ea5a45c9..36e7823648a9d 100644
--- a/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindowedKStream.java
+++ b/streams/src/main/java/org/apache/kafka/streams/kstream/SessionWindowedKStream.java
@@ -21,6 +21,7 @@
 import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.KeyValue;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.Topology;
 import org.apache.kafka.streams.state.KeyValueStore;
 import org.apache.kafka.streams.state.QueryableStoreType;
 import org.apache.kafka.streams.state.SessionStore;
@@ -91,6 +92,17 @@ public interface SessionWindowedKStream {
      * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to
      * query the value of the key on a parallel running instance of your Kafka Streams application.
      *
+     * 

+     * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka.
+     * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII
+     * alphanumerics, '.', '_' and '-'.
+     * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is
+     * user-specified in {@link StreamsConfig} via parameter
+     * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the
+     * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix.
+     *
+     * You can retrieve all generated internal topic names via {@link Topology#describe()}.
+     *
      * @param materialized  an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}.
      *                      Note: the valueSerde will be automatically set to {@link Serdes#Long()} if there is no valueSerde provided
      * @return a windowed {@link KTable} that contains "update" records with unmodified keys and {@link Long} values
@@ -168,6 +180,18 @@  KTable, VR> aggregate(final Initializer initializer,
      * String key = "some-key";
      * KeyValueIterator, Long> aggForKeyForSession = localWindowStore.fetch(key); // key must be local (application state is shared over all running Kafka Streams instances)
      * }
+ * + *

+ * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. + * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII + * alphanumerics, '.', '_' and '-'. + * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is + * user-specified in {@link StreamsConfig} via parameter + * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the + * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix. + * + * You can retrieve all generated internal topic names via {@link Topology#describe()}. + * * @param initializer the instance of {@link Initializer} * @param aggregator the instance of {@link Aggregator} * @param sessionMerger the instance of {@link Merger} @@ -251,15 +275,18 @@ KTable, VR> aggregate(final Initializer initializer, * } * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to * query the value of the key on a parallel running instance of your Kafka Streams application. + * *

* For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. - * Therefore, the store name must be a valid Kafka topic name and cannot contain characters other than ASCII + * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII * alphanumerics, '.', '_' and '-'. - * The changelog topic will be named "${applicationId}-${queryableStoreName}-changelog", where "applicationId" is + * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is * user-specified in {@link StreamsConfig} via parameter - * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "queryableStoreName" is the - * provide {@code queryableStoreName}, and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. + * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the + * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix. + * You can retrieve all generated internal topic names via {@link Topology#describe()}. + * + * * @param reducer a {@link Reducer} that computes a new aggregate result. Cannot be {@code null}. * @param materializedAs an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null} * @return a windowed {@link KTable} that contains "update" records with unmodified keys, and values that represent diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindowedKStream.java b/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindowedKStream.java index 7f9752b6c2fd1..03bc986bc6683 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindowedKStream.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/TimeWindowedKStream.java @@ -20,6 +20,7 @@ import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.QueryableStoreType; import org.apache.kafka.streams.state.WindowStore; @@ -67,7 +68,9 @@ public interface TimeWindowedKStream { * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name * and "-changelog" is a fixed suffix. * Note that the internal store name may not be queriable through Interactive Queries. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. + * + * You can retrieve all generated internal topic names via {@link Topology#describe()}. + * * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that * represent the latest (rolling) count (i.e., number of records) for each key */ @@ -99,7 +102,19 @@ public interface TimeWindowedKStream { * } * For non-local keys, a custom RPC mechanism must be implemented using {@link KafkaStreams#allMetadata()} to * query the value of the key on a parallel running instance of your Kafka Streams application. - ** @param materialized an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}. + * + *

+ * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. + * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII + * alphanumerics, '.', '_' and '-'. + * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is + * user-specified in {@link StreamsConfig} via parameter + * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the + * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix. + * + * You can retrieve all generated internal topic names via {@link Topology#describe()}. + * + * @param materialized an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}. * Note: the valueSerde will be automatically set to {@link org.apache.kafka.common.serialization.Serdes#Long() Serdes#Long()} * if there is no valueSerde provided * @return a {@link KTable} that contains "update" records with unmodified keys and {@link Long} values that @@ -140,7 +155,9 @@ public interface TimeWindowedKStream { * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name * and "-changelog" is a fixed suffix. * Note that the internal store name may not be queriable through Interactive Queries. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. + * + * You can retrieve all generated internal topic names via {@link Topology#describe()}. + * * * @param the value type of the resulting {@link KTable} * @param initializer an {@link Initializer} that computes an initial intermediate aggregation result @@ -189,6 +206,17 @@ KTable, VR> aggregate(final Initializer initializer, * WindowStoreIterator aggregateStore = localWindowStore.fetch(key, timeFrom, timeTo); // key must be local (application state is shared over all running Kafka Streams instances) * } * + *

+ * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. + * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII + * alphanumerics, '.', '_' and '-'. + * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is + * user-specified in {@link StreamsConfig} via parameter + * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the + * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix. + * + * You can retrieve all generated internal topic names via {@link Topology#describe()}. + * * @param initializer an {@link Initializer} that computes an initial intermediate aggregation result * @param aggregator an {@link Aggregator} that computes a new aggregate result * @param materialized an instance of {@link Materialized} used to materialize a state store. Cannot be {@code null}. @@ -226,7 +254,8 @@ KTable, VR> aggregate(final Initializer initializer, * user-specified in {@link StreamsConfig} via parameter * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "internalStoreName" is an internal name * and "-changelog" is a fixed suffix. - * You can retrieve all generated internal topic names via {@link KafkaStreams#toString()}. + * + * You can retrieve all generated internal topic names via {@link Topology#describe()}. * * @param reducer a {@link Reducer} that computes a new aggregate result * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the @@ -268,6 +297,16 @@ KTable, VR> aggregate(final Initializer initializer, * WindowStoreIterator reduceStore = localWindowStore.fetch(key, timeFrom, timeTo); // key must be local (application state is shared over all running Kafka Streams instances) * } * + *

+ * For failure and recovery the store will be backed by an internal changelog topic that will be created in Kafka. + * Therefore, the store name defined by the Materialized instance must be a valid Kafka topic name and cannot contain characters other than ASCII + * alphanumerics, '.', '_' and '-'. + * The changelog topic will be named "${applicationId}-${storeName}-changelog", where "applicationId" is + * user-specified in {@link StreamsConfig} via parameter + * {@link StreamsConfig#APPLICATION_ID_CONFIG APPLICATION_ID_CONFIG}, "storeName" is the + * provide store name defined in {@code Materialized}, and "-changelog" is a fixed suffix. + * + * You can retrieve all generated internal topic names via {@link Topology#describe()}. * * @param reducer a {@link Reducer} that computes a new aggregate result * @return a {@link KTable} that contains "update" records with unmodified keys, and values that represent the diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java index 3caddf7959e09..1c5ad4d19c94c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KTableImpl.java @@ -118,42 +118,27 @@ private KTable doFilter(final Predicate predicate, final String name = builder.newProcessorName(FILTER_NAME); // only materialize if the state store is queryable - KTableProcessorSupplier processorSupplier; + final boolean shouldMaterialize = materialized != null && materialized.isQueryable(); - if (materialized != null && materialized.isQueryable()) { - processorSupplier = new KTableFilter<>(this, - predicate, - filterNot, - materialized.storeName()); + KTableProcessorSupplier processorSupplier = new KTableFilter<>(this, + predicate, + filterNot, + shouldMaterialize ? materialized.storeName() : null); - builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); - this.builder.internalTopologyBuilder.addStateStore(new KeyValueStoreMaterializer<>(materialized).materialize(), name); + builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); - return new KTableImpl<>(builder, - name, - processorSupplier, - this.keySerde, - this.valSerde, - sourceNodes, - materialized.storeName(), - true); - } else { - processorSupplier = new KTableFilter<>(this, - predicate, - filterNot, - null); - - builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); - - return new KTableImpl<>(builder, - name, - processorSupplier, - this.keySerde, - this.valSerde, - sourceNodes, - this.queryableStoreName, - false); + if (shouldMaterialize) { + this.builder.internalTopologyBuilder.addStateStore(new KeyValueStoreMaterializer<>(materialized).materialize(), name); } + + return new KTableImpl<>(builder, + name, + processorSupplier, + this.keySerde, + this.valSerde, + sourceNodes, + shouldMaterialize ? materialized.storeName() : this.queryableStoreName, + shouldMaterialize); } @Override @@ -184,29 +169,46 @@ public KTable filterNot(final Predicate predicate, return doFilter(predicate, new MaterializedInternal<>(materialized, builder, FILTER_NAME), true); } - @Override - public KTable mapValues(final ValueMapper mapper) { - return mapValues(withKey(mapper)); - } - - @Override - public KTable mapValues(final ValueMapperWithKey mapper) { - Objects.requireNonNull(mapper, "mapper can't be null"); + private KTable doMapValues(final ValueMapperWithKey mapper, + final MaterializedInternal> materialized) { final String name = builder.newProcessorName(MAPVALUES_NAME); + // only materialize if the state store is queryable + final boolean shouldMaterialize = materialized != null && materialized.isQueryable(); + final KTableProcessorSupplier processorSupplier = new KTableMapValues<>( this, mapper, - null); + shouldMaterialize ? materialized.storeName() : null); + builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); - return new KTableImpl<>(builder, name, processorSupplier, sourceNodes, this.queryableStoreName, false); + if (shouldMaterialize) { + this.builder.internalTopologyBuilder.addStateStore(new KeyValueStoreMaterializer<>(materialized).materialize(), name); + } + + return new KTableImpl<>(builder, name, processorSupplier, sourceNodes, shouldMaterialize ? materialized.storeName() : this.queryableStoreName, shouldMaterialize); + } + + @Override + public KTable mapValues(final ValueMapper mapper) { + Objects.requireNonNull(mapper, "mapper can't be null"); + return doMapValues(withKey(mapper), null); + } + + @Override + public KTable mapValues(final ValueMapperWithKey mapper) { + Objects.requireNonNull(mapper, "mapper can't be null"); + return doMapValues(mapper, null); } @Override public KTable mapValues(final ValueMapper mapper, final Materialized> materialized) { - return mapValues(withKey(mapper), materialized); + Objects.requireNonNull(mapper, "mapper can't be null"); + Objects.requireNonNull(materialized, "materialized can't be null"); + + return doMapValues(withKey(mapper), new MaterializedInternal<>(materialized, builder, MAPVALUES_NAME)); } @Override @@ -215,30 +217,7 @@ public KTable mapValues(final ValueMapperWithKey> materializedInternal - = new MaterializedInternal<>(materialized, builder, MAPVALUES_NAME); - final String name = builder.newProcessorName(MAPVALUES_NAME); - - // only materialize if the state store is queryable - KTableProcessorSupplier processorSupplier; - - if (materializedInternal.isQueryable()) { - processorSupplier = new KTableMapValues<>( - this, - mapper, - materializedInternal.storeName()); - - builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); - this.builder.internalTopologyBuilder.addStateStore(new KeyValueStoreMaterializer<>(materializedInternal).materialize(), name); - } else { - processorSupplier = new KTableMapValues<>( - this, - mapper, - null); - builder.internalTopologyBuilder.addProcessor(name, processorSupplier, this.name); - } - - return new KTableImpl<>(builder, name, processorSupplier, sourceNodes, materializedInternal.storeName(), materializedInternal.isQueryable()); + return doMapValues(mapper, new MaterializedInternal<>(materialized, builder, MAPVALUES_NAME)); } @Override diff --git a/streams/src/test/java/org/apache/kafka/streams/TopologyWrapper.java b/streams/src/test/java/org/apache/kafka/streams/TopologyWrapper.java new file mode 100644 index 0000000000000..f1067667f3162 --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/TopologyWrapper.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams; + +import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; + +/** + * This class allows to access the {@link InternalTopologyBuilder} a {@link Topology} object. + * + */ +public class TopologyWrapper extends Topology { + + public InternalTopologyBuilder getInternalBuilder() { + return internalTopologyBuilder; + } + + public void setApplicationId(String applicationId) { + internalTopologyBuilder.setApplicationId(applicationId); + } +} diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java index 7f44c1ea9c1cd..6dc0db87dfb9d 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java @@ -63,7 +63,7 @@ import static org.hamcrest.core.Is.is; /** - * Similar to KStreamAggregationIntegrationTest but with de dupping enabled + * Similar to KStreamAggregationIntegrationTest but with dedupping enabled * by virtue of having a large commit interval */ @Category({IntegrationTest.class}) diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/PurgeRepartitionTopicIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/PurgeRepartitionTopicIntegrationTest.java index ac113ce64d6cf..9dfb6dda37ec2 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/PurgeRepartitionTopicIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/PurgeRepartitionTopicIntegrationTest.java @@ -166,8 +166,6 @@ public void setup() { .groupBy(MockMapper.selectKeyKeyValueMapper()) .count(); - System.out.print(builder.build().describe()); - kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration, time); } From bb9d4fb373294a83d6beebf282eef1472efcb7fc Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 7 May 2018 17:14:39 -0700 Subject: [PATCH 12/13] remove TopologyWrapper --- .../apache/kafka/streams/TopologyWrapper.java | 34 ------------------- 1 file changed, 34 deletions(-) delete mode 100644 streams/src/test/java/org/apache/kafka/streams/TopologyWrapper.java diff --git a/streams/src/test/java/org/apache/kafka/streams/TopologyWrapper.java b/streams/src/test/java/org/apache/kafka/streams/TopologyWrapper.java deleted file mode 100644 index f1067667f3162..0000000000000 --- a/streams/src/test/java/org/apache/kafka/streams/TopologyWrapper.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.streams; - -import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; - -/** - * This class allows to access the {@link InternalTopologyBuilder} a {@link Topology} object. - * - */ -public class TopologyWrapper extends Topology { - - public InternalTopologyBuilder getInternalBuilder() { - return internalTopologyBuilder; - } - - public void setApplicationId(String applicationId) { - internalTopologyBuilder.setApplicationId(applicationId); - } -} From 52b7257a5b75a831530a76548620be9f546948a7 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 7 May 2018 17:27:00 -0700 Subject: [PATCH 13/13] github comments --- .../kafka/streams/kstream/internals/KGroupedStreamImpl.java | 2 +- .../streams/kstream/internals/SessionWindowedKStreamImpl.java | 2 +- .../streams/kstream/internals/TimeWindowedKStreamImpl.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java index 35fda941e81a7..3a9f9197ac7c8 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/KGroupedStreamImpl.java @@ -75,7 +75,7 @@ public KTable reduce(final Reducer reducer, Objects.requireNonNull(reducer, "reducer can't be null"); Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal - = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); + = new MaterializedInternal<>(materialized, builder, REDUCE_NAME); if (materializedInternal.keySerde() == null) { materializedInternal.withKeySerde(keySerde); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java index c3b295b0cfb01..c29c6566c2f4b 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/SessionWindowedKStreamImpl.java @@ -104,7 +104,7 @@ public KTable, V> reduce(final Reducer reducer, Objects.requireNonNull(materialized, "materialized can't be null"); final Aggregator reduceAggregator = aggregatorForReducer(reducer); final MaterializedInternal> materializedInternal - = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); + = new MaterializedInternal<>(materialized, builder, REDUCE_NAME); if (materializedInternal.keySerde() == null) { materializedInternal.withKeySerde(keySerde); } diff --git a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImpl.java b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImpl.java index cee0192a08fab..d1e5a1758476a 100644 --- a/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImpl.java +++ b/streams/src/main/java/org/apache/kafka/streams/kstream/internals/TimeWindowedKStreamImpl.java @@ -123,7 +123,7 @@ public KTable, V> reduce(final Reducer reducer, final Materialize Objects.requireNonNull(materialized, "materialized can't be null"); final MaterializedInternal> materializedInternal - = new MaterializedInternal<>(materialized, builder, AGGREGATE_NAME); + = new MaterializedInternal<>(materialized, builder, REDUCE_NAME); if (materializedInternal.keySerde() == null) { materializedInternal.withKeySerde(keySerde); }