diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 716f55d18befd..fcaca1ddb7e81 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -152,7 +152,7 @@ + files="(KafkaStreams|KStreamImpl|KTableImpl|InternalTopologyBuilder|StreamsPartitionAssignor).java"/> diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java index b508396ae18f3..f1f474c7a0995 100644 --- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java +++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java @@ -44,7 +44,6 @@ import org.apache.kafka.streams.errors.StreamsException; import org.apache.kafka.streams.errors.StreamsNotStartedException; import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler; -import org.apache.kafka.streams.errors.TopologyException; import org.apache.kafka.streams.errors.UnknownStateStoreException; import org.apache.kafka.streams.errors.InvalidStateStorePartitionException; import org.apache.kafka.streams.internals.metrics.ClientMetrics; @@ -54,8 +53,7 @@ import org.apache.kafka.streams.processor.internals.ClientUtils; import org.apache.kafka.streams.processor.internals.DefaultKafkaClientSupplier; import org.apache.kafka.streams.processor.internals.GlobalStreamThread; -import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; -import org.apache.kafka.streams.processor.internals.ProcessorTopology; +import org.apache.kafka.streams.processor.internals.TopologyMetadata; import org.apache.kafka.streams.processor.internals.StateDirectory; import org.apache.kafka.streams.processor.internals.StreamThread; import org.apache.kafka.streams.processor.internals.StreamsMetadataState; @@ -159,15 +157,13 @@ public class KafkaStreams implements AutoCloseable { private final ScheduledExecutorService rocksDBMetricsRecordingService; private final Admin adminClient; private final StreamsMetricsImpl streamsMetrics; - private final ProcessorTopology taskTopology; - private final ProcessorTopology globalTaskTopology; private final long totalCacheSize; private final StreamStateListener streamStateListener; private final StateRestoreListener delegatingStateRestoreListener; private final Map threadState; private final UUID processId; private final KafkaClientSupplier clientSupplier; - private final InternalTopologyBuilder internalTopologyBuilder; + protected final TopologyMetadata topologyMetadata; private final QueryableStoreProvider queryableStoreProvider; GlobalStreamThread globalStreamThread; @@ -690,7 +686,7 @@ public void onRestoreEnd(final TopicPartition topicPartition, final String store */ public KafkaStreams(final Topology topology, final Properties props) { - this(topology.internalTopologyBuilder, new StreamsConfig(props), new DefaultKafkaClientSupplier()); + this(topology, new StreamsConfig(props), new DefaultKafkaClientSupplier()); } /** @@ -708,7 +704,7 @@ public KafkaStreams(final Topology topology, public KafkaStreams(final Topology topology, final Properties props, final KafkaClientSupplier clientSupplier) { - this(topology.internalTopologyBuilder, new StreamsConfig(props), clientSupplier, Time.SYSTEM); + this(topology, new StreamsConfig(props), clientSupplier, Time.SYSTEM); } /** @@ -725,7 +721,7 @@ public KafkaStreams(final Topology topology, public KafkaStreams(final Topology topology, final Properties props, final Time time) { - this(topology.internalTopologyBuilder, new StreamsConfig(props), new DefaultKafkaClientSupplier(), time); + this(topology, new StreamsConfig(props), new DefaultKafkaClientSupplier(), time); } /** @@ -745,7 +741,7 @@ public KafkaStreams(final Topology topology, final Properties props, final KafkaClientSupplier clientSupplier, final Time time) { - this(topology.internalTopologyBuilder, new StreamsConfig(props), clientSupplier, time); + this(topology, new StreamsConfig(props), clientSupplier, time); } /** @@ -778,7 +774,7 @@ public KafkaStreams(final Topology topology, public KafkaStreams(final Topology topology, final StreamsConfig config, final KafkaClientSupplier clientSupplier) { - this(topology.internalTopologyBuilder, config, clientSupplier); + this(new TopologyMetadata(topology.internalTopologyBuilder, config), config, clientSupplier); } /** @@ -795,41 +791,41 @@ public KafkaStreams(final Topology topology, public KafkaStreams(final Topology topology, final StreamsConfig config, final Time time) { - this(topology.internalTopologyBuilder, config, new DefaultKafkaClientSupplier(), time); + this(new TopologyMetadata(topology.internalTopologyBuilder, config), config, new DefaultKafkaClientSupplier(), time); } - private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, + private KafkaStreams(final Topology topology, final StreamsConfig config, - final KafkaClientSupplier clientSupplier) throws StreamsException { - this(internalTopologyBuilder, config, clientSupplier, Time.SYSTEM); + final KafkaClientSupplier clientSupplier, + final Time time) throws StreamsException { + this(new TopologyMetadata(topology.internalTopologyBuilder, config), config, clientSupplier, time); + } + + protected KafkaStreams(final TopologyMetadata topologyMetadata, + final StreamsConfig config, + final KafkaClientSupplier clientSupplier) throws StreamsException { + this(topologyMetadata, config, clientSupplier, Time.SYSTEM); } - private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, + private KafkaStreams(final TopologyMetadata topologyMetadata, final StreamsConfig config, final KafkaClientSupplier clientSupplier, final Time time) throws StreamsException { this.config = config; this.time = time; - this.internalTopologyBuilder = internalTopologyBuilder; - internalTopologyBuilder.rewriteTopology(config); + this.topologyMetadata = topologyMetadata; + this.topologyMetadata.buildAndRewriteTopology(); - // sanity check to fail-fast in case we cannot build a ProcessorTopology due to an exception - taskTopology = internalTopologyBuilder.buildTopology(); - globalTaskTopology = internalTopologyBuilder.buildGlobalStateTopology(); - - final boolean hasGlobalTopology = globalTaskTopology != null; - final boolean hasPersistentStores = taskTopology.hasPersistentLocalStore() || - (hasGlobalTopology && globalTaskTopology.hasPersistentGlobalStore()); + final boolean hasGlobalTopology = topologyMetadata.hasGlobalTopology(); try { - stateDirectory = new StateDirectory(config, time, hasPersistentStores, internalTopologyBuilder.hasNamedTopologies()); + stateDirectory = new StateDirectory(config, time, topologyMetadata.hasPersistentStores(), topologyMetadata.hasNamedTopologies()); processId = stateDirectory.initializeProcessId(); } catch (final ProcessorStateException fatal) { throw new StreamsException(fatal); } - // The application ID is a required config and hence should always have value final String userClientId = config.getString(StreamsConfig.CLIENT_ID_CONFIG); final String applicationId = config.getString(StreamsConfig.APPLICATION_ID_CONFIG); @@ -859,12 +855,12 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, ClientMetrics.addVersionMetric(streamsMetrics); ClientMetrics.addCommitIdMetric(streamsMetrics); ClientMetrics.addApplicationIdMetric(streamsMetrics, config.getString(StreamsConfig.APPLICATION_ID_CONFIG)); - ClientMetrics.addTopologyDescriptionMetric(streamsMetrics, internalTopologyBuilder.describe().toString()); + ClientMetrics.addTopologyDescriptionMetric(streamsMetrics, this.topologyMetadata.topologyDescriptionString()); ClientMetrics.addStateMetric(streamsMetrics, (metricsConfig, now) -> state); ClientMetrics.addNumAliveStreamThreadMetric(streamsMetrics, (metricsConfig, now) -> getNumLiveStreamThreads()); streamsMetadataState = new StreamsMetadataState( - internalTopologyBuilder, + this.topologyMetadata, parseHostInfo(config.getString(StreamsConfig.APPLICATION_SERVER_CONFIG))); oldHandler = false; @@ -872,14 +868,14 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, delegatingStateRestoreListener = new DelegatingStateRestoreListener(); totalCacheSize = config.getLong(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG); - final int numStreamThreads = getNumStreamThreads(hasGlobalTopology); + final int numStreamThreads = topologyMetadata.getNumStreamThreads(config); final long cacheSizePerThread = getCacheSizePerThread(numStreamThreads); GlobalStreamThread.State globalThreadState = null; if (hasGlobalTopology) { final String globalThreadId = clientId + "-GlobalStreamThread"; globalStreamThread = new GlobalStreamThread( - globalTaskTopology, + topologyMetadata.globalTaskTopology(), config, clientSupplier.getGlobalConsumer(config.getGlobalConsumerConfigs(clientId)), stateDirectory, @@ -897,7 +893,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, threadState = new HashMap<>(numStreamThreads); streamStateListener = new StreamStateListener(threadState, globalThreadState); - final GlobalStateStoreProvider globalStateStoreProvider = new GlobalStateStoreProvider(internalTopologyBuilder.globalStateStores()); + final GlobalStateStoreProvider globalStateStoreProvider = new GlobalStateStoreProvider(this.topologyMetadata.globalStateStores()); if (hasGlobalTopology) { globalStreamThread.setStateListener(streamStateListener); @@ -914,7 +910,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, private StreamThread createAndAddStreamThread(final long cacheSizePerThread, final int threadIdx) { final StreamThread streamThread = StreamThread.create( - internalTopologyBuilder, + topologyMetadata, config, clientSupplier, adminClient, @@ -953,23 +949,6 @@ private static Metrics getMetrics(final StreamsConfig config, final Time time, f return new Metrics(metricConfig, reporters, time, metricsContext); } - private int getNumStreamThreads(final boolean hasGlobalTopology) { - final int numStreamThreads; - if (internalTopologyBuilder.hasNoNonGlobalTopology()) { - log.info("Overriding number of StreamThreads to zero for global-only topology"); - numStreamThreads = 0; - } else { - numStreamThreads = config.getInt(StreamsConfig.NUM_STREAM_THREADS_CONFIG); - } - - if (numStreamThreads == 0 && !hasGlobalTopology) { - log.error("Topology with no input topics will create no stream threads and no global thread."); - throw new TopologyException("Topology has no stream threads and no global threads, " + - "must subscribe to at least one source topic or global table."); - } - return numStreamThreads; - } - /** * Adds and starts a stream thread in addition to the stream threads that are already running in this * Kafka Streams client. @@ -1195,7 +1174,7 @@ private long getCacheSizePerThread(final int numStreamThreads) { if (numStreamThreads == 0) { return totalCacheSize; } - return totalCacheSize / (numStreamThreads + ((globalTaskTopology != null) ? 1 : 0)); + return totalCacheSize / (numStreamThreads + (topologyMetadata.hasGlobalTopology() ? 1 : 0)); } private void resizeThreadCache(final long cacheSizePerThread) { @@ -1284,6 +1263,14 @@ public synchronized void start() throws IllegalStateException, StreamsException } else { throw new IllegalStateException("The client is either already started or already stopped, cannot re-start"); } + + if (topologyMetadata.isEmpty()) { + if (setState(State.RUNNING)) { + log.debug("Transitioning directly to RUNNING for app with no named topologies"); + } else { + throw new IllegalStateException("Unexpected error in transitioning KafkaStreams with empty processing topology to RUNNING"); + } + } } /** @@ -1575,8 +1562,7 @@ public KeyQueryMetadata queryMetadataForKey(final String storeName, public T store(final StoreQueryParameters storeQueryParameters) { validateIsRunningOrRebalancing(); final String storeName = storeQueryParameters.storeName(); - if ((taskTopology == null || !taskTopology.hasStore(storeName)) - && (globalTaskTopology == null || !globalTaskTopology.hasStore(storeName))) { + if (!topologyMetadata.hasStore(storeName)) { throw new UnknownStateStoreException( "Cannot get state store " + storeName + " because no such store is registered in the topology." ); diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/TaskId.java b/streams/src/main/java/org/apache/kafka/streams/processor/TaskId.java index f4d8349eadd95..58cfa8afba0d5 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/TaskId.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/TaskId.java @@ -37,6 +37,8 @@ public class TaskId implements Comparable { private static final Logger LOG = LoggerFactory.getLogger(TaskId.class); + public static final String NAMED_TOPOLOGY_DELIMITER = "__"; + /** The ID of the subtopology, aka topicGroupId. */ @Deprecated public final int topicGroupId; @@ -80,30 +82,30 @@ public String namedTopology() { @Override public String toString() { - return namedTopology != null ? namedTopology + "_" + topicGroupId + "_" + partition : topicGroupId + "_" + partition; + return namedTopology != null ? namedTopology + NAMED_TOPOLOGY_DELIMITER + topicGroupId + "_" + partition : topicGroupId + "_" + partition; } /** * @throws TaskIdFormatException if the taskIdStr is not a valid {@link TaskId} */ public static TaskId parse(final String taskIdStr) { - final int firstIndex = taskIdStr.indexOf('_'); - final int secondIndex = taskIdStr.indexOf('_', firstIndex + 1); - if (firstIndex <= 0 || firstIndex + 1 >= taskIdStr.length()) { - throw new TaskIdFormatException(taskIdStr); - } - try { - // If only one copy of '_' exists, there is no named topology in the string - if (secondIndex < 0) { - final int topicGroupId = Integer.parseInt(taskIdStr.substring(0, firstIndex)); - final int partition = Integer.parseInt(taskIdStr.substring(firstIndex + 1)); + final int namedTopologyDelimiterIndex = taskIdStr.indexOf(NAMED_TOPOLOGY_DELIMITER); + // If there is no copy of the NamedTopology delimiter, this task has no named topology and only one `_` char + if (namedTopologyDelimiterIndex < 0) { + final int index = taskIdStr.indexOf('_'); + + final int topicGroupId = Integer.parseInt(taskIdStr.substring(0, index)); + final int partition = Integer.parseInt(taskIdStr.substring(index + 1)); return new TaskId(topicGroupId, partition); } else { - final String namedTopology = taskIdStr.substring(0, firstIndex); - final int topicGroupId = Integer.parseInt(taskIdStr.substring(firstIndex + 1, secondIndex)); - final int partition = Integer.parseInt(taskIdStr.substring(secondIndex + 1)); + final int topicGroupIdIndex = namedTopologyDelimiterIndex + 2; + final int subtopologyPartitionDelimiterIndex = taskIdStr.indexOf('_', topicGroupIdIndex); + + final String namedTopology = taskIdStr.substring(0, namedTopologyDelimiterIndex); + final int topicGroupId = Integer.parseInt(taskIdStr.substring(topicGroupIdIndex, subtopologyPartitionDelimiterIndex)); + final int partition = Integer.parseInt(taskIdStr.substring(subtopologyPartitionDelimiterIndex + 1)); return new TaskId(topicGroupId, partition, namedTopology); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreator.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreator.java index 322ff56e74d00..0ff79d59dff27 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreator.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreator.java @@ -49,7 +49,7 @@ import static org.apache.kafka.streams.processor.internals.StreamThread.ProcessingMode.EXACTLY_ONCE_V2; class ActiveTaskCreator { - private final InternalTopologyBuilder builder; + private final TopologyMetadata topologyMetadata; private final StreamsConfig config; private final StreamsMetricsImpl streamsMetrics; private final StateDirectory stateDirectory; @@ -64,7 +64,7 @@ class ActiveTaskCreator { private final Map taskProducers; private final StreamThread.ProcessingMode processingMode; - ActiveTaskCreator(final InternalTopologyBuilder builder, + ActiveTaskCreator(final TopologyMetadata topologyMetadata, final StreamsConfig config, final StreamsMetricsImpl streamsMetrics, final StateDirectory stateDirectory, @@ -75,7 +75,7 @@ class ActiveTaskCreator { final String threadId, final UUID processId, final Logger log) { - this.builder = builder; + this.topologyMetadata = topologyMetadata; this.config = config; this.streamsMetrics = streamsMetrics; this.stateDirectory = stateDirectory; @@ -143,7 +143,7 @@ Collection createTasks(final Consumer consumer, final LogContext logContext = getLogContext(taskId); - final ProcessorTopology topology = builder.buildSubtopology(taskId.subtopology()); + final ProcessorTopology topology = topologyMetadata.buildSubtopology(taskId); final ProcessorStateManager stateManager = new ProcessorStateManager( taskId, @@ -194,7 +194,7 @@ StreamTask createActiveTaskFromStandby(final StandbyTask standbyTask, inputPartitions, consumer, logContext, - builder.buildSubtopology(standbyTask.id.subtopology()), + topologyMetadata.buildSubtopology(standbyTask.id), stateManager, context ); diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java index 9b392640570bc..fc5333700c8d2 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilder.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.streams.processor.internals; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serializer; @@ -56,10 +57,13 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import static org.apache.kafka.clients.consumer.OffsetResetStrategy.EARLIEST; +import static org.apache.kafka.clients.consumer.OffsetResetStrategy.LATEST; +import static org.apache.kafka.clients.consumer.OffsetResetStrategy.NONE; + public class InternalTopologyBuilder { private static final Logger log = LoggerFactory.getLogger(InternalTopologyBuilder.class); - private static final Pattern EMPTY_ZERO_LENGTH_PATTERN = Pattern.compile(""); private static final String[] NO_PREDECESSORS = {}; // node factories in a topological order @@ -122,7 +126,7 @@ public class InternalTopologyBuilder { private String applicationId = null; - private Pattern sourceTopicPattern = null; + private String sourceTopicPatternString = null; private List sourceTopicCollection = null; @@ -131,15 +135,9 @@ public class InternalTopologyBuilder { private StreamsConfig config = null; // The name of the topology this builder belongs to, or null if none - private final String namedTopology; + private String topologyName; - public InternalTopologyBuilder() { - this.namedTopology = null; - } - - public InternalTopologyBuilder(final String namedTopology) { - this.namedTopology = namedTopology; - } + private boolean hasPersistentStores = false; public static class StateStoreFactory { private final StoreBuilder builder; @@ -346,8 +344,17 @@ Sink describe() { } } + public void setTopologyName(final String namedTopology) { + Objects.requireNonNull(namedTopology, "named topology can't be null"); + if (this.topologyName != null) { + log.error("Tried to reset the namedTopology to {} but it was already set to {}", namedTopology, this.topologyName); + throw new IllegalStateException("NamedTopology has already been set to " + this.topologyName); + } + this.topologyName = namedTopology; + } + // public for testing only - public synchronized final InternalTopologyBuilder setApplicationId(final String applicationId) { + public final InternalTopologyBuilder setApplicationId(final String applicationId) { Objects.requireNonNull(applicationId, "applicationId can't be null"); this.applicationId = applicationId; @@ -365,6 +372,10 @@ public synchronized final StreamsConfig getStreamsConfig() { return config; } + public String topologyName() { + return topologyName; + } + public synchronized final InternalTopologyBuilder rewriteTopology(final StreamsConfig config) { Objects.requireNonNull(config, "config can't be null"); @@ -631,8 +642,8 @@ public final void connectProcessorAndStateStores(final String processorName, nodeGroups = null; } - public Map getChangelogTopicToStore() { - return changelogTopicToStore; + public String getStoreForChangelogTopic(final String topicName) { + return changelogTopicToStore.get(topicName); } public void connectSourceStoreAndTopic(final String sourceStoreName, @@ -1038,18 +1049,26 @@ private void buildProcessorNode(final Map> pro } for (final String stateStoreName : factory.stateStoreNames) { if (!stateStoreMap.containsKey(stateStoreName)) { + final StateStore store; if (stateFactories.containsKey(stateStoreName)) { final StateStoreFactory stateStoreFactory = stateFactories.get(stateStoreName); // remember the changelog topic if this state store is change-logging enabled if (stateStoreFactory.loggingEnabled() && !storeToChangelogTopic.containsKey(stateStoreName)) { - final String changelogTopic = ProcessorStateManager.storeChangelogTopic(applicationId, stateStoreName); + final String changelogTopic = + ProcessorStateManager.storeChangelogTopic(applicationId, stateStoreName, topologyName); storeToChangelogTopic.put(stateStoreName, changelogTopic); changelogTopicToStore.put(changelogTopic, stateStoreName); } - stateStoreMap.put(stateStoreName, stateStoreFactory.build()); + store = stateStoreFactory.build(); + stateStoreMap.put(stateStoreName, store); } else { - stateStoreMap.put(stateStoreName, globalStateStores.get(stateStoreName)); + store = globalStateStores.get(stateStoreName); + stateStoreMap.put(stateStoreName, store); + } + + if (store.persistent()) { + hasPersistentStores = true; } } } @@ -1066,7 +1085,7 @@ public Map globalStateStores() { return Collections.unmodifiableMap(globalStateStores); } - public Set allStateStoreName() { + public Set allStateStoreNames() { Objects.requireNonNull(applicationId, "topology has not completed optimization"); final Set allNames = new HashSet<>(stateFactories.keySet()); @@ -1074,6 +1093,14 @@ public Set allStateStoreName() { return Collections.unmodifiableSet(allNames); } + public boolean hasStore(final String name) { + return stateFactories.containsKey(name) || globalStateStores.containsKey(name); + } + + public boolean hasPersistentStores() { + return hasPersistentStores; + } + /** * Returns the map of topic groups keyed by the group id. * A topic group is a group of topics in the same task. @@ -1144,7 +1171,7 @@ public synchronized Map topicGroups() { } } if (!sourceTopics.isEmpty()) { - topicGroups.put(new Subtopology(entry.getKey(), namedTopology), new TopicsInfo( + topicGroups.put(new Subtopology(entry.getKey(), topologyName), new TopicsInfo( Collections.unmodifiableSet(sinkTopics), Collections.unmodifiableSet(sourceTopics), Collections.unmodifiableMap(repartitionTopics), @@ -1217,39 +1244,26 @@ private InternalTopicConfig createChangelogTopicConfig(fi } } - public synchronized Pattern earliestResetTopicsPattern() { - return resetTopicsPattern(earliestResetTopics, earliestResetPatterns); - } - - public synchronized Pattern latestResetTopicsPattern() { - return resetTopicsPattern(latestResetTopics, latestResetPatterns); + public boolean hasOffsetResetOverrides() { + return !(earliestResetTopics.isEmpty() && earliestResetPatterns.isEmpty() + && latestResetTopics.isEmpty() && latestResetPatterns.isEmpty()); } - private Pattern resetTopicsPattern(final Set resetTopics, - final Set resetPatterns) { - final List topics = maybeDecorateInternalSourceTopics(resetTopics); - - return buildPattern(topics, resetPatterns); - } - - private static Pattern buildPattern(final Collection sourceTopics, - final Collection sourcePatterns) { - final StringBuilder builder = new StringBuilder(); - - for (final String topic : sourceTopics) { - builder.append(topic).append("|"); - } - - for (final Pattern sourcePattern : sourcePatterns) { - builder.append(sourcePattern.pattern()).append("|"); - } - - if (builder.length() > 0) { - builder.setLength(builder.length() - 1); - return Pattern.compile(builder.toString()); + public OffsetResetStrategy offsetResetStrategy(final String topic) { + if (maybeDecorateInternalSourceTopics(earliestResetTopics).contains(topic) || + earliestResetPatterns.stream().anyMatch(p -> p.matcher(topic).matches())) { + return EARLIEST; + } else if (maybeDecorateInternalSourceTopics(latestResetTopics).contains(topic) || + latestResetPatterns.stream().anyMatch(p -> p.matcher(topic).matches())) { + return LATEST; + } else if (maybeDecorateInternalSourceTopics(sourceTopicNames).contains(topic) + || !hasNamedTopology() + || (usesPatternSubscription() && Pattern.compile(sourceTopicPatternString).matcher(topic).matches())) { + return NONE; + } else { + // return null if the topic wasn't found at all while using NamedTopologies as it's likely in another + return null; } - - return EMPTY_ZERO_LENGTH_PATTERN; } public Map> stateStoreNameToSourceTopics() { @@ -1322,19 +1336,20 @@ public String decoratePseudoTopic(final String topic) { private String decorateTopic(final String topic) { if (applicationId == null) { throw new TopologyException("there are internal topics and " - + "applicationId hasn't been set. Call " - + "setApplicationId first"); + + "applicationId hasn't been set. Call " + + "setApplicationId first"); + } + if (hasNamedTopology()) { + return applicationId + "-" + topologyName + "-" + topic; + } else { + return applicationId + "-" + topic; } - - return applicationId + "-" + topic; } void initializeSubscription() { if (usesPatternSubscription()) { log.debug("Found pattern subscribed source topics, initializing consumer's subscription pattern."); - final List allSourceTopics = maybeDecorateInternalSourceTopics(sourceTopicNames); - Collections.sort(allSourceTopics); - sourceTopicPattern = buildPattern(allSourceTopics, nodeToSourcePatterns.values()); + sourceTopicPatternString = buildSourceTopicsPatternString(); } else { log.debug("No source topics using pattern subscription found, initializing consumer's subscription collection."); sourceTopicCollection = maybeDecorateInternalSourceTopics(sourceTopicNames); @@ -1342,6 +1357,27 @@ void initializeSubscription() { } } + private String buildSourceTopicsPatternString() { + final List allSourceTopics = maybeDecorateInternalSourceTopics(sourceTopicNames); + Collections.sort(allSourceTopics); + + final StringBuilder builder = new StringBuilder(); + + for (final String topic : allSourceTopics) { + builder.append(topic).append("|"); + } + + for (final Pattern sourcePattern : nodeToSourcePatterns.values()) { + builder.append(sourcePattern.pattern()).append("|"); + } + + if (builder.length() > 0) { + builder.setLength(builder.length() - 1); + } + + return builder.toString(); + } + boolean usesPatternSubscription() { return !nodeToSourcePatterns.isEmpty(); } @@ -1350,12 +1386,21 @@ synchronized Collection sourceTopicCollection() { return sourceTopicCollection; } - synchronized Pattern sourceTopicPattern() { - return sourceTopicPattern; + synchronized String sourceTopicsPatternString() { + // With a NamedTopology, it may be that this topology does not use pattern subscription but another one does + // in which case we would need to initialize the pattern string where we would otherwise have not + if (sourceTopicPatternString == null && hasNamedTopology()) { + sourceTopicPatternString = buildSourceTopicsPatternString(); + } + return sourceTopicPatternString; } public boolean hasNoNonGlobalTopology() { - return !usesPatternSubscription() && sourceTopicCollection().isEmpty(); + return nodeToSourcePatterns.isEmpty() && sourceTopicNames.isEmpty(); + } + + public boolean hasGlobalStores() { + return !globalStateStores.isEmpty(); } private boolean isGlobalSource(final String nodeName) { @@ -1369,7 +1414,7 @@ private boolean isGlobalSource(final String nodeName) { } public TopologyDescription describe() { - final TopologyDescription description = new TopologyDescription(); + final TopologyDescription description = new TopologyDescription(topologyName); for (final Map.Entry> nodeGroup : makeNodeGroups().entrySet()) { @@ -1905,6 +1950,15 @@ public int compare(final TopologyDescription.Subtopology subtopology1, public final static class TopologyDescription implements org.apache.kafka.streams.TopologyDescription { private final TreeSet subtopologies = new TreeSet<>(SUBTOPOLOGY_COMPARATOR); private final TreeSet globalStores = new TreeSet<>(GLOBALSTORE_COMPARATOR); + private final String namedTopology; + + public TopologyDescription() { + this(null); + } + + public TopologyDescription(final String namedTopology) { + this.namedTopology = namedTopology; + } public void addSubtopology(final TopologyDescription.Subtopology subtopology) { subtopologies.add(subtopology); @@ -1927,7 +1981,12 @@ public Set globalStores() { @Override public String toString() { final StringBuilder sb = new StringBuilder(); - sb.append("Topologies:\n "); + + if (namedTopology == null) { + sb.append("Topologies:\n "); + } else { + sb.append("Topology - ").append(namedTopology).append(":\n "); + } final TopologyDescription.Subtopology[] sortedSubtopologies = subtopologies.descendingSet().toArray(new TopologyDescription.Subtopology[0]); final TopologyDescription.GlobalStore[] sortedGlobalStores = @@ -2039,13 +2098,22 @@ private void updateSubscribedTopics(final Set topics, final String logPr setRegexMatchedTopicToStateStore(); } + /** + * @return a copy of all source topic names, including the application id and named topology prefix if applicable + */ public synchronized List fullSourceTopicNames() { - return maybeDecorateInternalSourceTopics(sourceTopicNames); + return new ArrayList<>(maybeDecorateInternalSourceTopics(sourceTopicNames)); } - public boolean hasNamedTopologies() { - // TODO KAFKA-12648: covered by Pt. 2 - return false; + /** + * @return a copy of the string representation of any pattern subscribed source nodes + */ + public synchronized List allSourcePatternStrings() { + return nodeToSourcePatterns.values().stream().map(Pattern::pattern).collect(Collectors.toList()); + } + + public boolean hasNamedTopology() { + return topologyName != null; } // following functions are for test only diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextUtils.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextUtils.java index 33547c9a9f9cf..6984922070c26 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextUtils.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorContextUtils.java @@ -56,13 +56,13 @@ public static StreamsMetricsImpl getMetricsImpl(final StateStoreContext context) public static String changelogFor(final ProcessorContext context, final String storeName) { return context instanceof InternalProcessorContext ? ((InternalProcessorContext) context).changelogFor(storeName) - : ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName); + : ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName, context.taskId().namedTopology()); } public static String changelogFor(final StateStoreContext context, final String storeName) { return context instanceof InternalProcessorContext ? ((InternalProcessorContext) context).changelogFor(storeName) - : ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName); + : ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName, context.taskId().namedTopology()); } public static InternalProcessorContext asInternalProcessorContext(final ProcessorContext context) { diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java index 625dffee3cf48..b507aea549a9d 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java @@ -162,8 +162,12 @@ public String toString() { private TaskType taskType; - public static String storeChangelogTopic(final String applicationId, final String storeName) { - return applicationId + "-" + storeName + STATE_CHANGELOG_TOPIC_SUFFIX; + public static String storeChangelogTopic(final String applicationId, final String storeName, final String namedTopology) { + if (namedTopology == null) { + return applicationId + "-" + storeName + STATE_CHANGELOG_TOPIC_SUFFIX; + } else { + return applicationId + "-" + namedTopology + "-" + storeName + STATE_CHANGELOG_TOPIC_SUFFIX; + } } /** diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorTopology.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorTopology.java index 8f1a460e54cbd..d2383c7adbb1c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorTopology.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorTopology.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; public class ProcessorTopology { private final Logger log = LoggerFactory.getLogger(ProcessorTopology.class); @@ -38,7 +37,6 @@ public class ProcessorTopology { private final Map> sinksByTopic; private final Set terminalNodes; private final List stateStores; - private final Set stateStoreNames; private final Set repartitionTopics; // the following contains entries for the entire topology, eg stores that do not belong to this ProcessorTopology @@ -56,7 +54,6 @@ public ProcessorTopology(final List> processorNodes, this.sourceNodesByTopic = new HashMap<>(sourceNodesByTopic); this.sinksByTopic = Collections.unmodifiableMap(sinksByTopic); this.stateStores = Collections.unmodifiableList(stateStores); - stateStoreNames = stateStores.stream().map(StateStore::name).collect(Collectors.toSet()); this.globalStateStores = Collections.unmodifiableList(globalStateStores); this.storeToChangelogTopic = Collections.unmodifiableMap(storeToChangelogTopic); this.repartitionTopics = Collections.unmodifiableSet(repartitionTopics); @@ -106,10 +103,6 @@ public List stateStores() { return stateStores; } - public boolean hasStore(final String storeName) { - return stateStoreNames.contains(storeName); - } - public List globalStateStores() { return Collections.unmodifiableList(globalStateStores); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopics.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopics.java index 8656198c966d4..24c6e81cee9bc 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopics.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopics.java @@ -40,18 +40,18 @@ public class RepartitionTopics { private final InternalTopicManager internalTopicManager; - private final InternalTopologyBuilder internalTopologyBuilder; + private final TopologyMetadata topologyMetadata; private final Cluster clusterMetadata; private final CopartitionedTopicsEnforcer copartitionedTopicsEnforcer; private final Logger log; private final Map topicPartitionInfos = new HashMap<>(); - public RepartitionTopics(final InternalTopologyBuilder internalTopologyBuilder, + public RepartitionTopics(final TopologyMetadata topologyMetadata, final InternalTopicManager internalTopicManager, final CopartitionedTopicsEnforcer copartitionedTopicsEnforcer, final Cluster clusterMetadata, final String logPrefix) { - this.internalTopologyBuilder = internalTopologyBuilder; + this.topologyMetadata = topologyMetadata; this.internalTopicManager = internalTopicManager; this.clusterMetadata = clusterMetadata; this.copartitionedTopicsEnforcer = copartitionedTopicsEnforcer; @@ -60,13 +60,13 @@ public RepartitionTopics(final InternalTopologyBuilder internalTopologyBuilder, } public void setup() { - final Map topicGroups = internalTopologyBuilder.topicGroups(); + final Map topicGroups = topologyMetadata.topicGroups(); final Map repartitionTopicMetadata = computeRepartitionTopicConfig(topicGroups, clusterMetadata); // ensure the co-partitioning topics within the group have the same number of partitions, // and enforce the number of partitions for those repartition topics to be the same if they // are co-partitioned as well. - ensureCopartitioning(internalTopologyBuilder.copartitionGroups(), repartitionTopicMetadata, clusterMetadata); + ensureCopartitioning(topologyMetadata.copartitionGroups(), repartitionTopicMetadata, clusterMetadata); // make sure the repartition source topics exist with the right number of partitions, // create these topics if necessary diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTaskCreator.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTaskCreator.java index 3f0dd22f1bda2..dc1a7a65fb30f 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTaskCreator.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StandbyTaskCreator.java @@ -34,7 +34,7 @@ import java.util.Set; class StandbyTaskCreator { - private final InternalTopologyBuilder builder; + private final TopologyMetadata topologyMetadata; private final StreamsConfig config; private final StreamsMetricsImpl streamsMetrics; private final StateDirectory stateDirectory; @@ -43,14 +43,14 @@ class StandbyTaskCreator { private final Logger log; private final Sensor createTaskSensor; - StandbyTaskCreator(final InternalTopologyBuilder builder, + StandbyTaskCreator(final TopologyMetadata topologyMetadata, final StreamsConfig config, final StreamsMetricsImpl streamsMetrics, final StateDirectory stateDirectory, final ChangelogReader storeChangelogReader, final String threadId, final Logger log) { - this.builder = builder; + this.topologyMetadata = topologyMetadata; this.config = config; this.streamsMetrics = streamsMetrics; this.stateDirectory = stateDirectory; @@ -73,8 +73,7 @@ Collection createTasks(final Map> tasksToBeCre for (final Map.Entry> newTaskAndPartitions : tasksToBeCreated.entrySet()) { final TaskId taskId = newTaskAndPartitions.getKey(); final Set partitions = newTaskAndPartitions.getValue(); - - final ProcessorTopology topology = builder.buildSubtopology(taskId.subtopology()); + final ProcessorTopology topology = topologyMetadata.buildSubtopology(taskId); if (topology.hasStateWithChangelogs()) { final ProcessorStateManager stateManager = new ProcessorStateManager( @@ -120,7 +119,7 @@ StandbyTask createStandbyTaskFromActive(final StreamTask streamTask, return createStandbyTask( streamTask.id(), inputPartitions, - builder.buildSubtopology(streamTask.id.subtopology()), + topologyMetadata.buildSubtopology(streamTask.id), stateManager, context ); @@ -148,14 +147,6 @@ StandbyTask createStandbyTask(final TaskId taskId, return task; } - public InternalTopologyBuilder builder() { - return builder; - } - - public StateDirectory stateDirectory() { - return stateDirectory; - } - private LogContext getLogContext(final TaskId taskId) { final String threadIdPrefix = String.format("stream-thread [%s] ", Thread.currentThread().getName()); final String logPrefix = threadIdPrefix + String.format("%s [%s] ", "standby-task", taskId); diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StateDirectory.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StateDirectory.java index 90a7f80fe5deb..c0be5b356b662 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StateDirectory.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StateDirectory.java @@ -23,34 +23,34 @@ import org.apache.kafka.streams.errors.StreamsException; import org.apache.kafka.streams.processor.TaskId; -import java.io.FileFilter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; +import java.io.FileFilter; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; -import java.nio.file.Path; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.apache.kafka.streams.processor.internals.StateManagerUtil.CHECKPOINT_FILE_NAME; import static org.apache.kafka.streams.processor.internals.StateManagerUtil.parseTaskDirectoryName; @@ -146,12 +146,6 @@ public StateDirectory(final StreamsConfig config, final Time time, final boolean } } - public StateDirectory(final StreamsConfig config, final Time time, final boolean hasPersistentStores) { - // TODO KAFKA-12648: Explicitly set hasNamedTopology in each test and remove this constructor - // Will be done in Pt. 2 as we want some of these integration tests to use named topologies - this(config, time, hasPersistentStores, false); - } - private void configurePermissions(final File file) { final Path path = file.toPath(); if (path.getFileSystem().supportedFileAttributeViews().contains("posix")) { diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java index 224733e2d840b..c4ba999a7a497 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java @@ -296,7 +296,7 @@ public boolean isRunning() { private final Consumer mainConsumer; private final Consumer restoreConsumer; private final Admin adminClient; - private final InternalTopologyBuilder builder; + private final TopologyMetadata topologyMetadata; private final java.util.function.Consumer cacheResizer; private java.util.function.Consumer streamsUncaughtExceptionHandler; @@ -305,7 +305,7 @@ public boolean isRunning() { private final AtomicLong cacheResizeSize; private final AtomicBoolean leaveGroupRequested; - public static StreamThread create(final InternalTopologyBuilder builder, + public static StreamThread create(final TopologyMetadata topologyMetadata, final StreamsConfig config, final KafkaClientSupplier clientSupplier, final Admin adminClient, @@ -347,7 +347,7 @@ public static StreamThread create(final InternalTopologyBuilder builder, final ThreadCache cache = new ThreadCache(logContext, cacheSizeBytes, streamsMetrics); final ActiveTaskCreator activeTaskCreator = new ActiveTaskCreator( - builder, + topologyMetadata, config, streamsMetrics, stateDirectory, @@ -360,7 +360,7 @@ public static StreamThread create(final InternalTopologyBuilder builder, log ); final StandbyTaskCreator standbyTaskCreator = new StandbyTaskCreator( - builder, + topologyMetadata, config, streamsMetrics, stateDirectory, @@ -376,7 +376,7 @@ public static StreamThread create(final InternalTopologyBuilder builder, streamsMetrics, activeTaskCreator, standbyTaskCreator, - builder, + topologyMetadata, adminClient, stateDirectory, processingMode(config) @@ -390,7 +390,7 @@ public static StreamThread create(final InternalTopologyBuilder builder, final String originalReset = (String) consumerConfigs.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG); // If there are any overrides, we never fall through to the consumer, but only handle offset management ourselves. - if (!builder.latestResetTopicsPattern().pattern().isEmpty() || !builder.earliestResetTopicsPattern().pattern().isEmpty()) { + if (topologyMetadata.hasOffsetResetOverrides()) { consumerConfigs.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none"); } @@ -409,7 +409,7 @@ public static StreamThread create(final InternalTopologyBuilder builder, originalReset, taskManager, streamsMetrics, - builder, + topologyMetadata, threadId, logContext, referenceContainer.assignmentErrorCode, @@ -469,7 +469,7 @@ public StreamThread(final Time time, final String originalReset, final TaskManager taskManager, final StreamsMetricsImpl streamsMetrics, - final InternalTopologyBuilder builder, + final TopologyMetadata topologyMetadata, final String threadId, final LogContext logContext, final AtomicInteger assignmentErrorCode, @@ -509,7 +509,7 @@ public StreamThread(final Time time, ThreadMetrics.closeTaskSensor(threadId, streamsMetrics); this.time = time; - this.builder = builder; + this.topologyMetadata = topologyMetadata; this.logPrefix = logContext.logPrefix(); this.log = logContext.logger(StreamThread.class); this.rebalanceListener = new StreamsRebalanceListener(time, taskManager, this, this.log, this.assignmentErrorCode); @@ -683,10 +683,10 @@ private void handleTaskMigrated(final TaskMigratedException e) { } private void subscribeConsumer() { - if (builder.usesPatternSubscription()) { - mainConsumer.subscribe(builder.sourceTopicPattern(), rebalanceListener); + if (topologyMetadata.usesPatternSubscription()) { + mainConsumer.subscribe(topologyMetadata.sourceTopicPattern(), rebalanceListener); } else { - mainConsumer.subscribe(builder.sourceTopicCollection(), rebalanceListener); + mainConsumer.subscribe(topologyMetadata.sourceTopicCollection(), rebalanceListener); } } @@ -944,18 +944,24 @@ private void resetOffsets(final Set partitions, final Exception final Set notReset = new HashSet<>(); for (final TopicPartition partition : partitions) { - if (builder.earliestResetTopicsPattern().matcher(partition.topic()).matches()) { - addToResetList(partition, seekToBeginning, "Setting topic '{}' to consume from {} offset", "earliest", loggedTopics); - } else if (builder.latestResetTopicsPattern().matcher(partition.topic()).matches()) { - addToResetList(partition, seekToEnd, "Setting topic '{}' to consume from {} offset", "latest", loggedTopics); - } else { - if ("earliest".equals(originalReset)) { - addToResetList(partition, seekToBeginning, "No custom setting defined for topic '{}' using original config '{}' for offset reset", "earliest", loggedTopics); - } else if ("latest".equals(originalReset)) { - addToResetList(partition, seekToEnd, "No custom setting defined for topic '{}' using original config '{}' for offset reset", "latest", loggedTopics); - } else { - notReset.add(partition); - } + switch (topologyMetadata.offsetResetStrategy(partition.topic())) { + case EARLIEST: + addToResetList(partition, seekToBeginning, "Setting topic '{}' to consume from {} offset", "earliest", loggedTopics); + break; + case LATEST: + addToResetList(partition, seekToEnd, "Setting topic '{}' to consume from {} offset", "latest", loggedTopics); + break; + case NONE: + if ("earliest".equals(originalReset)) { + addToResetList(partition, seekToBeginning, "No custom setting defined for topic '{}' using original config '{}' for offset reset", "earliest", loggedTopics); + } else if ("latest".equals(originalReset)) { + addToResetList(partition, seekToEnd, "No custom setting defined for topic '{}' using original config '{}' for offset reset", "latest", loggedTopics); + } else { + notReset.add(partition); + } + break; + default: + throw new IllegalStateException("Unable to locate topic " + partition.topic() + " in the topology"); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsMetadataState.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsMetadataState.java index d8fdbde5aa5be..8cc10166d794e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsMetadataState.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsMetadataState.java @@ -47,16 +47,16 @@ */ public class StreamsMetadataState { public static final HostInfo UNKNOWN_HOST = HostInfo.unavailable(); - private final InternalTopologyBuilder builder; + private final TopologyMetadata topologyMetadata; private final Set globalStores; private final HostInfo thisHost; private List allMetadata = Collections.emptyList(); private Cluster clusterMetadata; private final AtomicReference localMetadata = new AtomicReference<>(null); - public StreamsMetadataState(final InternalTopologyBuilder builder, final HostInfo thisHost) { - this.builder = builder; - this.globalStores = builder.globalStateStores().keySet(); + public StreamsMetadataState(final TopologyMetadata topologyMetadata, final HostInfo thisHost) { + this.topologyMetadata = topologyMetadata; + this.globalStores = this.topologyMetadata.globalStateStores().keySet(); this.thisHost = thisHost; } @@ -111,8 +111,8 @@ public synchronized Collection getAllMetadataForStore(final Str return allMetadata; } - final Collection sourceTopics = builder.sourceTopicsForStore(storeName); - if (sourceTopics == null) { + final Collection sourceTopics = topologyMetadata.sourceTopicsForStore(storeName); + if (sourceTopics.isEmpty()) { return Collections.emptyList(); } @@ -241,7 +241,7 @@ private void rebuildMetadata(final Map> activePart } final List rebuiltMetadata = new ArrayList<>(); - final Map> storeToSourceTopics = builder.stateStoreNameToSourceTopics(); + final Map> storeToSourceTopics = topologyMetadata.stateStoreNameToSourceTopics(); Stream.concat(activePartitionHostMap.keySet().stream(), standbyPartitionHostMap.keySet().stream()) .distinct() .forEach(hostInfo -> { @@ -309,7 +309,7 @@ private KeyQueryMetadata getKeyQueryMetadataForKey(final String storeName, } private SourceTopicsInfo getSourceTopicsInfo(final String storeName) { - final List sourceTopics = new ArrayList<>(builder.sourceTopicsForStore(storeName)); + final List sourceTopics = new ArrayList<>(topologyMetadata.sourceTopicsForStore(storeName)); if (sourceTopics.isEmpty()) { return null; } @@ -322,7 +322,7 @@ private boolean isInitialized() { } public String getStoreForChangelogTopic(final String topicName) { - return builder.getChangelogTopicToStore().get(topicName); + return topologyMetadata.getStoreForChangelogTopic(topicName); } private class SourceTopicsInfo { diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java index 27674290164c2..bb5e2d2330e4c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java @@ -370,7 +370,8 @@ public GroupAssignment assign(final Cluster metadata, final GroupSubscription gr // construct the assignment of tasks to clients - final Map topicGroups = taskManager.builder().topicGroups(); + final Map topicGroups = taskManager.topologyMetadata().topicGroups(); + final Set allSourceTopics = new HashSet<>(); final Map> sourceTopicsByGroup = new HashMap<>(); for (final Map.Entry entry : topicGroups.entrySet()) { @@ -476,7 +477,7 @@ private boolean checkMetadataVersions(final int minReceivedMetadataVersion, private Map prepareRepartitionTopics(final Cluster metadata) { final RepartitionTopics repartitionTopics = new RepartitionTopics( - taskManager.builder(), + taskManager.topologyMetadata(), internalTopicManager, copartitionedTopicsEnforcer, metadata, diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java index 7f32526683eaf..509c882d12483 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TaskManager.java @@ -79,7 +79,7 @@ public class TaskManager { private final ChangelogReader changelogReader; private final UUID processId; private final String logPrefix; - private final InternalTopologyBuilder builder; + private final TopologyMetadata topologyMetadata; private final Admin adminClient; private final StateDirectory stateDirectory; private final StreamThread.ProcessingMode processingMode; @@ -101,7 +101,7 @@ public class TaskManager { final StreamsMetricsImpl streamsMetrics, final ActiveTaskCreator activeTaskCreator, final StandbyTaskCreator standbyTaskCreator, - final InternalTopologyBuilder builder, + final TopologyMetadata topologyMetadata, final Admin adminClient, final StateDirectory stateDirectory, final StreamThread.ProcessingMode processingMode) { @@ -109,11 +109,11 @@ public class TaskManager { this.changelogReader = changelogReader; this.processId = processId; this.logPrefix = logPrefix; - this.builder = builder; + this.topologyMetadata = topologyMetadata; this.adminClient = adminClient; this.stateDirectory = stateDirectory; this.processingMode = processingMode; - this.tasks = new Tasks(logPrefix, builder, streamsMetrics, activeTaskCreator, standbyTaskCreator); + this.tasks = new Tasks(logPrefix, topologyMetadata, streamsMetrics, activeTaskCreator, standbyTaskCreator); final LogContext logContext = new LogContext(logPrefix); log = logContext.logger(getClass()); @@ -128,8 +128,8 @@ public UUID processId() { return processId; } - InternalTopologyBuilder builder() { - return builder; + public TopologyMetadata topologyMetadata() { + return topologyMetadata; } boolean isRebalanceInProgress() { @@ -137,7 +137,7 @@ boolean isRebalanceInProgress() { } void handleRebalanceStart(final Set subscribedTopics) { - builder.addSubscribedTopicsFromMetadata(subscribedTopics, logPrefix); + topologyMetadata.addSubscribedTopicsFromMetadata(subscribedTopics, logPrefix); tryToLockAllNonEmptyTaskDirectories(); @@ -267,7 +267,7 @@ public void handleAssignment(final Map> activeTasks, "\tExisting standby tasks: {}", activeTasks.keySet(), standbyTasks.keySet(), activeTaskIds(), standbyTaskIds()); - builder.addSubscribedTopicsFromAssignment( + topologyMetadata.addSubscribedTopicsFromAssignment( activeTasks.values().stream().flatMap(Collection::stream).collect(Collectors.toList()), logPrefix ); diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/Tasks.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/Tasks.java index 4193deb6f7d70..2f1394535e17c 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/Tasks.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/Tasks.java @@ -37,7 +37,7 @@ class Tasks { private final Logger log; - private final InternalTopologyBuilder builder; + private final TopologyMetadata topologyMetadata; private final StreamsMetricsImpl streamsMetrics; private final Map allTasksPerId = new TreeMap<>(); @@ -66,7 +66,7 @@ class Tasks { private Consumer mainConsumer; Tasks(final String logPrefix, - final InternalTopologyBuilder builder, + final TopologyMetadata topologyMetadata, final StreamsMetricsImpl streamsMetrics, final ActiveTaskCreator activeTaskCreator, final StandbyTaskCreator standbyTaskCreator) { @@ -74,7 +74,7 @@ class Tasks { final LogContext logContext = new LogContext(logPrefix); log = logContext.logger(getClass()); - this.builder = builder; + this.topologyMetadata = topologyMetadata; this.streamsMetrics = streamsMetrics; this.activeTaskCreator = activeTaskCreator; this.standbyTaskCreator = standbyTaskCreator; @@ -168,7 +168,7 @@ void updateInputPartitionsAndResume(final Task task, final Set t activeTasksPerPartition.put(topicPartition, task); } } - task.updateInputPartitions(topicPartitions, builder.nodeToSourceTopics()); + task.updateInputPartitions(topicPartitions, topologyMetadata.nodeToSourceTopics(task.id())); } task.resume(); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TopologyMetadata.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TopologyMetadata.java index 2b7392ab5bbb7..85cf09ae6e2e0 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/TopologyMetadata.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/TopologyMetadata.java @@ -16,10 +16,304 @@ */ package org.apache.kafka.streams.processor.internals; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.errors.TopologyException; +import org.apache.kafka.streams.processor.StateStore; +import org.apache.kafka.streams.processor.TaskId; +import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// TODO KAFKA-12648: +// 1) synchronize on these methods instead of individual InternalTopologyBuilder methods, where applicable public class TopologyMetadata { - //TODO KAFKA-12648: the TopologyMetadata class is filled in by Pt. 2 (PR #10683) + private final Logger log = LoggerFactory.getLogger(TopologyMetadata.class); + + // the "__" (double underscore) string is not allowed for topology names, so it's safe to use to indicate + // that it's not a named topology + private static final String UNNAMED_TOPOLOGY = "__UNNAMED_TOPOLOGY__"; + private static final Pattern EMPTY_ZERO_LENGTH_PATTERN = Pattern.compile(""); + + private final StreamsConfig config; + private final SortedMap builders; // Keep sorted by topology name for readability + + private ProcessorTopology globalTopology; + private Map globalStateStores = new HashMap<>(); + final Set allInputTopics = new HashSet<>(); + + public TopologyMetadata(final InternalTopologyBuilder builder, final StreamsConfig config) { + this.config = config; + builders = new TreeMap<>(); + if (builder.hasNamedTopology()) { + builders.put(builder.topologyName(), builder); + } else { + builders.put(UNNAMED_TOPOLOGY, builder); + } + } + + public TopologyMetadata(final SortedMap builders, final StreamsConfig config) { + this.config = config; + this.builders = builders; + if (builders.isEmpty()) { + log.debug("Building KafkaStreams app with no empty topology"); + } + } + + public int getNumStreamThreads(final StreamsConfig config) { + final int configuredNumStreamThreads = config.getInt(StreamsConfig.NUM_STREAM_THREADS_CONFIG); + + // If the application uses named topologies, it's possible to start up with no topologies at all and only add them later + if (builders.isEmpty()) { + if (configuredNumStreamThreads != 0) { + log.info("Overriding number of StreamThreads to zero for empty topology"); + } + return 0; + } + + // If there are named topologies but some are empty, this indicates a bug in user code + if (hasNamedTopologies()) { + if (hasNoNonGlobalTopology() && !hasGlobalTopology()) { + log.error("Detected a named topology with no input topics, a named topology may not be empty."); + throw new TopologyException("Topology has no stream threads and no global threads, " + + "must subscribe to at least one source topic or pattern."); + } + } else { + // If both the global and non-global topologies are empty, this indicates a bug in user code + if (hasNoNonGlobalTopology() && !hasGlobalTopology()) { + log.error("Topology with no input topics will create no stream threads and no global thread."); + throw new TopologyException("Topology has no stream threads and no global threads, " + + "must subscribe to at least one source topic or global table."); + } + } + + // Lastly we check for an empty non-global topology and override the threads to zero if set otherwise + if (configuredNumStreamThreads != 0 && hasNoNonGlobalTopology()) { + log.info("Overriding number of StreamThreads to zero for global-only topology"); + return 0; + } + + return configuredNumStreamThreads; + } + + public boolean hasNamedTopologies() { + // This includes the case of starting up with no named topologies at all + return !builders.containsKey(UNNAMED_TOPOLOGY); + } + + public boolean hasGlobalTopology() { + return evaluateConditionIsTrueForAnyBuilders(InternalTopologyBuilder::hasGlobalStores); + } + + public boolean hasNoNonGlobalTopology() { + return evaluateConditionIsTrueForAnyBuilders(InternalTopologyBuilder::hasNoNonGlobalTopology); + } + + public boolean hasPersistentStores() { + // If the app is using named topologies, there may not be any persistent state when it first starts up + // but a new NamedTopology may introduce it later, so we must return true + if (hasNamedTopologies()) { + return true; + } + return evaluateConditionIsTrueForAnyBuilders(InternalTopologyBuilder::hasPersistentStores); + } + + public boolean hasStore(final String name) { + return evaluateConditionIsTrueForAnyBuilders(b -> b.hasStore(name)); + } + + public boolean hasOffsetResetOverrides() { + return evaluateConditionIsTrueForAnyBuilders(InternalTopologyBuilder::hasOffsetResetOverrides); + } + + public OffsetResetStrategy offsetResetStrategy(final String topic) { + for (final InternalTopologyBuilder builder : builders.values()) { + final OffsetResetStrategy resetStrategy = builder.offsetResetStrategy(topic); + if (resetStrategy != null) { + return resetStrategy; + } + } + return null; + } + + Collection sourceTopicCollection() { + final List sourceTopics = new ArrayList<>(); + applyToEachBuilder(b -> sourceTopics.addAll(b.sourceTopicCollection())); + return sourceTopics; + } + + Pattern sourceTopicPattern() { + final StringBuilder patternBuilder = new StringBuilder(); + + applyToEachBuilder(b -> { + final String patternString = b.sourceTopicsPatternString(); + if (patternString.length() > 0) { + patternBuilder.append(patternString).append("|"); + } + }); + + if (patternBuilder.length() > 0) { + patternBuilder.setLength(patternBuilder.length() - 1); + return Pattern.compile(patternBuilder.toString()); + } else { + return EMPTY_ZERO_LENGTH_PATTERN; + } + } + + public boolean usesPatternSubscription() { + return evaluateConditionIsTrueForAnyBuilders(InternalTopologyBuilder::usesPatternSubscription); + } + + // Can be empty if app is started up with no Named Topologies, in order to add them on later + public boolean isEmpty() { + return builders.isEmpty(); + } + + public String topologyDescriptionString() { + if (isEmpty()) { + return ""; + } + final StringBuilder sb = new StringBuilder(); + + applyToEachBuilder(b -> { + sb.append(b.describe().toString()); + }); + + return sb.toString(); + } + + public final void buildAndRewriteTopology() { + applyToEachBuilder(builder -> { + builder.rewriteTopology(config); + builder.buildTopology(); + + // As we go, check each topology for overlap in the set of input topics/patterns + final int numInputTopics = allInputTopics.size(); + final List inputTopics = builder.fullSourceTopicNames(); + final Collection inputPatterns = builder.allSourcePatternStrings(); + + final int numNewInputTopics = inputTopics.size() + inputPatterns.size(); + allInputTopics.addAll(inputTopics); + allInputTopics.addAll(inputPatterns); + if (allInputTopics.size() != numInputTopics + numNewInputTopics) { + inputTopics.retainAll(allInputTopics); + inputPatterns.retainAll(allInputTopics); + inputTopics.addAll(inputPatterns); + log.error("Tried to add the NamedTopology {} but it had overlap with other input topics: {}", builder.topologyName(), inputTopics); + throw new TopologyException("Named Topologies may not subscribe to the same input topics or patterns"); + } + + final ProcessorTopology globalTopology = builder.buildGlobalStateTopology(); + if (globalTopology != null) { + if (builder.topologyName() != null) { + throw new IllegalStateException("Global state stores are not supported with Named Topologies"); + } else if (this.globalTopology == null) { + this.globalTopology = globalTopology; + } else { + throw new IllegalStateException("Topology builder had global state, but global topology has already been set"); + } + } + globalStateStores.putAll(builder.globalStateStores()); + }); + } + + public ProcessorTopology buildSubtopology(final TaskId task) { + return lookupBuilderForTask(task).buildSubtopology(task.subtopology()); + } + + public ProcessorTopology globalTaskTopology() { + if (hasNamedTopologies()) { + throw new IllegalStateException("Global state stores are not supported with Named Topologies"); + } + return globalTopology; + } + + public Map globalStateStores() { + return globalStateStores; + } + + public Map> stateStoreNameToSourceTopics() { + final Map> stateStoreNameToSourceTopics = new HashMap<>(); + applyToEachBuilder(b -> stateStoreNameToSourceTopics.putAll(b.stateStoreNameToSourceTopics())); + return stateStoreNameToSourceTopics; + } + + public String getStoreForChangelogTopic(final String topicName) { + for (final InternalTopologyBuilder builder : builders.values()) { + final String store = builder.getStoreForChangelogTopic(topicName); + if (store != null) { + return store; + } + } + log.warn("Unable to locate any store for topic {}", topicName); + return ""; + } + + public Collection sourceTopicsForStore(final String storeName) { + final List sourceTopics = new ArrayList<>(); + applyToEachBuilder(b -> sourceTopics.addAll(b.sourceTopicsForStore(storeName))); + return sourceTopics; + } + + public Map topicGroups() { + final Map topicGroups = new HashMap<>(); + applyToEachBuilder(b -> topicGroups.putAll(b.topicGroups())); + return topicGroups; + } + + public Map> nodeToSourceTopics(final TaskId task) { + return lookupBuilderForTask(task).nodeToSourceTopics(); + } + + void addSubscribedTopicsFromMetadata(final Set topics, final String logPrefix) { + applyToEachBuilder(b -> b.addSubscribedTopicsFromMetadata(topics, logPrefix)); + } + + void addSubscribedTopicsFromAssignment(final List partitions, final String logPrefix) { + applyToEachBuilder(b -> b.addSubscribedTopicsFromAssignment(partitions, logPrefix)); + } + + public Collection> copartitionGroups() { + final List> copartitionGroups = new ArrayList<>(); + applyToEachBuilder(b -> copartitionGroups.addAll(b.copartitionGroups())); + return copartitionGroups; + } + + private InternalTopologyBuilder lookupBuilderForTask(final TaskId task) { + return task.namedTopology() == null ? builders.get(UNNAMED_TOPOLOGY) : builders.get(task.namedTopology()); + } + + private boolean evaluateConditionIsTrueForAnyBuilders(final Function condition) { + for (final InternalTopologyBuilder builder : builders.values()) { + if (condition.apply(builder)) { + return true; + } + } + return false; + } + + private void applyToEachBuilder(final Consumer function) { + for (final InternalTopologyBuilder builder : builders.values()) { + function.accept(builder); + } + } public static class Subtopology { final int nodeGroupId; diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/namedtopology/KafkaStreamsNamedTopologyWrapper.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/namedtopology/KafkaStreamsNamedTopologyWrapper.java index e11a78fbf34e8..b3da0b8890c47 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/namedtopology/KafkaStreamsNamedTopologyWrapper.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/namedtopology/KafkaStreamsNamedTopologyWrapper.java @@ -16,27 +16,98 @@ */ package org.apache.kafka.streams.processor.internals.namedtopology; +import org.apache.kafka.common.annotation.InterfaceStability.Unstable; import org.apache.kafka.streams.KafkaClientSupplier; import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.errors.TopologyException; +import org.apache.kafka.streams.processor.internals.TopologyMetadata; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; +import java.util.TreeMap; +import java.util.stream.Collectors; +/** + * This is currently an internal and experimental feature for enabling certain kinds of topology upgrades. Use at + * your own risk. + * + * Status: basic architecture implemented but no actual upgrades are supported yet + * + * Note: some standard features of Kafka Streams are not yet supported with NamedTopologies. These include: + * - global state stores + * - interactive queries (IQ) + * - TopologyTestDriver (TTD) + */ +@Unstable public class KafkaStreamsNamedTopologyWrapper extends KafkaStreams { - //TODO It should be possible to start up streams with no NamedTopology (or regular Topology) at all, in the meantime we can just pass in an empty NamedTopology + final Map nameToTopology = new HashMap<>(); + + /** + * A Kafka Streams application with a single initial NamedTopology + */ public KafkaStreamsNamedTopologyWrapper(final NamedTopology topology, final Properties props, final KafkaClientSupplier clientSupplier) { - super(topology, props, clientSupplier); + this(Collections.singleton(topology), new StreamsConfig(props), clientSupplier); + } + + /** + * An empty Kafka Streams application that allows NamedTopologies to be added at a later point + */ + public KafkaStreamsNamedTopologyWrapper(final Properties props, final KafkaClientSupplier clientSupplier) { + this(Collections.emptyList(), new StreamsConfig(props), clientSupplier); + } + + /** + * A Kafka Streams application with a multiple initial NamedTopologies + * + * @throws IllegalArgumentException if any of the named topologies have the same name + * @throws TopologyException if multiple NamedTopologies subscribe to the same input topics or pattern + */ + public KafkaStreamsNamedTopologyWrapper(final Collection topologies, final Properties props, final KafkaClientSupplier clientSupplier) { + this(topologies, new StreamsConfig(props), clientSupplier); + } + + private KafkaStreamsNamedTopologyWrapper(final Collection topologies, final StreamsConfig config, final KafkaClientSupplier clientSupplier) { + super( + new TopologyMetadata( + topologies.stream().collect(Collectors.toMap( + NamedTopology::name, + NamedTopology::internalTopologyBuilder, + (v1, v2) -> { + throw new IllegalArgumentException("Topology names must be unique"); + }, + () -> new TreeMap<>())), + config), + config, + clientSupplier + ); + for (final NamedTopology topology : topologies) { + nameToTopology.put(topology.name(), topology); + } } public NamedTopology getTopologyByName(final String name) { - throw new UnsupportedOperationException(); + if (nameToTopology.containsKey(name)) { + return nameToTopology.get(name); + } else { + throw new IllegalArgumentException("Unable to locate a NamedTopology called " + name); + } } public void addNamedTopology(final NamedTopology topology) { + nameToTopology.put(topology.name(), topology); throw new UnsupportedOperationException(); } - public void removeNamedTopology(final NamedTopology topology) { + public void removeNamedTopology(final String namedTopology) { throw new UnsupportedOperationException(); } + + public String getFullTopologyDescription() { + return topologyMetadata.topologyDescriptionString(); + } } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/namedtopology/NamedTopology.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/namedtopology/NamedTopology.java index c1588a930bdf8..17138d8727336 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/namedtopology/NamedTopology.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/namedtopology/NamedTopology.java @@ -17,6 +17,8 @@ package org.apache.kafka.streams.processor.internals.namedtopology; import org.apache.kafka.streams.Topology; +import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +35,7 @@ void setTopologyName(final String newTopologyName) { throw new IllegalStateException("Tried to set topologyName but the name was already set"); } name = newTopologyName; + internalTopologyBuilder.setTopologyName(name); } public String name() { @@ -42,4 +45,8 @@ public String name() { public List sourceTopics() { return super.internalTopologyBuilder.fullSourceTopicNames(); } + + InternalTopologyBuilder internalTopologyBuilder() { + return internalTopologyBuilder; + } } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/namedtopology/NamedTopologyStreamsBuilder.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/namedtopology/NamedTopologyStreamsBuilder.java index 291ece7c49a2a..5d3fad83732fd 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/namedtopology/NamedTopologyStreamsBuilder.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/namedtopology/NamedTopologyStreamsBuilder.java @@ -18,16 +18,23 @@ import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.Topology; +import org.apache.kafka.streams.processor.TaskId; import java.util.Properties; public class NamedTopologyStreamsBuilder extends StreamsBuilder { - final String topologyName; + /** + * @param topologyName any string representing your NamedTopology, all characters allowed except for "__" + * @throws IllegalArgumentException if the name contains the character sequence "__" + */ public NamedTopologyStreamsBuilder(final String topologyName) { super(); this.topologyName = topologyName; + if (topologyName.contains(TaskId.NAMED_TOPOLOGY_DELIMITER)) { + throw new IllegalArgumentException("The character sequence '__' is not allowed in a NamedTopology, please select a new name"); + } } public synchronized NamedTopology buildNamedTopology(final Properties props) { diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java index f2349fcde61a8..4dadff9369256 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/CachingWindowStore.java @@ -84,7 +84,7 @@ public void init(final StateStoreContext context, final StateStore root) { private void initInternal(final InternalProcessorContext context) { this.context = context; - final String topic = ProcessorStateManager.storeChangelogTopic(context.applicationId(), name()); + final String topic = ProcessorStateManager.storeChangelogTopic(context.applicationId(), name(), context.taskId().namedTopology()); bytesSerdes = new StateSerdes<>( topic, diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryTimeOrderedKeyValueBuffer.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryTimeOrderedKeyValueBuffer.java index 39fc9bd2abef1..af7e6caba4142 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryTimeOrderedKeyValueBuffer.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/InMemoryTimeOrderedKeyValueBuffer.java @@ -227,7 +227,7 @@ private void init(final StateStore root) { ); context.register(root, (RecordBatchingStateRestoreCallback) this::restoreBatch); - changelogTopic = ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName); + changelogTopic = ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName, context.taskId().namedTopology()); updateBufferMetrics(); open = true; partition = context.taskId().partition(); diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java index e64cdf4652c0c..acb41871cc4cd 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStore.java @@ -29,6 +29,7 @@ import org.apache.kafka.streams.processor.internals.SerdeGetter; import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.StateStoreContext; +import org.apache.kafka.streams.processor.TaskId; import org.apache.kafka.streams.processor.api.Record; import org.apache.kafka.streams.processor.internals.InternalProcessorContext; import org.apache.kafka.streams.processor.internals.ProcessorContextUtils; @@ -77,8 +78,7 @@ public class MeteredKeyValueStore private Sensor e2eLatencySensor; private InternalProcessorContext context; private StreamsMetricsImpl streamsMetrics; - private final String threadId; - private String taskId; + private TaskId taskId; MeteredKeyValueStore(final KeyValueStore inner, final String metricsScope, @@ -87,7 +87,6 @@ public class MeteredKeyValueStore final Serde valueSerde) { super(inner); this.metricsScope = metricsScope; - threadId = Thread.currentThread().getName(); this.time = time != null ? time : Time.SYSTEM; this.keySerde = keySerde; this.valueSerde = valueSerde; @@ -98,13 +97,13 @@ public class MeteredKeyValueStore public void init(final ProcessorContext context, final StateStore root) { this.context = context instanceof InternalProcessorContext ? (InternalProcessorContext) context : null; - taskId = context.taskId().toString(); + taskId = context.taskId(); initStoreSerde(context); streamsMetrics = (StreamsMetricsImpl) context.metrics(); registerMetrics(); final Sensor restoreSensor = - StateStoreMetrics.restoreSensor(taskId, metricsScope, name(), streamsMetrics); + StateStoreMetrics.restoreSensor(taskId.toString(), metricsScope, name(), streamsMetrics); // register and possibly restore the state from the logs maybeMeasureLatency(() -> super.init(context, root), time, restoreSensor); @@ -114,29 +113,29 @@ public void init(final ProcessorContext context, public void init(final StateStoreContext context, final StateStore root) { this.context = context instanceof InternalProcessorContext ? (InternalProcessorContext) context : null; - taskId = context.taskId().toString(); + taskId = context.taskId(); initStoreSerde(context); streamsMetrics = (StreamsMetricsImpl) context.metrics(); registerMetrics(); final Sensor restoreSensor = - StateStoreMetrics.restoreSensor(taskId, metricsScope, name(), streamsMetrics); + StateStoreMetrics.restoreSensor(taskId.toString(), metricsScope, name(), streamsMetrics); // register and possibly restore the state from the logs maybeMeasureLatency(() -> super.init(context, root), time, restoreSensor); } private void registerMetrics() { - putSensor = StateStoreMetrics.putSensor(taskId, metricsScope, name(), streamsMetrics); - putIfAbsentSensor = StateStoreMetrics.putIfAbsentSensor(taskId, metricsScope, name(), streamsMetrics); - putAllSensor = StateStoreMetrics.putAllSensor(taskId, metricsScope, name(), streamsMetrics); - getSensor = StateStoreMetrics.getSensor(taskId, metricsScope, name(), streamsMetrics); - allSensor = StateStoreMetrics.allSensor(taskId, metricsScope, name(), streamsMetrics); - rangeSensor = StateStoreMetrics.rangeSensor(taskId, metricsScope, name(), streamsMetrics); - prefixScanSensor = StateStoreMetrics.prefixScanSensor(taskId, metricsScope, name(), streamsMetrics); - flushSensor = StateStoreMetrics.flushSensor(taskId, metricsScope, name(), streamsMetrics); - deleteSensor = StateStoreMetrics.deleteSensor(taskId, metricsScope, name(), streamsMetrics); - e2eLatencySensor = StateStoreMetrics.e2ELatencySensor(taskId, metricsScope, name(), streamsMetrics); + putSensor = StateStoreMetrics.putSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + putIfAbsentSensor = StateStoreMetrics.putIfAbsentSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + putAllSensor = StateStoreMetrics.putAllSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + getSensor = StateStoreMetrics.getSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + allSensor = StateStoreMetrics.allSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + rangeSensor = StateStoreMetrics.rangeSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + prefixScanSensor = StateStoreMetrics.prefixScanSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + flushSensor = StateStoreMetrics.flushSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + deleteSensor = StateStoreMetrics.deleteSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + e2eLatencySensor = StateStoreMetrics.e2ELatencySensor(taskId.toString(), metricsScope, name(), streamsMetrics); } protected Serde prepareValueSerdeForStore(final Serde valueSerde, final SerdeGetter getter) { @@ -151,7 +150,7 @@ private void initStoreSerde(final ProcessorContext context) { serdes = new StateSerdes<>( changelogTopic != null ? changelogTopic : - ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName), + ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName, taskId.namedTopology()), prepareKeySerde(keySerde, new SerdeGetter(context)), prepareValueSerdeForStore(valueSerde, new SerdeGetter(context)) ); @@ -163,7 +162,7 @@ private void initStoreSerde(final StateStoreContext context) { serdes = new StateSerdes<>( changelogTopic != null ? changelogTopic : - ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName), + ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName, taskId.namedTopology()), prepareKeySerde(keySerde, new SerdeGetter(context)), prepareValueSerdeForStore(valueSerde, new SerdeGetter(context)) ); @@ -311,7 +310,7 @@ public void close() { try { wrapped().close(); } finally { - streamsMetrics.removeAllStoreLevelSensorsAndMetrics(taskId, name()); + streamsMetrics.removeAllStoreLevelSensorsAndMetrics(taskId.toString(), name()); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java index 7434c0ef3c685..746ec85fc26a0 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredSessionStore.java @@ -28,6 +28,7 @@ import org.apache.kafka.streams.processor.internals.SerdeGetter; import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.StateStoreContext; +import org.apache.kafka.streams.processor.TaskId; import org.apache.kafka.streams.processor.api.Record; import org.apache.kafka.streams.processor.internals.InternalProcessorContext; import org.apache.kafka.streams.processor.internals.ProcessorContextUtils; @@ -58,7 +59,8 @@ public class MeteredSessionStore private Sensor removeSensor; private Sensor e2eLatencySensor; private InternalProcessorContext context; - private String taskId; + private TaskId taskId; + MeteredSessionStore(final SessionStore inner, final String metricsScope, @@ -77,13 +79,13 @@ public class MeteredSessionStore public void init(final ProcessorContext context, final StateStore root) { this.context = context instanceof InternalProcessorContext ? (InternalProcessorContext) context : null; + taskId = context.taskId(); initStoreSerde(context); - taskId = context.taskId().toString(); streamsMetrics = (StreamsMetricsImpl) context.metrics(); registerMetrics(); final Sensor restoreSensor = - StateStoreMetrics.restoreSensor(taskId, metricsScope, name(), streamsMetrics); + StateStoreMetrics.restoreSensor(taskId.toString(), metricsScope, name(), streamsMetrics); // register and possibly restore the state from the logs maybeMeasureLatency(() -> super.init(context, root), time, restoreSensor); @@ -93,24 +95,25 @@ public void init(final ProcessorContext context, public void init(final StateStoreContext context, final StateStore root) { this.context = context instanceof InternalProcessorContext ? (InternalProcessorContext) context : null; + taskId = context.taskId(); initStoreSerde(context); - taskId = context.taskId().toString(); streamsMetrics = (StreamsMetricsImpl) context.metrics(); registerMetrics(); final Sensor restoreSensor = - StateStoreMetrics.restoreSensor(taskId, metricsScope, name(), streamsMetrics); + StateStoreMetrics.restoreSensor(taskId.toString(), metricsScope, name(), streamsMetrics); // register and possibly restore the state from the logs maybeMeasureLatency(() -> super.init(context, root), time, restoreSensor); } private void registerMetrics() { - putSensor = StateStoreMetrics.putSensor(taskId, metricsScope, name(), streamsMetrics); - fetchSensor = StateStoreMetrics.fetchSensor(taskId, metricsScope, name(), streamsMetrics); - flushSensor = StateStoreMetrics.flushSensor(taskId, metricsScope, name(), streamsMetrics); - removeSensor = StateStoreMetrics.removeSensor(taskId, metricsScope, name(), streamsMetrics); - e2eLatencySensor = StateStoreMetrics.e2ELatencySensor(taskId, metricsScope, name(), streamsMetrics); + + putSensor = StateStoreMetrics.putSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + fetchSensor = StateStoreMetrics.fetchSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + flushSensor = StateStoreMetrics.flushSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + removeSensor = StateStoreMetrics.removeSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + e2eLatencySensor = StateStoreMetrics.e2ELatencySensor(taskId.toString(), metricsScope, name(), streamsMetrics); } @@ -120,9 +123,9 @@ private void initStoreSerde(final ProcessorContext context) { serdes = new StateSerdes<>( changelogTopic != null ? changelogTopic : - ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName), - WrappingNullableUtils.prepareKeySerde(keySerde, new SerdeGetter(context)), - WrappingNullableUtils.prepareValueSerde(valueSerde, new SerdeGetter(context)) + ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName, taskId.namedTopology()), + WrappingNullableUtils.prepareKeySerde(keySerde, new SerdeGetter(context)), + WrappingNullableUtils.prepareValueSerde(valueSerde, new SerdeGetter(context)) ); } @@ -132,9 +135,9 @@ private void initStoreSerde(final StateStoreContext context) { serdes = new StateSerdes<>( changelogTopic != null ? changelogTopic : - ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName), - WrappingNullableUtils.prepareKeySerde(keySerde, new SerdeGetter(context)), - WrappingNullableUtils.prepareValueSerde(valueSerde, new SerdeGetter(context)) + ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName, taskId.namedTopology()), + WrappingNullableUtils.prepareKeySerde(keySerde, new SerdeGetter(context)), + WrappingNullableUtils.prepareValueSerde(valueSerde, new SerdeGetter(context)) ); } @@ -377,7 +380,7 @@ public void close() { try { wrapped().close(); } finally { - streamsMetrics.removeAllStoreLevelSensorsAndMetrics(taskId, name()); + streamsMetrics.removeAllStoreLevelSensorsAndMetrics(taskId.toString(), name()); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java index 27cec1d9dc345..bc64db8214803 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/MeteredWindowStore.java @@ -28,6 +28,7 @@ import org.apache.kafka.streams.processor.internals.SerdeGetter; import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.StateStoreContext; +import org.apache.kafka.streams.processor.TaskId; import org.apache.kafka.streams.processor.api.Record; import org.apache.kafka.streams.processor.internals.InternalProcessorContext; import org.apache.kafka.streams.processor.internals.ProcessorContextUtils; @@ -60,8 +61,7 @@ public class MeteredWindowStore private Sensor flushSensor; private Sensor e2eLatencySensor; private InternalProcessorContext context; - private final String threadId; - private String taskId; + private TaskId taskId; MeteredWindowStore(final WindowStore inner, final long windowSizeMs, @@ -71,7 +71,6 @@ public class MeteredWindowStore final Serde valueSerde) { super(inner); this.windowSizeMs = windowSizeMs; - threadId = Thread.currentThread().getName(); this.metricsScope = metricsScope; this.time = time; this.keySerde = keySerde; @@ -83,13 +82,13 @@ public class MeteredWindowStore public void init(final ProcessorContext context, final StateStore root) { this.context = context instanceof InternalProcessorContext ? (InternalProcessorContext) context : null; + taskId = context.taskId(); initStoreSerde(context); streamsMetrics = (StreamsMetricsImpl) context.metrics(); - taskId = context.taskId().toString(); registerMetrics(); final Sensor restoreSensor = - StateStoreMetrics.restoreSensor(taskId, metricsScope, name(), streamsMetrics); + StateStoreMetrics.restoreSensor(taskId.toString(), metricsScope, name(), streamsMetrics); // register and possibly restore the state from the logs maybeMeasureLatency(() -> super.init(context, root), time, restoreSensor); @@ -99,13 +98,13 @@ public void init(final ProcessorContext context, public void init(final StateStoreContext context, final StateStore root) { this.context = context instanceof InternalProcessorContext ? (InternalProcessorContext) context : null; + taskId = context.taskId(); initStoreSerde(context); streamsMetrics = (StreamsMetricsImpl) context.metrics(); - taskId = context.taskId().toString(); registerMetrics(); final Sensor restoreSensor = - StateStoreMetrics.restoreSensor(taskId, metricsScope, name(), streamsMetrics); + StateStoreMetrics.restoreSensor(taskId.toString(), metricsScope, name(), streamsMetrics); // register and possibly restore the state from the logs maybeMeasureLatency(() -> super.init(context, root), time, restoreSensor); @@ -115,10 +114,10 @@ protected Serde prepareValueSerde(final Serde valueSerde, final SerdeGette } private void registerMetrics() { - putSensor = StateStoreMetrics.putSensor(taskId, metricsScope, name(), streamsMetrics); - fetchSensor = StateStoreMetrics.fetchSensor(taskId, metricsScope, name(), streamsMetrics); - flushSensor = StateStoreMetrics.flushSensor(taskId, metricsScope, name(), streamsMetrics); - e2eLatencySensor = StateStoreMetrics.e2ELatencySensor(taskId, metricsScope, name(), streamsMetrics); + putSensor = StateStoreMetrics.putSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + fetchSensor = StateStoreMetrics.fetchSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + flushSensor = StateStoreMetrics.flushSensor(taskId.toString(), metricsScope, name(), streamsMetrics); + e2eLatencySensor = StateStoreMetrics.e2ELatencySensor(taskId.toString(), metricsScope, name(), streamsMetrics); } @Deprecated @@ -128,7 +127,7 @@ private void initStoreSerde(final ProcessorContext context) { serdes = new StateSerdes<>( changelogTopic != null ? changelogTopic : - ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName), + ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName, taskId.namedTopology()), prepareKeySerde(keySerde, new SerdeGetter(context)), prepareValueSerde(valueSerde, new SerdeGetter(context))); } @@ -139,7 +138,7 @@ private void initStoreSerde(final StateStoreContext context) { serdes = new StateSerdes<>( changelogTopic != null ? changelogTopic : - ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName), + ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName, taskId.namedTopology()), prepareKeySerde(keySerde, new SerdeGetter(context)), prepareValueSerde(valueSerde, new SerdeGetter(context))); } @@ -313,7 +312,7 @@ public void close() { try { wrapped().close(); } finally { - streamsMetrics.removeAllStoreLevelSensorsAndMetrics(taskId, name()); + streamsMetrics.removeAllStoreLevelSensorsAndMetrics(taskId.toString(), name()); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java index a2d542ec0527c..0d626dd54781f 100644 --- a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java @@ -46,11 +46,11 @@ import org.apache.kafka.streams.processor.api.ProcessorContext; import org.apache.kafka.streams.processor.api.Record; import org.apache.kafka.streams.processor.internals.GlobalStreamThread; -import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder; import org.apache.kafka.streams.processor.internals.ProcessorTopology; import org.apache.kafka.streams.processor.internals.StateDirectory; import org.apache.kafka.streams.processor.internals.StreamThread; import org.apache.kafka.streams.processor.internals.StreamsMetadataState; +import org.apache.kafka.streams.processor.internals.TopologyMetadata; import org.apache.kafka.streams.processor.internals.ThreadMetadataImpl; import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl; import org.apache.kafka.streams.state.KeyValueStore; @@ -96,7 +96,6 @@ import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.safeUniqueTestName; import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.waitForApplicationState; -import static org.easymock.EasyMock.anyBoolean; import static org.apache.kafka.test.TestUtils.waitForCondition; import static org.easymock.EasyMock.anyInt; import static org.easymock.EasyMock.anyLong; @@ -216,7 +215,7 @@ private void prepareStreams() throws Exception { // setup stream threads PowerMock.mockStatic(StreamThread.class); EasyMock.expect(StreamThread.create( - anyObject(InternalTopologyBuilder.class), + anyObject(TopologyMetadata.class), anyObject(StreamsConfig.class), anyObject(KafkaClientSupplier.class), anyObject(Admin.class), @@ -948,7 +947,7 @@ public void shouldCleanupOldStateDirs() throws Exception { anyObject(StreamsConfig.class), anyObject(Time.class), EasyMock.eq(true), - anyBoolean() + EasyMock.eq(false) ).andReturn(stateDirectory); EasyMock.expect(stateDirectory.initializeProcessId()).andReturn(UUID.randomUUID()); stateDirectory.close(); @@ -1121,7 +1120,7 @@ private void startStreamsAndCheckDirExists(final Topology topology, anyObject(StreamsConfig.class), anyObject(Time.class), EasyMock.eq(shouldFilesExist), - anyBoolean() + EasyMock.eq(false) ).andReturn(stateDirectory); EasyMock.expect(stateDirectory.initializeProcessId()).andReturn(UUID.randomUUID()); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java index 504d9f6385eb1..29c61ec764779 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java @@ -208,7 +208,7 @@ public void shouldCompactTopicsForKeyValueStoreChangelogs() { waitForCompletion(streams, 2, 30000L); streams.close(); - final Properties changelogProps = getTopicProperties(ProcessorStateManager.storeChangelogTopic(appID, "Counts")); + final Properties changelogProps = getTopicProperties(ProcessorStateManager.storeChangelogTopic(appID, "Counts", null)); assertEquals(LogConfig.Compact(), changelogProps.getProperty(LogConfig.CleanupPolicyProp())); final Properties repartitionProps = getTopicProperties(appID + "-Counts-repartition"); @@ -247,7 +247,7 @@ public void shouldCompactAndDeleteTopicsForWindowStoreChangelogs() { // waitForCompletion(streams, 2, 30000L); streams.close(); - final Properties properties = getTopicProperties(ProcessorStateManager.storeChangelogTopic(appID, "CountWindows")); + final Properties properties = getTopicProperties(ProcessorStateManager.storeChangelogTopic(appID, "CountWindows", null)); final List policies = Arrays.asList(properties.getProperty(LogConfig.CleanupPolicyProp()).split(",")); assertEquals(2, policies.size()); assertTrue(policies.contains(LogConfig.Compact())); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/NamedTopologyIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/NamedTopologyIntegrationTest.java index 5ea3d391f9580..6bf8525b524bc 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/NamedTopologyIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/NamedTopologyIntegrationTest.java @@ -16,17 +16,240 @@ */ package org.apache.kafka.streams.integration; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.LongDeserializer; +import org.apache.kafka.common.serialization.LongSerializer; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.streams.KafkaClientSupplier; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; +import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.processor.internals.DefaultKafkaClientSupplier; +import org.apache.kafka.streams.processor.internals.namedtopology.KafkaStreamsNamedTopologyWrapper; +import org.apache.kafka.streams.processor.internals.namedtopology.NamedTopology; +import org.apache.kafka.streams.processor.internals.namedtopology.NamedTopologyStreamsBuilder; +import org.apache.kafka.streams.state.Stores; +import org.apache.kafka.test.TestUtils; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.regex.Pattern; + +import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.safeUniqueTestName; +import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; + public class NamedTopologyIntegrationTest { - //TODO KAFKA-12648 - /** - * Things to test in Pt. 2 - Introduce TopologyMetadata to wrap InternalTopologyBuilders of named topologies: - * 1. Verify changelog & repartition topics decorated with named topology - * 2. Make sure app run and works with - * -multiple subtopologies - * -persistent state - * -multi-partition input & output topics - * -standbys - * -piped input and verified output records - * 3. Is the task assignment balanced? Does KIP-441/warmup replica placement work as intended? - */ + + private static final int NUM_BROKERS = 1; + + public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS); + + @BeforeClass + public static void startCluster() throws IOException { + CLUSTER.start(); + } + + @AfterClass + public static void closeCluster() { + CLUSTER.stop(); + } + + @Rule + public final TestName testName = new TestName(); + private String appId; + + private String inputStream1; + private String inputStream2; + private String inputStream3; + private String outputStream1; + private String outputStream2; + private String outputStream3; + private String storeChangelog1; + private String storeChangelog2; + private String storeChangelog3; + + final List> standardInputData = asList(KeyValue.pair("A", 100L), KeyValue.pair("B", 200L), KeyValue.pair("A", 300L), KeyValue.pair("C", 400L)); + final List> standardOutputData = asList(KeyValue.pair("B", 1L), KeyValue.pair("A", 2L), KeyValue.pair("C", 1L)); // output of basic count topology with caching + + final KafkaClientSupplier clientSupplier = new DefaultKafkaClientSupplier(); + final Properties producerConfig = TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, LongSerializer.class); + final Properties consumerConfig = TestUtils.consumerConfig(CLUSTER.bootstrapServers(), StringDeserializer.class, LongDeserializer.class); + + final NamedTopologyStreamsBuilder builder1 = new NamedTopologyStreamsBuilder("topology-1"); + final NamedTopologyStreamsBuilder builder2 = new NamedTopologyStreamsBuilder("topology-2"); + final NamedTopologyStreamsBuilder builder3 = new NamedTopologyStreamsBuilder("topology-3"); + + Properties props; + KafkaStreamsNamedTopologyWrapper streams; + + private Properties configProps() { + final Properties streamsConfiguration = new Properties(); + streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, appId); + streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); + streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory(appId).getPath()); + streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); + streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass()); + streamsConfiguration.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 2); + streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000L); + streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + return streamsConfiguration; + } + + @Before + public void setup() throws InterruptedException { + appId = safeUniqueTestName(NamedTopologyIntegrationTest.class, testName); + inputStream1 = appId + "-input-stream-1"; + inputStream2 = appId + "-input-stream-2"; + inputStream3 = appId + "-input-stream-3"; + outputStream1 = appId + "-output-stream-1"; + outputStream2 = appId + "-output-stream-2"; + outputStream3 = appId + "-output-stream-3"; + storeChangelog1 = appId + "-topology-1-store-changelog"; + storeChangelog2 = appId + "-topology-2-store-changelog"; + storeChangelog3 = appId + "-topology-3-store-changelog"; + props = configProps(); + CLUSTER.createTopic(inputStream1, 2, 1); + CLUSTER.createTopic(inputStream2, 2, 1); + CLUSTER.createTopic(inputStream3, 2, 1); + CLUSTER.createTopic(outputStream1, 2, 1); + CLUSTER.createTopic(outputStream2, 2, 1); + CLUSTER.createTopic(outputStream3, 2, 1); + } + + @After + public void shutdown() throws Exception { + if (streams != null) { + streams.close(Duration.ofSeconds(30)); + } + CLUSTER.deleteTopics(inputStream1, inputStream2, inputStream3, outputStream1, outputStream2, outputStream3); + } + + @Test + public void shouldProcessSingleNamedTopologyAndPrefixInternalTopics() throws Exception { + produceToInputTopics(inputStream1, standardInputData); + builder1.stream(inputStream1) + .selectKey((k, v) -> k) + .groupByKey() + .count(Materialized.as(Stores.persistentKeyValueStore("store"))) + .toStream().to(outputStream1); + streams = new KafkaStreamsNamedTopologyWrapper(builder1.buildNamedTopology(props), props, clientSupplier); + IntegrationTestUtils.startApplicationAndWaitUntilRunning(singletonList(streams), Duration.ofSeconds(15)); + final List> results = waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream1, 3); + assertThat(results, equalTo(standardOutputData)); + + final Set allTopics = CLUSTER.getAllTopicsInCluster(); + assertThat(allTopics.contains(appId + "-" + "topology-1" + "-store-changelog"), is(true)); + assertThat(allTopics.contains(appId + "-" + "topology-1" + "-store-repartition"), is(true)); + } + + @Test + public void shouldProcessMultipleIdenticalNamedTopologiesWithPersistentStateStores() throws Exception { + produceToInputTopics(inputStream1, standardInputData); + produceToInputTopics(inputStream2, standardInputData); + produceToInputTopics(inputStream3, standardInputData); + + builder1.stream(inputStream1).selectKey((k, v) -> k).groupByKey().count(Materialized.as(Stores.persistentKeyValueStore("store"))).toStream().to(outputStream1); + builder2.stream(inputStream2).selectKey((k, v) -> k).groupByKey().count(Materialized.as(Stores.persistentKeyValueStore("store"))).toStream().to(outputStream2); + builder3.stream(inputStream3).selectKey((k, v) -> k).groupByKey().count(Materialized.as(Stores.persistentKeyValueStore("store"))).toStream().to(outputStream3); + streams = new KafkaStreamsNamedTopologyWrapper(buildNamedTopologies(builder1, builder2, builder3), props, clientSupplier); + IntegrationTestUtils.startApplicationAndWaitUntilRunning(singletonList(streams), Duration.ofSeconds(15)); + + assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream1, 3), equalTo(standardOutputData)); + assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream2, 3), equalTo(standardOutputData)); + assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream3, 3), equalTo(standardOutputData)); + + assertThat(CLUSTER.getAllTopicsInCluster().containsAll(asList(storeChangelog1, storeChangelog2, storeChangelog3)), is(true)); + } + + @Test + public void shouldProcessMultipleIdenticalNamedTopologiesWithInMemoryStateStores() throws Exception { + produceToInputTopics(inputStream1, standardInputData); + produceToInputTopics(inputStream2, standardInputData); + produceToInputTopics(inputStream3, standardInputData); + + builder1.stream(inputStream1).selectKey((k, v) -> k).groupByKey().count(Materialized.as(Stores.inMemoryKeyValueStore("store"))).toStream().to(outputStream1); + builder2.stream(inputStream2).selectKey((k, v) -> k).groupByKey().count(Materialized.as(Stores.inMemoryKeyValueStore("store"))).toStream().to(outputStream2); + builder3.stream(inputStream3).selectKey((k, v) -> k).groupByKey().count(Materialized.as(Stores.inMemoryKeyValueStore("store"))).toStream().to(outputStream3); + streams = new KafkaStreamsNamedTopologyWrapper(buildNamedTopologies(builder1, builder2, builder3), props, clientSupplier); + IntegrationTestUtils.startApplicationAndWaitUntilRunning(singletonList(streams), Duration.ofSeconds(15)); + + assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream1, 3), equalTo(standardOutputData)); + assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream2, 3), equalTo(standardOutputData)); + assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream3, 3), equalTo(standardOutputData)); + + assertThat(CLUSTER.getAllTopicsInCluster().containsAll(asList(storeChangelog1, storeChangelog2, storeChangelog3)), is(true)); + } + + @Test + public void shouldAllowPatternSubscriptionWithMultipleNamedTopologies() throws Exception { + produceToInputTopics(inputStream1, standardInputData); + produceToInputTopics(inputStream2, standardInputData); + produceToInputTopics(inputStream3, standardInputData); + + builder1.stream(Pattern.compile(inputStream1)).selectKey((k, v) -> k).groupByKey().count().toStream().to(outputStream1); + builder2.stream(Pattern.compile(inputStream2)).selectKey((k, v) -> k).groupByKey().count().toStream().to(outputStream2); + builder3.stream(Pattern.compile(inputStream3)).selectKey((k, v) -> k).groupByKey().count().toStream().to(outputStream3); + streams = new KafkaStreamsNamedTopologyWrapper(buildNamedTopologies(builder1, builder2, builder3), props, clientSupplier); + IntegrationTestUtils.startApplicationAndWaitUntilRunning(singletonList(streams), Duration.ofSeconds(15)); + + assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream1, 3), equalTo(standardOutputData)); + assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream2, 3), equalTo(standardOutputData)); + assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream3, 3), equalTo(standardOutputData)); + } + + @Test + public void shouldAllowMixedCollectionAndPatternSubscriptionWithMultipleNamedTopologies() throws Exception { + produceToInputTopics(inputStream1, standardInputData); + produceToInputTopics(inputStream2, standardInputData); + produceToInputTopics(inputStream3, standardInputData); + + builder1.stream(inputStream1).selectKey((k, v) -> k).groupByKey().count().toStream().to(outputStream1); + builder2.stream(Pattern.compile(inputStream2)).selectKey((k, v) -> k).groupByKey().count().toStream().to(outputStream2); + builder3.stream(Pattern.compile(inputStream3)).selectKey((k, v) -> k).groupByKey().count().toStream().to(outputStream3); + streams = new KafkaStreamsNamedTopologyWrapper(buildNamedTopologies(builder1, builder2, builder3), props, clientSupplier); + IntegrationTestUtils.startApplicationAndWaitUntilRunning(singletonList(streams), Duration.ofSeconds(15)); + + assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream1, 3), equalTo(standardOutputData)); + assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream2, 3), equalTo(standardOutputData)); + assertThat(waitUntilMinKeyValueRecordsReceived(consumerConfig, outputStream3, 3), equalTo(standardOutputData)); + } + + private void produceToInputTopics(final String topic, final Collection> records) { + IntegrationTestUtils.produceKeyValuesSynchronously( + topic, + records, + producerConfig, + CLUSTER.time + ); + } + + private List buildNamedTopologies(final NamedTopologyStreamsBuilder... builders) { + final List topologies = new ArrayList<>(); + for (final NamedTopologyStreamsBuilder builder : builders) { + topologies.add(builder.buildNamedTopology(props)); + } + return topologies; + } } diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/RestoreIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/RestoreIntegrationTest.java index 9deb5eb81f622..04d38f042895b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/RestoreIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/RestoreIntegrationTest.java @@ -151,7 +151,7 @@ public void shouldRestoreStateFromSourceTopic() throws Exception { createStateForRestoration(inputStream, 0); setCommittedOffset(inputStream, offsetLimitDelta); - final StateDirectory stateDirectory = new StateDirectory(new StreamsConfig(props), new MockTime(), true); + final StateDirectory stateDirectory = new StateDirectory(new StreamsConfig(props), new MockTime(), true, false); // note here the checkpointed offset is the last processed record's offset, so without control message we should write this offset - 1 new OffsetCheckpoint(new File(stateDirectory.getOrCreateDirectoryForTask(new TaskId(0, 0)), ".checkpoint")) .write(Collections.singletonMap(new TopicPartition(inputStream, 0), (long) offsetCheckpointed - 1)); @@ -217,7 +217,7 @@ public void shouldRestoreStateFromChangelogTopic() throws Exception { createStateForRestoration(changelog, 0); createStateForRestoration(inputStream, 10000); - final StateDirectory stateDirectory = new StateDirectory(new StreamsConfig(props), new MockTime(), true); + final StateDirectory stateDirectory = new StateDirectory(new StreamsConfig(props), new MockTime(), true, false); // note here the checkpointed offset is the last processed record's offset, so without control message we should write this offset - 1 new OffsetCheckpoint(new File(stateDirectory.getOrCreateDirectoryForTask(new TaskId(0, 0)), ".checkpoint")) .write(Collections.singletonMap(new TopicPartition(changelog, 0), (long) offsetCheckpointed - 1)); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/StandbyTaskEOSIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/StandbyTaskEOSIntegrationTest.java index 16cefd4ccaf76..4fbe73438b91c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/StandbyTaskEOSIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/StandbyTaskEOSIntegrationTest.java @@ -186,7 +186,7 @@ private KafkaStreams buildStreamWithDirtyStateDir(final String stateDirPath, final Properties props = props(stateDirPath); final StateDirectory stateDirectory = new StateDirectory( - new StreamsConfig(props), new MockTime(), true); + new StreamsConfig(props), new MockTime(), true, false); new OffsetCheckpoint(new File(stateDirectory.getOrCreateDirectoryForTask(taskId), ".checkpoint")) .write(Collections.singletonMap(new TopicPartition("unknown-topic", 0), 5L)); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java index 630ffca765d58..da457cbd21bdb 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java @@ -49,6 +49,7 @@ import org.apache.kafka.streams.processor.internals.StreamThread; import org.apache.kafka.streams.processor.internals.ThreadStateTransitionValidator; import org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentListener; +import org.apache.kafka.streams.processor.internals.namedtopology.KafkaStreamsNamedTopologyWrapper; import org.apache.kafka.streams.state.QueryableStoreType; import org.apache.kafka.test.TestCondition; import org.apache.kafka.test.TestUtils; @@ -991,9 +992,15 @@ public static boolean isEmptyConsumerGroup(final Admin adminClient, private static StateListener getStateListener(final KafkaStreams streams) { try { - final Field field = streams.getClass().getDeclaredField("stateListener"); - field.setAccessible(true); - return (StateListener) field.get(streams); + if (streams instanceof KafkaStreamsNamedTopologyWrapper) { + final Field field = streams.getClass().getSuperclass().getDeclaredField("stateListener"); + field.setAccessible(true); + return (StateListener) field.get(streams); + } else { + final Field field = streams.getClass().getDeclaredField("stateListener"); + field.setAccessible(true); + return (StateListener) field.get(streams); + } } catch (final IllegalAccessException | NoSuchFieldException e) { throw new RuntimeException("Failed to get StateListener through reflection", e); } diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilderTest.java index 76ae717a56061..73155da114092 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/InternalStreamsBuilderTest.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.streams.kstream.internals; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.streams.StreamsConfig; @@ -44,10 +45,10 @@ import static java.util.Arrays.asList; import static org.apache.kafka.streams.Topology.AutoOffsetReset; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.core.IsInstanceOf.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -250,8 +251,7 @@ public void shouldAddTopicToEarliestAutoOffsetResetList() { builder.stream(Collections.singleton(topicName), consumed); builder.buildAndOptimizeTopology(); - assertTrue(builder.internalTopologyBuilder.earliestResetTopicsPattern().matcher(topicName).matches()); - assertFalse(builder.internalTopologyBuilder.latestResetTopicsPattern().matcher(topicName).matches()); + assertThat(builder.internalTopologyBuilder.offsetResetStrategy(topicName), equalTo(OffsetResetStrategy.EARLIEST)); } @Test @@ -261,8 +261,7 @@ public void shouldAddTopicToLatestAutoOffsetResetList() { final ConsumedInternal consumed = new ConsumedInternal<>(Consumed.with(AutoOffsetReset.LATEST)); builder.stream(Collections.singleton(topicName), consumed); builder.buildAndOptimizeTopology(); - assertTrue(builder.internalTopologyBuilder.latestResetTopicsPattern().matcher(topicName).matches()); - assertFalse(builder.internalTopologyBuilder.earliestResetTopicsPattern().matcher(topicName).matches()); + assertThat(builder.internalTopologyBuilder.offsetResetStrategy(topicName), equalTo(OffsetResetStrategy.LATEST)); } @Test @@ -270,8 +269,7 @@ public void shouldAddTableToEarliestAutoOffsetResetList() { final String topicName = "topic-1"; builder.table(topicName, new ConsumedInternal<>(Consumed.with(AutoOffsetReset.EARLIEST)), materialized); builder.buildAndOptimizeTopology(); - assertTrue(builder.internalTopologyBuilder.earliestResetTopicsPattern().matcher(topicName).matches()); - assertFalse(builder.internalTopologyBuilder.latestResetTopicsPattern().matcher(topicName).matches()); + assertThat(builder.internalTopologyBuilder.offsetResetStrategy(topicName), equalTo(OffsetResetStrategy.EARLIEST)); } @Test @@ -279,8 +277,7 @@ public void shouldAddTableToLatestAutoOffsetResetList() { final String topicName = "topic-1"; builder.table(topicName, new ConsumedInternal<>(Consumed.with(AutoOffsetReset.LATEST)), materialized); builder.buildAndOptimizeTopology(); - assertTrue(builder.internalTopologyBuilder.latestResetTopicsPattern().matcher(topicName).matches()); - assertFalse(builder.internalTopologyBuilder.earliestResetTopicsPattern().matcher(topicName).matches()); + assertThat(builder.internalTopologyBuilder.offsetResetStrategy(topicName), equalTo(OffsetResetStrategy.LATEST)); } @Test @@ -289,8 +286,7 @@ public void shouldNotAddTableToOffsetResetLists() { builder.table(topicName, consumed, materialized); - assertFalse(builder.internalTopologyBuilder.latestResetTopicsPattern().matcher(topicName).matches()); - assertFalse(builder.internalTopologyBuilder.earliestResetTopicsPattern().matcher(topicName).matches()); + assertThat(builder.internalTopologyBuilder.offsetResetStrategy(topicName), equalTo(OffsetResetStrategy.NONE)); } @Test @@ -300,9 +296,7 @@ public void shouldNotAddRegexTopicsToOffsetResetLists() { builder.stream(topicPattern, consumed); - assertFalse(builder.internalTopologyBuilder.latestResetTopicsPattern().matcher(topic).matches()); - assertFalse(builder.internalTopologyBuilder.earliestResetTopicsPattern().matcher(topic).matches()); - + assertThat(builder.internalTopologyBuilder.offsetResetStrategy(topic), equalTo(OffsetResetStrategy.NONE)); } @Test @@ -313,8 +307,7 @@ public void shouldAddRegexTopicToEarliestAutoOffsetResetList() { builder.stream(topicPattern, new ConsumedInternal<>(Consumed.with(AutoOffsetReset.EARLIEST))); builder.buildAndOptimizeTopology(); - assertTrue(builder.internalTopologyBuilder.earliestResetTopicsPattern().matcher(topicTwo).matches()); - assertFalse(builder.internalTopologyBuilder.latestResetTopicsPattern().matcher(topicTwo).matches()); + assertThat(builder.internalTopologyBuilder.offsetResetStrategy(topicTwo), equalTo(OffsetResetStrategy.EARLIEST)); } @Test @@ -325,8 +318,7 @@ public void shouldAddRegexTopicToLatestAutoOffsetResetList() { builder.stream(topicPattern, new ConsumedInternal<>(Consumed.with(AutoOffsetReset.LATEST))); builder.buildAndOptimizeTopology(); - assertTrue(builder.internalTopologyBuilder.latestResetTopicsPattern().matcher(topicTwo).matches()); - assertFalse(builder.internalTopologyBuilder.earliestResetTopicsPattern().matcher(topicTwo).matches()); + assertThat(builder.internalTopologyBuilder.offsetResetStrategy(topicTwo), equalTo(OffsetResetStrategy.LATEST)); } @Test diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractProcessorContextTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractProcessorContextTest.java index 3c4ca1797c9d7..71d2d64e3b4b5 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractProcessorContextTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/AbstractProcessorContextTest.java @@ -259,7 +259,7 @@ public void registerCacheFlushListener(final String namespace, final DirtyEntryF @Override public String changelogFor(final String storeName) { - return ProcessorStateManager.storeChangelogTopic(applicationId(), storeName); + return ProcessorStateManager.storeChangelogTopic(applicationId(), storeName, taskId().namedTopology()); } } } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreatorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreatorTest.java index 54c92dfef4caf..b689dc18c51b4 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreatorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ActiveTaskCreatorTest.java @@ -462,9 +462,10 @@ private void createTasks() { expect(topology.sources()).andStubReturn(Collections.singleton(sourceNode)); replay(builder, stateDirectory, topology, sourceNode); + final StreamsConfig config = new StreamsConfig(properties); activeTaskCreator = new ActiveTaskCreator( - builder, - new StreamsConfig(properties), + new TopologyMetadata(builder, config), + config, streamsMetrics, stateDirectory, changeLogReader, diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImplTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImplTest.java index 0ad905727bdfe..9b50eadfb3add 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImplTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStateManagerImplTest.java @@ -133,7 +133,7 @@ public void before() { put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()); } }); - stateDirectory = new StateDirectory(streamsConfig, time, true); + stateDirectory = new StateDirectory(streamsConfig, time, true, false); consumer = new MockConsumer<>(OffsetResetStrategy.NONE); stateManager = new GlobalStateManagerImpl( new LogContext("test"), diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java index 3d735f8573696..30fc49e97cead 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/GlobalStreamThreadTest.java @@ -112,7 +112,7 @@ public String newStoreName(final String prefix) { builder.rewriteTopology(config).buildGlobalStateTopology(), config, mockConsumer, - new StateDirectory(config, time, true), + new StateDirectory(config, time, true, false), 0, new StreamsMetricsImpl(new Metrics(), "test-client", StreamsConfig.METRICS_LATEST, time), time, @@ -151,7 +151,7 @@ public List partitionsFor(final String topic) { builder.buildGlobalStateTopology(), config, mockConsumer, - new StateDirectory(config, time, true), + new StateDirectory(config, time, true, false), 0, new StreamsMetricsImpl(new Metrics(), "test-client", StreamsConfig.METRICS_LATEST, time), time, diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/HighAvailabilityStreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/HighAvailabilityStreamsPartitionAssignorTest.java index d020cb37f1367..56df1faf25a20 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/HighAvailabilityStreamsPartitionAssignorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/HighAvailabilityStreamsPartitionAssignorTest.java @@ -108,6 +108,7 @@ public class HighAvailabilityStreamsPartitionAssignorTest { private Admin adminClient; private StreamsConfig streamsConfig = new StreamsConfig(configProps()); private final InternalTopologyBuilder builder = new InternalTopologyBuilder(); + private TopologyMetadata topologyMetadata = new TopologyMetadata(builder, streamsConfig); private final StreamsMetadataState streamsMetadataState = EasyMock.createNiceMock(StreamsMetadataState.class); private final Map subscriptions = new HashMap<>(); @@ -147,11 +148,10 @@ private void createMockTaskManager(final Set activeTasks) { private void createMockTaskManager(final Map taskOffsetSums) { taskManager = EasyMock.createNiceMock(TaskManager.class); - expect(taskManager.builder()).andReturn(builder).anyTimes(); + expect(taskManager.topologyMetadata()).andStubReturn(topologyMetadata); expect(taskManager.getTaskOffsetSums()).andReturn(taskOffsetSums).anyTimes(); expect(taskManager.processId()).andReturn(UUID_1).anyTimes(); - builder.setApplicationId(APPLICATION_ID); - builder.buildTopology(); + topologyMetadata.buildAndRewriteTopology(); } // If you don't care about setting the end offsets for each specific topic partition, the helper method diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java index ef5bebb5d8201..74f40599e9e0c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.streams.processor.internals; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.common.config.TopicConfig; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; @@ -87,8 +88,8 @@ public void shouldAddSourceWithOffsetReset() { builder.addSource(Topology.AutoOffsetReset.LATEST, "source2", null, null, null, latestTopic); builder.initializeSubscription(); - assertTrue(builder.earliestResetTopicsPattern().matcher(earliestTopic).matches()); - assertTrue(builder.latestResetTopicsPattern().matcher(latestTopic).matches()); + assertThat(builder.offsetResetStrategy(earliestTopic), equalTo(OffsetResetStrategy.EARLIEST)); + assertThat(builder.offsetResetStrategy(latestTopic), equalTo(OffsetResetStrategy.LATEST)); } @Test @@ -100,8 +101,8 @@ public void shouldAddSourcePatternWithOffsetReset() { builder.addSource(Topology.AutoOffsetReset.LATEST, "source2", null, null, null, Pattern.compile(latestTopicPattern)); builder.initializeSubscription(); - assertTrue(builder.earliestResetTopicsPattern().matcher("earliestTestTopic").matches()); - assertTrue(builder.latestResetTopicsPattern().matcher("latestTestTopic").matches()); + assertThat(builder.offsetResetStrategy("earliestTestTopic"), equalTo(OffsetResetStrategy.EARLIEST)); + assertThat(builder.offsetResetStrategy("latestTestTopic"), equalTo(OffsetResetStrategy.LATEST)); } @Test @@ -110,8 +111,8 @@ public void shouldAddSourceWithoutOffsetReset() { builder.initializeSubscription(); assertEquals(Collections.singletonList("test-topic"), builder.sourceTopicCollection()); - assertEquals(builder.earliestResetTopicsPattern().pattern(), ""); - assertEquals(builder.latestResetTopicsPattern().pattern(), ""); + + assertThat(builder.offsetResetStrategy("test-topic"), equalTo(OffsetResetStrategy.NONE)); } @Test @@ -121,9 +122,9 @@ public void shouldAddPatternSourceWithoutOffsetReset() { builder.addSource(null, "source", null, stringSerde.deserializer(), stringSerde.deserializer(), Pattern.compile("test-.*")); builder.initializeSubscription(); - assertEquals(expectedPattern.pattern(), builder.sourceTopicPattern().pattern()); - assertEquals(builder.earliestResetTopicsPattern().pattern(), ""); - assertEquals(builder.latestResetTopicsPattern().pattern(), ""); + assertThat(expectedPattern.pattern(), builder.sourceTopicsPatternString(), equalTo("test-.*")); + + assertThat(builder.offsetResetStrategy("test-topic"), equalTo(OffsetResetStrategy.NONE)); } @Test @@ -303,8 +304,9 @@ public void testPatternAndNameSourceTopics() { builder.initializeSubscription(); final Pattern expectedPattern = Pattern.compile("X-topic-3|topic-1|topic-2|topic-4|topic-5"); + final String patternString = builder.sourceTopicsPatternString(); - assertEquals(expectedPattern.pattern(), builder.sourceTopicPattern().pattern()); + assertEquals(expectedPattern.pattern(), Pattern.compile(patternString).pattern()); } @Test @@ -326,7 +328,9 @@ public void testPatternSourceTopicsWithGlobalTopics() { final Pattern expectedPattern = Pattern.compile("topic-1|topic-2"); - assertThat(builder.sourceTopicPattern().pattern(), equalTo(expectedPattern.pattern())); + final String patternString = builder.sourceTopicsPatternString(); + + assertEquals(expectedPattern.pattern(), Pattern.compile(patternString).pattern()); } @Test @@ -354,7 +358,9 @@ public void testPatternSourceTopic() { final Pattern expectedPattern = Pattern.compile("topic-\\d"); builder.addSource(null, "source-1", null, null, null, expectedPattern); builder.initializeSubscription(); - assertEquals(expectedPattern.pattern(), builder.sourceTopicPattern().pattern()); + final String patternString = builder.sourceTopicsPatternString(); + + assertEquals(expectedPattern.pattern(), Pattern.compile(patternString).pattern()); } @Test @@ -363,7 +369,9 @@ public void testAddMoreThanOnePatternSourceNode() { builder.addSource(null, "source-1", null, null, null, Pattern.compile("topics[A-Z]")); builder.addSource(null, "source-2", null, null, null, Pattern.compile(".*-\\d")); builder.initializeSubscription(); - assertEquals(expectedPattern.pattern(), builder.sourceTopicPattern().pattern()); + final String patternString = builder.sourceTopicsPatternString(); + + assertEquals(expectedPattern.pattern(), Pattern.compile(patternString).pattern()); } @Test @@ -372,7 +380,9 @@ public void testSubscribeTopicNameAndPattern() { builder.addSource(null, "source-1", null, null, null, "topic-foo", "topic-bar"); builder.addSource(null, "source-2", null, null, null, Pattern.compile(".*-\\d")); builder.initializeSubscription(); - assertEquals(expectedPattern.pattern(), builder.sourceTopicPattern().pattern()); + final String patternString = builder.sourceTopicsPatternString(); + + assertEquals(expectedPattern.pattern(), Pattern.compile(patternString).pattern()); } @Test @@ -616,9 +626,9 @@ public void testTopicGroupsByStateStore() { final Map topicGroups = builder.topicGroups(); final Map expectedTopicGroups = new HashMap<>(); - final String store1 = ProcessorStateManager.storeChangelogTopic("X", "store-1"); - final String store2 = ProcessorStateManager.storeChangelogTopic("X", "store-2"); - final String store3 = ProcessorStateManager.storeChangelogTopic("X", "store-3"); + final String store1 = ProcessorStateManager.storeChangelogTopic("X", "store-1", builder.topologyName()); + final String store2 = ProcessorStateManager.storeChangelogTopic("X", "store-2", builder.topologyName()); + final String store3 = ProcessorStateManager.storeChangelogTopic("X", "store-3", builder.topologyName()); expectedTopicGroups.put(SUBTOPOLOGY_0, new InternalTopologyBuilder.TopicsInfo( Collections.emptySet(), mkSet("topic-1", "topic-1x", "topic-2"), Collections.emptyMap(), diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/NamedTopologyTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/NamedTopologyTest.java new file mode 100644 index 0000000000000..48ab10bec59db --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/NamedTopologyTest.java @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.processor.internals; + +import org.apache.kafka.streams.KafkaClientSupplier; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.errors.TopologyException; +import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.processor.internals.namedtopology.KafkaStreamsNamedTopologyWrapper; +import org.apache.kafka.streams.processor.internals.namedtopology.NamedTopology; +import org.apache.kafka.streams.processor.internals.namedtopology.NamedTopologyStreamsBuilder; +import org.apache.kafka.streams.state.Stores; +import org.apache.kafka.test.TestUtils; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import java.time.Duration; +import java.util.Properties; +import java.util.regex.Pattern; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertThrows; +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; + +public class NamedTopologyTest { + final KafkaClientSupplier clientSupplier = new DefaultKafkaClientSupplier(); + final Properties props = configProps(); + + final NamedTopologyStreamsBuilder builder1 = new NamedTopologyStreamsBuilder("topology-1"); + final NamedTopologyStreamsBuilder builder2 = new NamedTopologyStreamsBuilder("topology-2"); + final NamedTopologyStreamsBuilder builder3 = new NamedTopologyStreamsBuilder("topology-3"); + + KafkaStreamsNamedTopologyWrapper streams; + + @Before + public void setup() { + builder1.stream("input-1"); + builder2.stream("input-2"); + builder3.stream("input-3"); + } + + @After + public void cleanup() { + if (streams != null) { + streams.close(); + } + } + + private static Properties configProps() { + final Properties props = new Properties(); + props.put(StreamsConfig.APPLICATION_ID_CONFIG, "Named-Topology-App"); + props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:2018"); + props.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()); + return props; + } + + @Test + public void shouldThrowIllegalArgumentOnIllegalName() { + assertThrows(IllegalArgumentException.class, () -> new NamedTopologyStreamsBuilder("__not-allowed__")); + } + + @Test + public void shouldStartUpAndGoToRunningWithEmptyNamedTopology() throws Exception { + streams = new KafkaStreamsNamedTopologyWrapper(props, clientSupplier); + IntegrationTestUtils.startApplicationAndWaitUntilRunning(singletonList(streams), Duration.ofSeconds(15)); + } + + @Test + public void shouldBuildSingleNamedTopology() { + builder1.stream("stream-1").filter((k, v) -> !k.equals(v)).to("output-1"); + + streams = new KafkaStreamsNamedTopologyWrapper(builder1.buildNamedTopology(props), props, clientSupplier); + } + + @Test + public void shouldBuildMultipleIdenticalNamedTopologyWithRepartition() { + builder1.stream("stream-1").selectKey((k, v) -> v).groupByKey().count().toStream().to("output-1"); + builder2.stream("stream-2").selectKey((k, v) -> v).groupByKey().count().toStream().to("output-2"); + builder3.stream("stream-3").selectKey((k, v) -> v).groupByKey().count().toStream().to("output-3"); + + streams = new KafkaStreamsNamedTopologyWrapper( + asList( + builder1.buildNamedTopology(props), + builder2.buildNamedTopology(props), + builder3.buildNamedTopology(props)), + props, + clientSupplier + ); + } + + @Test + public void shouldReturnTopologyByName() { + final NamedTopology topology1 = builder1.buildNamedTopology(props); + final NamedTopology topology2 = builder2.buildNamedTopology(props); + final NamedTopology topology3 = builder3.buildNamedTopology(props); + streams = new KafkaStreamsNamedTopologyWrapper(asList(topology1, topology2, topology3), props, clientSupplier); + assertThat(streams.getTopologyByName("topology-1"), equalTo(topology1)); + assertThat(streams.getTopologyByName("topology-2"), equalTo(topology2)); + assertThat(streams.getTopologyByName("topology-3"), equalTo(topology3)); + } + + @Test + public void shouldThrowIllegalArgumentWhenLookingUpNonExistentTopologyByName() { + streams = new KafkaStreamsNamedTopologyWrapper(builder1.buildNamedTopology(props), props, clientSupplier); + assertThrows(IllegalArgumentException.class, () -> streams.getTopologyByName("non-existent-topology")); + } + + @Test + public void shouldAllowSameStoreNameToBeUsedByMultipleNamedTopologies() { + builder1.stream("stream-1").selectKey((k, v) -> v).groupByKey().count(Materialized.as(Stores.inMemoryKeyValueStore("store"))); + builder2.stream("stream-2").selectKey((k, v) -> v).groupByKey().count(Materialized.as(Stores.inMemoryKeyValueStore("store"))); + + streams = new KafkaStreamsNamedTopologyWrapper(asList( + builder1.buildNamedTopology(props), + builder2.buildNamedTopology(props)), + props, + clientSupplier + ); + } + + @Test + public void shouldThrowTopologyExceptionWhenMultipleNamedTopologiesCreateStreamFromSameInputTopic() { + builder1.stream("stream"); + builder2.stream("stream"); + + assertThrows( + TopologyException.class, + () -> streams = new KafkaStreamsNamedTopologyWrapper( + asList( + builder1.buildNamedTopology(props), + builder2.buildNamedTopology(props)), + props, + clientSupplier) + ); + } + + @Test + public void shouldThrowTopologyExceptionWhenMultipleNamedTopologiesCreateTableFromSameInputTopic() { + builder1.table("table"); + builder2.table("table"); + + assertThrows( + TopologyException.class, + () -> streams = new KafkaStreamsNamedTopologyWrapper( + asList( + builder1.buildNamedTopology(props), + builder2.buildNamedTopology(props)), + props, + clientSupplier) + ); + } + + @Test + public void shouldThrowTopologyExceptionWhenMultipleNamedTopologiesCreateStreamAndTableFromSameInputTopic() { + builder1.stream("input"); + builder2.table("input"); + + assertThrows( + TopologyException.class, + () -> streams = new KafkaStreamsNamedTopologyWrapper( + asList( + builder1.buildNamedTopology(props), + builder2.buildNamedTopology(props)), + props, + clientSupplier) + ); + } + + @Test + public void shouldThrowTopologyExceptionWhenMultipleNamedTopologiesCreateStreamFromOverlappingInputTopicCollection() { + builder1.stream("stream"); + builder2.stream(asList("unique-input", "stream")); + + assertThrows( + TopologyException.class, + () -> streams = new KafkaStreamsNamedTopologyWrapper( + asList( + builder1.buildNamedTopology(props), + builder2.buildNamedTopology(props)), + props, + clientSupplier) + ); + } + + @Test + public void shouldThrowTopologyExceptionWhenMultipleNamedTopologiesCreateStreamFromSamePattern() { + builder1.stream(Pattern.compile("some-regex")); + builder2.stream(Pattern.compile("some-regex")); + + assertThrows( + TopologyException.class, + () -> streams = new KafkaStreamsNamedTopologyWrapper( + asList( + builder1.buildNamedTopology(props), + builder2.buildNamedTopology(props)), + props, + clientSupplier) + ); + } + + @Test + public void shouldDescribeWithSingleNamedTopology() { + builder1.stream("input").filter((k, v) -> !k.equals(v)).to("output"); + streams = new KafkaStreamsNamedTopologyWrapper(builder1.buildNamedTopology(props), props, clientSupplier); + + assertThat( + streams.getFullTopologyDescription(), + equalTo( + "Topology - topology-1:\n" + + " Sub-topology: 0\n" + + " Source: KSTREAM-SOURCE-0000000000 (topics: [input-1])\n" + + " --> none\n" + + "\n" + + " Sub-topology: 1\n" + + " Source: KSTREAM-SOURCE-0000000001 (topics: [input])\n" + + " --> KSTREAM-FILTER-0000000002\n" + + " Processor: KSTREAM-FILTER-0000000002 (stores: [])\n" + + " --> KSTREAM-SINK-0000000003\n" + + " <-- KSTREAM-SOURCE-0000000001\n" + + " Sink: KSTREAM-SINK-0000000003 (topic: output)\n" + + " <-- KSTREAM-FILTER-0000000002\n\n") + ); + } + + @Test + public void shouldDescribeWithMultipleNamedTopologies() { + builder1.stream("stream-1").filter((k, v) -> !k.equals(v)).to("output-1"); + builder2.stream("stream-2").filter((k, v) -> !k.equals(v)).to("output-2"); + builder3.stream("stream-3").filter((k, v) -> !k.equals(v)).to("output-3"); + + streams = new KafkaStreamsNamedTopologyWrapper( + asList( + builder1.buildNamedTopology(props), + builder2.buildNamedTopology(props), + builder3.buildNamedTopology(props)), + props, + clientSupplier + ); + + assertThat( + streams.getFullTopologyDescription(), + equalTo( + "Topology - topology-1:\n" + + " Sub-topology: 0\n" + + " Source: KSTREAM-SOURCE-0000000000 (topics: [input-1])\n" + + " --> none\n" + + "\n" + + " Sub-topology: 1\n" + + " Source: KSTREAM-SOURCE-0000000001 (topics: [stream-1])\n" + + " --> KSTREAM-FILTER-0000000002\n" + + " Processor: KSTREAM-FILTER-0000000002 (stores: [])\n" + + " --> KSTREAM-SINK-0000000003\n" + + " <-- KSTREAM-SOURCE-0000000001\n" + + " Sink: KSTREAM-SINK-0000000003 (topic: output-1)\n" + + " <-- KSTREAM-FILTER-0000000002\n" + + "\n" + + "Topology - topology-2:\n" + + " Sub-topology: 0\n" + + " Source: KSTREAM-SOURCE-0000000000 (topics: [input-2])\n" + + " --> none\n" + + "\n" + + " Sub-topology: 1\n" + + " Source: KSTREAM-SOURCE-0000000001 (topics: [stream-2])\n" + + " --> KSTREAM-FILTER-0000000002\n" + + " Processor: KSTREAM-FILTER-0000000002 (stores: [])\n" + + " --> KSTREAM-SINK-0000000003\n" + + " <-- KSTREAM-SOURCE-0000000001\n" + + " Sink: KSTREAM-SINK-0000000003 (topic: output-2)\n" + + " <-- KSTREAM-FILTER-0000000002\n" + + "\n" + + "Topology - topology-3:\n" + + " Sub-topology: 0\n" + + " Source: KSTREAM-SOURCE-0000000000 (topics: [input-3])\n" + + " --> none\n" + + "\n" + + " Sub-topology: 1\n" + + " Source: KSTREAM-SOURCE-0000000001 (topics: [stream-3])\n" + + " --> KSTREAM-FILTER-0000000002\n" + + " Processor: KSTREAM-FILTER-0000000002 (stores: [])\n" + + " --> KSTREAM-SINK-0000000003\n" + + " <-- KSTREAM-SOURCE-0000000001\n" + + " Sink: KSTREAM-SINK-0000000003 (topic: output-3)\n" + + " <-- KSTREAM-FILTER-0000000002\n\n") + ); + } + + @Test + public void shouldDescribeWithEmptyNamedTopology() { + streams = new KafkaStreamsNamedTopologyWrapper(props, clientSupplier); + + assertThat(streams.getFullTopologyDescription(), equalTo("")); + } +} diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorStateManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorStateManagerTest.java index aa5ac9b2ac127..9f8d6bd645e12 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorStateManagerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorStateManagerTest.java @@ -85,15 +85,16 @@ public class ProcessorStateManagerTest { private final String applicationId = "test-application"; + private final TaskId taskId = new TaskId(0, 1, "My-Topology"); private final String persistentStoreName = "persistentStore"; private final String persistentStoreTwoName = "persistentStore2"; private final String nonPersistentStoreName = "nonPersistentStore"; private final String persistentStoreTopicName = - ProcessorStateManager.storeChangelogTopic(applicationId, persistentStoreName); + ProcessorStateManager.storeChangelogTopic(applicationId, persistentStoreName, taskId.namedTopology()); private final String persistentStoreTwoTopicName = - ProcessorStateManager.storeChangelogTopic(applicationId, persistentStoreTwoName); + ProcessorStateManager.storeChangelogTopic(applicationId, persistentStoreTwoName, taskId.namedTopology()); private final String nonPersistentStoreTopicName = - ProcessorStateManager.storeChangelogTopic(applicationId, nonPersistentStoreName); + ProcessorStateManager.storeChangelogTopic(applicationId, nonPersistentStoreName, taskId.namedTopology()); private final MockKeyValueStore persistentStore = new MockKeyValueStore(persistentStoreName, true); private final MockKeyValueStore persistentStoreTwo = new MockKeyValueStore(persistentStoreTwoName, true); private final MockKeyValueStore nonPersistentStore = new MockKeyValueStore(nonPersistentStoreName, false); @@ -101,7 +102,6 @@ public class ProcessorStateManagerTest { private final TopicPartition persistentStoreTwoPartition = new TopicPartition(persistentStoreTwoTopicName, 1); private final TopicPartition nonPersistentStorePartition = new TopicPartition(nonPersistentStoreTopicName, 1); private final TopicPartition irrelevantPartition = new TopicPartition("other-topic", 1); - private final TaskId taskId = new TaskId(0, 1); private final Integer key = 1; private final String value = "the-value"; private final byte[] keyBytes = new byte[] {0x0, 0x0, 0x0, 0x1}; @@ -134,7 +134,7 @@ public void setup() { put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:1234"); put(StreamsConfig.STATE_DIR_CONFIG, baseDir.getPath()); } - }), new MockTime(), true); + }), new MockTime(), true, true); checkpointFile = new File(stateDirectory.getOrCreateDirectoryForTask(taskId), CHECKPOINT_FILE_NAME); checkpoint = new OffsetCheckpoint(checkpointFile); @@ -155,11 +155,23 @@ public void shouldReturnDefaultChangelogTopicName() { final String storeName = "store"; assertThat( - ProcessorStateManager.storeChangelogTopic(applicationId, storeName), + ProcessorStateManager.storeChangelogTopic(applicationId, storeName, null), is(applicationId + "-" + storeName + "-changelog") ); } + @Test + public void shouldReturnDefaultChangelogTopicNameWithNamedTopology() { + final String applicationId = "appId"; + final String namedTopology = "namedTopology"; + final String storeName = "store"; + + assertThat( + ProcessorStateManager.storeChangelogTopic(applicationId, storeName, namedTopology), + is(applicationId + "-" + namedTopology + "-" + storeName + "-changelog") + ); + } + @Test public void shouldReturnBaseDir() { final ProcessorStateManager stateMgr = getStateManager(Task.TaskType.ACTIVE); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionTopicsTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionTopicsTest.java index 30a4641e3d2a9..88b70e0c32d55 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionTopicsTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RepartitionTopicsTest.java @@ -20,10 +20,13 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.errors.MissingSourceTopicException; import org.apache.kafka.streams.errors.TaskAssignmentException; import org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo; import org.apache.kafka.streams.processor.internals.assignment.CopartitionedTopicsEnforcer; +import org.apache.kafka.streams.processor.internals.testutil.DummyStreamsConfig; + import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -90,6 +93,7 @@ public class RepartitionTopicsTest { mkMap(mkEntry(REPARTITION_TOPIC_NAME1, REPARTITION_TOPIC_CONFIG1)), Collections.emptyMap() ); + final StreamsConfig config = new DummyStreamsConfig(); final InternalTopologyBuilder internalTopologyBuilder = mock(InternalTopologyBuilder.class); final InternalTopicManager internalTopicManager = mock(InternalTopicManager.class); @@ -98,6 +102,7 @@ public class RepartitionTopicsTest { @Test public void shouldSetupRepartitionTopics() { + expect(internalTopologyBuilder.hasNamedTopology()).andStubReturn(false); expect(internalTopologyBuilder.topicGroups()) .andReturn(mkMap(mkEntry(SUBTOPOLOGY_0, TOPICS_INFO1), mkEntry(SUBTOPOLOGY_1, TOPICS_INFO2))); final Set coPartitionGroup1 = mkSet(SOURCE_TOPIC_NAME1, SOURCE_TOPIC_NAME2); @@ -115,7 +120,7 @@ public void shouldSetupRepartitionTopics() { setupCluster(); replay(internalTopicManager, internalTopologyBuilder, clusterMetadata); final RepartitionTopics repartitionTopics = new RepartitionTopics( - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, @@ -137,6 +142,7 @@ public void shouldSetupRepartitionTopics() { @Test public void shouldThrowMissingSourceTopicException() { + expect(internalTopologyBuilder.hasNamedTopology()).andStubReturn(false); expect(internalTopologyBuilder.topicGroups()) .andReturn(mkMap(mkEntry(SUBTOPOLOGY_0, TOPICS_INFO1), mkEntry(SUBTOPOLOGY_1, TOPICS_INFO2))); expect(internalTopologyBuilder.copartitionGroups()).andReturn(Collections.emptyList()); @@ -149,7 +155,7 @@ public void shouldThrowMissingSourceTopicException() { setupClusterWithMissingTopics(mkSet(SOURCE_TOPIC_NAME1)); replay(internalTopicManager, internalTopologyBuilder, clusterMetadata); final RepartitionTopics repartitionTopics = new RepartitionTopics( - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, @@ -163,6 +169,7 @@ public void shouldThrowMissingSourceTopicException() { public void shouldThrowTaskAssignmentExceptionIfPartitionCountCannotBeComputedForAllRepartitionTopics() { final RepartitionTopicConfig repartitionTopicConfigWithoutPartitionCount = new RepartitionTopicConfig(REPARTITION_WITHOUT_PARTITION_COUNT, TOPIC_CONFIG5); + expect(internalTopologyBuilder.hasNamedTopology()).andStubReturn(false); expect(internalTopologyBuilder.topicGroups()) .andReturn(mkMap( mkEntry(SUBTOPOLOGY_0, TOPICS_INFO1), @@ -178,7 +185,7 @@ public void shouldThrowTaskAssignmentExceptionIfPartitionCountCannotBeComputedFo setupCluster(); replay(internalTopicManager, internalTopologyBuilder, clusterMetadata); final RepartitionTopics repartitionTopics = new RepartitionTopics( - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, @@ -201,6 +208,7 @@ public void shouldThrowTaskAssignmentExceptionIfSourceTopicHasNoPartitionCount() ), Collections.emptyMap() ); + expect(internalTopologyBuilder.hasNamedTopology()).andStubReturn(false); expect(internalTopologyBuilder.topicGroups()) .andReturn(mkMap( mkEntry(SUBTOPOLOGY_0, topicsInfo), @@ -216,7 +224,7 @@ public void shouldThrowTaskAssignmentExceptionIfSourceTopicHasNoPartitionCount() setupClusterWithMissingPartitionCounts(mkSet(SOURCE_TOPIC_NAME1)); replay(internalTopicManager, internalTopologyBuilder, clusterMetadata); final RepartitionTopics repartitionTopics = new RepartitionTopics( - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, @@ -244,6 +252,7 @@ public void shouldSetRepartitionTopicPartitionCountFromUpstreamExternalSourceTop ), Collections.emptyMap() ); + expect(internalTopologyBuilder.hasNamedTopology()).andStubReturn(false); expect(internalTopologyBuilder.topicGroups()) .andReturn(mkMap( mkEntry(SUBTOPOLOGY_0, topicsInfo), @@ -261,7 +270,7 @@ public void shouldSetRepartitionTopicPartitionCountFromUpstreamExternalSourceTop setupCluster(); replay(internalTopicManager, internalTopologyBuilder, clusterMetadata); final RepartitionTopics repartitionTopics = new RepartitionTopics( - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, @@ -298,6 +307,7 @@ public void shouldSetRepartitionTopicPartitionCountFromUpstreamInternalRepartiti ), Collections.emptyMap() ); + expect(internalTopologyBuilder.hasNamedTopology()).andStubReturn(false); expect(internalTopologyBuilder.topicGroups()) .andReturn(mkMap( mkEntry(SUBTOPOLOGY_0, topicsInfo), @@ -315,7 +325,7 @@ public void shouldSetRepartitionTopicPartitionCountFromUpstreamInternalRepartiti setupCluster(); replay(internalTopicManager, internalTopologyBuilder, clusterMetadata); final RepartitionTopics repartitionTopics = new RepartitionTopics( - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, @@ -347,6 +357,7 @@ public void shouldNotSetupRepartitionTopicsWhenTopologyDoesNotContainAnyRepartit Collections.emptyMap(), Collections.emptyMap() ); + expect(internalTopologyBuilder.hasNamedTopology()).andStubReturn(false); expect(internalTopologyBuilder.topicGroups()) .andReturn(mkMap(mkEntry(SUBTOPOLOGY_0, topicsInfo))); expect(internalTopologyBuilder.copartitionGroups()).andReturn(Collections.emptySet()); @@ -354,7 +365,7 @@ public void shouldNotSetupRepartitionTopicsWhenTopologyDoesNotContainAnyRepartit setupCluster(); replay(internalTopicManager, internalTopologyBuilder, clusterMetadata); final RepartitionTopics repartitionTopics = new RepartitionTopics( - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java index 60a95d4834427..82231bccbab3c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StandbyTaskTest.java @@ -77,13 +77,13 @@ public class StandbyTaskTest { private final String threadName = "threadName"; private final String threadId = Thread.currentThread().getName(); - private final TaskId taskId = new TaskId(0, 0); + private final TaskId taskId = new TaskId(0, 0, "My-Topology"); private final String storeName1 = "store1"; private final String storeName2 = "store2"; private final String applicationId = "test-application"; - private final String storeChangelogTopicName1 = ProcessorStateManager.storeChangelogTopic(applicationId, storeName1); - private final String storeChangelogTopicName2 = ProcessorStateManager.storeChangelogTopic(applicationId, storeName2); + private final String storeChangelogTopicName1 = ProcessorStateManager.storeChangelogTopic(applicationId, storeName1, taskId.namedTopology()); + private final String storeChangelogTopicName2 = ProcessorStateManager.storeChangelogTopic(applicationId, storeName2, taskId.namedTopology()); private final TopicPartition partition = new TopicPartition(storeChangelogTopicName1, 0); private final MockKeyValueStore store1 = (MockKeyValueStore) new MockKeyValueStoreBuilder(storeName1, false).build(); @@ -138,7 +138,7 @@ public void setup() throws Exception { )); baseDir = TestUtils.tempDirectory(); config = createConfig(baseDir); - stateDirectory = new StateDirectory(config, new MockTime(), true); + stateDirectory = new StateDirectory(config, new MockTime(), true, true); } @After diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StateDirectoryTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StateDirectoryTest.java index d91c860d964bd..c74619fdbcd0c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StateDirectoryTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StateDirectoryTest.java @@ -384,7 +384,9 @@ public void shouldReturnEmptyArrayIfListFilesReturnsNull() throws IOException { put(StreamsConfig.STATE_DIR_CONFIG, stateDir.getPath()); } }), - time, true); + time, + true, + false); appDir = new File(stateDir, applicationId); // make sure the File#listFiles returns null and StateDirectory#listAllTaskDirectories is able to handle null @@ -425,7 +427,9 @@ public void shouldCreateDirectoriesIfParentDoesntExist() { put(StreamsConfig.STATE_DIR_CONFIG, stateDir.getPath()); } }), - time, true); + time, + true, + false); final File taskDir = stateDirectory.getOrCreateDirectoryForTask(new TaskId(0, 0)); assertTrue(stateDir.exists()); assertTrue(taskDir.exists()); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java index 5b4125699caf2..f145e6dace735 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java @@ -246,7 +246,7 @@ public void setup() { consumer.assign(asList(partition1, partition2)); consumer.updateBeginningOffsets(mkMap(mkEntry(partition1, 0L), mkEntry(partition2, 0L))); - stateDirectory = new StateDirectory(createConfig("100"), new MockTime(), true); + stateDirectory = new StateDirectory(createConfig("100"), new MockTime(), true, false); } @After diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java index 0659f52dda8fb..8bda8f3d05b33 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java @@ -150,7 +150,7 @@ public class StreamThreadTest { private final StreamsConfig config = new StreamsConfig(configProps(false)); private final ConsumedInternal consumed = new ConsumedInternal<>(); private final ChangelogReader changelogReader = new MockChangelogReader(); - private final StateDirectory stateDirectory = new StateDirectory(config, mockTime, true); + private final StateDirectory stateDirectory = new StateDirectory(config, mockTime, true, false); private final InternalTopologyBuilder internalTopologyBuilder = new InternalTopologyBuilder(); private final InternalStreamsBuilder internalStreamsBuilder = new InternalStreamsBuilder(internalTopologyBuilder); @@ -169,7 +169,7 @@ public class StreamThreadTest { public void setUp() { Thread.currentThread().setName(CLIENT_ID + "-StreamThread-" + threadIdx); internalTopologyBuilder.setApplicationId(APPLICATION_ID); - streamsMetadataState = new StreamsMetadataState(internalTopologyBuilder, StreamsMetadataState.UNKNOWN_HOST); + streamsMetadataState = new StreamsMetadataState(new TopologyMetadata(internalTopologyBuilder, config), StreamsMetadataState.UNKNOWN_HOST); } private final String topic1 = "topic1"; @@ -228,7 +228,7 @@ private StreamThread createStreamThread(@SuppressWarnings("SameParameterValue") internalTopologyBuilder.buildTopology(); return StreamThread.create( - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), config, clientSupplier, clientSupplier.getAdmin(config.getAdminConfigs(clientId)), @@ -451,7 +451,7 @@ public void shouldNotCommitBeforeTheCommitInterval() { null, taskManager, streamsMetrics, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -491,7 +491,7 @@ public void shouldEnforceRebalanceAfterNextScheduledProbingRebalanceTime() throw mockClientSupplier.setCluster(createCluster()); EasyMock.replay(mockConsumer); final StreamThread thread = StreamThread.create( - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), config, mockClientSupplier, mockClientSupplier.getAdmin(config.getAdminConfigs(CLIENT_ID)), @@ -700,7 +700,7 @@ public void shouldNotCauseExceptionIfNothingCommitted() { null, taskManager, streamsMetrics, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -770,7 +770,7 @@ int commit(final Collection tasksToCommit) { null, taskManager, streamsMetrics, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -827,7 +827,7 @@ public void shouldRecordCommitLatency() { null, activeTaskCreator, null, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), null, null, null @@ -850,7 +850,7 @@ int commit(final Collection tasksToCommit) { null, taskManager, streamsMetrics, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -1123,6 +1123,8 @@ public void shouldShutdownTaskManagerOnClose() { final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime); + final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config); + topologyMetadata.buildAndRewriteTopology(); final StreamThread thread = new StreamThread( mockTime, config, @@ -1133,7 +1135,7 @@ public void shouldShutdownTaskManagerOnClose() { null, taskManager, streamsMetrics, - internalTopologyBuilder, + topologyMetadata, CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -1195,7 +1197,7 @@ public void restore(final Map tasks) { null, taskManager, streamsMetrics, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -1240,7 +1242,7 @@ public void shouldShutdownTaskManagerOnCloseWithoutStart() { null, taskManager, streamsMetrics, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -1278,7 +1280,7 @@ public void shouldOnlyShutdownOnce() { null, taskManager, streamsMetrics, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -1607,7 +1609,7 @@ public void shouldReturnActiveTaskMetadataWhileRunningState() { internalTopologyBuilder.buildTopology(); final StreamThread thread = StreamThread.create( - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), config, clientSupplier, clientSupplier.getAdmin(config.getAdminConfigs(CLIENT_ID)), @@ -2185,7 +2187,7 @@ public void shouldThrowTaskMigratedExceptionHandlingTaskLost() { null, taskManager, streamsMetrics, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -2233,7 +2235,7 @@ public void shouldThrowTaskMigratedExceptionHandlingRevocation() { null, taskManager, streamsMetrics, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -2284,6 +2286,8 @@ public void shouldCatchHandleCorruptionOnTaskCorruptedExceptionPath() { final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime); + final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config); + topologyMetadata.buildAndRewriteTopology(); final StreamThread thread = new StreamThread( mockTime, config, @@ -2294,7 +2298,7 @@ public void shouldCatchHandleCorruptionOnTaskCorruptedExceptionPath() { null, taskManager, streamsMetrics, - internalTopologyBuilder, + topologyMetadata, CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -2348,6 +2352,8 @@ public void shouldCatchTimeoutExceptionFromHandleCorruptionAndInvokeExceptionHan final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime); + final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config); + topologyMetadata.buildAndRewriteTopology(); final StreamThread thread = new StreamThread( mockTime, config, @@ -2358,7 +2364,7 @@ public void shouldCatchTimeoutExceptionFromHandleCorruptionAndInvokeExceptionHan null, taskManager, streamsMetrics, - internalTopologyBuilder, + topologyMetadata, CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -2420,6 +2426,8 @@ public void shouldCatchTaskMigratedExceptionOnOnTaskCorruptedExceptionPath() { final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime); + final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config); + topologyMetadata.buildAndRewriteTopology(); final StreamThread thread = new StreamThread( mockTime, config, @@ -2430,7 +2438,7 @@ public void shouldCatchTaskMigratedExceptionOnOnTaskCorruptedExceptionPath() { null, taskManager, streamsMetrics, - internalTopologyBuilder, + topologyMetadata, CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -2496,7 +2504,7 @@ public void shouldNotCommitNonRunningNonRestoringTasks() { null, taskManager, streamsMetrics, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -2638,7 +2646,7 @@ public void shouldTransmitTaskManagerMetrics() { null, taskManager, streamsMetrics, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -2679,7 +2687,7 @@ public void shouldConstructAdminMetrics() { null, taskManager, streamsMetrics, - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -2723,6 +2731,8 @@ public void runAndVerifyFailedStreamThreadRecording(final boolean shouldFail) { expect(taskManager.producerClientIds()).andStubReturn(Collections.emptySet()); final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime); + final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config); + topologyMetadata.buildAndRewriteTopology(); final StreamThread thread = new StreamThread( mockTime, config, @@ -2733,7 +2743,7 @@ public void runAndVerifyFailedStreamThreadRecording(final boolean shouldFail) { null, taskManager, streamsMetrics, - internalTopologyBuilder, + topologyMetadata, CLIENT_ID, new LogContext(""), new AtomicInteger(), @@ -2791,7 +2801,7 @@ private Collection createStandbyTask() { final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime); final StandbyTaskCreator standbyTaskCreator = new StandbyTaskCreator( - internalTopologyBuilder, + new TopologyMetadata(internalTopologyBuilder, config), config, streamsMetrics, stateDirectory, diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsAssignmentScaleTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsAssignmentScaleTest.java index a08611fa78b49..2782c1ac9ee8e 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsAssignmentScaleTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsAssignmentScaleTest.java @@ -159,23 +159,22 @@ private void completeLargeAssignment(final int numPartitions, emptySet(), emptySet() ); - + final Map configMap = new HashMap<>(); + configMap.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_ID); + configMap.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:8080"); final InternalTopologyBuilder builder = new InternalTopologyBuilder(); builder.addSource(null, "source", null, null, null, "topic"); builder.addProcessor("processor", new MockApiProcessorSupplier<>(), "source"); builder.addStateStore(new MockKeyValueStoreBuilder("store", false), "processor"); - builder.setApplicationId(APPLICATION_ID); - builder.buildTopology(); + final TopologyMetadata topologyMetadata = new TopologyMetadata(builder, new StreamsConfig(configMap)); + topologyMetadata.buildAndRewriteTopology(); final Consumer mainConsumer = EasyMock.createNiceMock(Consumer.class); final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class); - expect(taskManager.builder()).andStubReturn(builder); + expect(taskManager.topologyMetadata()).andStubReturn(topologyMetadata); expect(mainConsumer.committed(new HashSet<>())).andStubReturn(Collections.emptyMap()); final AdminClient adminClient = createMockAdminClientForAssignor(changelogEndOffsets); - final Map configMap = new HashMap<>(); - configMap.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_ID); - configMap.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:8080"); final ReferenceContainer referenceContainer = new ReferenceContainer(); referenceContainer.mainConsumer = mainConsumer; referenceContainer.adminClient = adminClient; diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetadataStateTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetadataStateTest.java index df52df37d4b54..84578b2d6ff0f 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetadataStateTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsMetadataStateTest.java @@ -30,6 +30,7 @@ import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.processor.StreamPartitioner; +import org.apache.kafka.streams.processor.internals.testutil.DummyStreamsConfig; import org.apache.kafka.streams.state.HostInfo; import org.apache.kafka.streams.state.internals.StreamsMetadataImpl; import org.junit.Before; @@ -125,7 +126,9 @@ public void before() { new PartitionInfo("topic-four", 0, null, null, null)); cluster = new Cluster(null, Collections.emptyList(), partitionInfos, Collections.emptySet(), Collections.emptySet()); - metadataState = new StreamsMetadataState(TopologyWrapper.getInternalTopologyBuilder(builder.build()), hostOne); + final TopologyMetadata topologyMetadata = new TopologyMetadata(TopologyWrapper.getInternalTopologyBuilder(builder.build()), new DummyStreamsConfig()); + topologyMetadata.buildAndRewriteTopology(); + metadataState = new StreamsMetadataState(topologyMetadata, hostOne); metadataState.onChange(hostToActivePartitions, hostToStandbyPartitions, cluster); partitioner = (topic, key, value, numPartitions) -> 1; storeNames = mkSet("table-one", "table-two", "merged-table", globalTable); @@ -134,7 +137,9 @@ public void before() { @Test public void shouldNotThrowExceptionWhenOnChangeNotCalled() { final Collection metadata = new StreamsMetadataState( - TopologyWrapper.getInternalTopologyBuilder(builder.build()), hostOne).getAllMetadataForStore("store"); + new TopologyMetadata(TopologyWrapper.getInternalTopologyBuilder(builder.build()), new DummyStreamsConfig()), + hostOne + ).getAllMetadataForStore("store"); assertEquals(0, metadata.size()); } @@ -325,7 +330,10 @@ public void shouldGetQueryMetadataForGlobalStoreWithKey() { @Test public void shouldGetAnyHostForGlobalStoreByKeyIfMyHostUnknown() { - final StreamsMetadataState streamsMetadataState = new StreamsMetadataState(TopologyWrapper.getInternalTopologyBuilder(builder.build()), StreamsMetadataState.UNKNOWN_HOST); + final StreamsMetadataState streamsMetadataState = new StreamsMetadataState( + new TopologyMetadata(TopologyWrapper.getInternalTopologyBuilder(builder.build()), new DummyStreamsConfig()), + StreamsMetadataState.UNKNOWN_HOST + ); streamsMetadataState.onChange(hostToActivePartitions, hostToStandbyPartitions, cluster); assertNotNull(streamsMetadataState.getKeyQueryMetadataForKey(globalTable, "key", Serdes.String().serializer())); } @@ -339,7 +347,10 @@ public void shouldGetQueryMetadataForGlobalStoreWithKeyAndPartitioner() { @Test public void shouldGetAnyHostForGlobalStoreByKeyAndPartitionerIfMyHostUnknown() { - final StreamsMetadataState streamsMetadataState = new StreamsMetadataState(TopologyWrapper.getInternalTopologyBuilder(builder.build()), StreamsMetadataState.UNKNOWN_HOST); + final StreamsMetadataState streamsMetadataState = new StreamsMetadataState( + new TopologyMetadata(TopologyWrapper.getInternalTopologyBuilder(builder.build()), new DummyStreamsConfig()), + StreamsMetadataState.UNKNOWN_HOST + ); streamsMetadataState.onChange(hostToActivePartitions, hostToStandbyPartitions, cluster); assertNotNull(streamsMetadataState.getKeyQueryMetadataForKey(globalTable, "key", partitioner)); } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java index 7244bd6e881dd..173fc141db426 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java @@ -191,6 +191,7 @@ public class StreamsPartitionAssignorTest { private TaskManager taskManager; private Admin adminClient; private InternalTopologyBuilder builder = new InternalTopologyBuilder(); + private TopologyMetadata topologyMetadata; private StreamsMetadataState streamsMetadataState = EasyMock.createNiceMock(StreamsMetadataState.class); private final Map subscriptions = new HashMap<>(); private final Class taskAssignor; @@ -241,11 +242,11 @@ private void createDefaultMockTaskManager() { private void createMockTaskManager(final Set activeTasks, final Set standbyTasks) { taskManager = EasyMock.createNiceMock(TaskManager.class); - expect(taskManager.builder()).andStubReturn(builder); + expect(taskManager.topologyMetadata()).andStubReturn(topologyMetadata); expect(taskManager.getTaskOffsetSums()).andStubReturn(getTaskOffsetSums(activeTasks, standbyTasks)); expect(taskManager.processId()).andStubReturn(UUID_1); builder.setApplicationId(APPLICATION_ID); - builder.buildTopology(); + topologyMetadata.buildAndRewriteTopology(); } // If mockCreateInternalTopics is true the internal topic manager will report that it had to create all internal @@ -273,6 +274,7 @@ public static Collection parameters() { public StreamsPartitionAssignorTest(final Class taskAssignor) { this.taskAssignor = taskAssignor; adminClient = createMockAdminClientForAssignor(EMPTY_CHANGELOG_END_OFFSETS); + topologyMetadata = new TopologyMetadata(builder, new StreamsConfig(configProps())); } @Test @@ -823,7 +825,7 @@ public void testAssignWithStates() { private static Set tasksForState(final String storeName, final List tasks, final Map topicGroups) { - final String changelogTopic = ProcessorStateManager.storeChangelogTopic(APPLICATION_ID, storeName); + final String changelogTopic = ProcessorStateManager.storeChangelogTopic(APPLICATION_ID, storeName, null); final Set ids = new HashSet<>(); for (final Map.Entry entry : topicGroups.entrySet()) { @@ -1111,6 +1113,7 @@ public void shouldGenerateTasksForAllCreatedPartitions() { final String client = "client1"; builder = TopologyWrapper.getInternalTopologyBuilder(streamsBuilder.build()); + topologyMetadata = new TopologyMetadata(builder, new StreamsConfig(configProps())); adminClient = createMockAdminClientForAssignor(getTopicPartitionOffsetsMap( asList(APPLICATION_ID + "-topic3-STATE-STORE-0000000002-changelog", @@ -1194,18 +1197,20 @@ public Set makeReady(final Map topics) { @Test public void shouldThrowTimeoutExceptionWhenCreatingChangelogTopicsTimesOut() { + final StreamsConfig config = new StreamsConfig(configProps()); final StreamsBuilder streamsBuilder = new StreamsBuilder(); streamsBuilder.table("topic1", Materialized.as("store")); final String client = "client1"; builder = TopologyWrapper.getInternalTopologyBuilder(streamsBuilder.build()); + topologyMetadata = new TopologyMetadata(builder, config); createDefaultMockTaskManager(); EasyMock.replay(taskManager); partitionAssignor.configure(configProps()); final MockInternalTopicManager mockInternalTopicManager = new MockInternalTopicManager( time, - new StreamsConfig(configProps()), + config, mockClientSupplier.restoreConsumer, false ) { @@ -1437,9 +1442,14 @@ public void shouldTriggerImmediateRebalanceOnTasksRevoked() { @Test public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() { + final Map props = configProps(); + props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1); + props.put(StreamsConfig.APPLICATION_SERVER_CONFIG, USER_END_POINT); + final StreamsBuilder streamsBuilder = new StreamsBuilder(); streamsBuilder.stream("topic1").groupByKey().count(); builder = TopologyWrapper.getInternalTopologyBuilder(streamsBuilder.build()); + topologyMetadata = new TopologyMetadata(builder, new StreamsConfig(props)); createDefaultMockTaskManager(); adminClient = createMockAdminClientForAssignor(getTopicPartitionOffsetsMap( @@ -1447,10 +1457,6 @@ public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() { singletonList(3)) ); - final Map props = new HashMap<>(); - props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1); - props.put(StreamsConfig.APPLICATION_SERVER_CONFIG, USER_END_POINT); - configurePartitionAssignorWith(props); subscriptions.put("consumer1", @@ -1917,8 +1923,10 @@ public void shouldRequestCommittedOffsetsForPreexistingSourceChangelogs() { streamsBuilder.table("topic1", Materialized.as("store")); final Properties props = new Properties(); + props.putAll(configProps()); props.put(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, StreamsConfig.OPTIMIZE); builder = TopologyWrapper.getInternalTopologyBuilder(streamsBuilder.build(props)); + topologyMetadata = new TopologyMetadata(builder, new StreamsConfig(props)); subscriptions.put("consumer10", new Subscription( @@ -1993,6 +2001,8 @@ public void testUniqueFieldOverflow() { @Test public void shouldThrowTaskAssignmentExceptionWhenUnableToResolvePartitionCount() { builder = new CorruptedInternalTopologyBuilder(); + topologyMetadata = new TopologyMetadata(builder, new StreamsConfig(configProps())); + final InternalStreamsBuilder streamsBuilder = new InternalStreamsBuilder(builder); final KStream inputTopic = streamsBuilder.stream(singleton("topic1"), new ConsumedInternal<>()); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java index 60dd1907fa1bf..16584040f1b9e 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/TaskManagerTest.java @@ -47,6 +47,7 @@ import org.apache.kafka.streams.processor.internals.StreamThread.ProcessingMode; import org.apache.kafka.streams.processor.internals.Task.State; import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl; +import org.apache.kafka.streams.processor.internals.testutil.DummyStreamsConfig; import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender; import org.apache.kafka.streams.state.internals.OffsetCheckpoint; @@ -188,12 +189,14 @@ private void setUpTaskManager(final StreamThread.ProcessingMode processingMode) new StreamsMetricsImpl(new Metrics(), "clientId", StreamsConfig.METRICS_LATEST, time), activeTaskCreator, standbyTaskCreator, - topologyBuilder, + new TopologyMetadata(topologyBuilder, new DummyStreamsConfig()), adminClient, stateDirectory, processingMode ); taskManager.setMainConsumer(consumer); + reset(topologyBuilder); + expect(topologyBuilder.hasNamedTopology()).andStubReturn(false); } @Test @@ -202,8 +205,10 @@ public void shouldIdempotentlyUpdateSubscriptionFromActiveAssignment() { final Map> assignment = mkMap(mkEntry(taskId01, mkSet(t1p1, newTopicPartition))); expect(activeTaskCreator.createTasks(anyObject(), eq(assignment))).andStubReturn(emptyList()); + topologyBuilder.addSubscribedTopicsFromAssignment(eq(asList(t1p1, newTopicPartition)), anyString()); expectLastCall(); + replay(activeTaskCreator, topologyBuilder); taskManager.handleAssignment(assignment, emptyMap()); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/testutil/DummyStreamsConfig.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/testutil/DummyStreamsConfig.java new file mode 100644 index 0000000000000..017461941e615 --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/testutil/DummyStreamsConfig.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.processor.internals.testutil; + +import org.apache.kafka.streams.StreamsConfig; + +import java.util.Properties; + +public class DummyStreamsConfig extends StreamsConfig { + + private final static Properties PROPS = dummyProps(); + + public DummyStreamsConfig() { + super(PROPS); + } + + private static Properties dummyProps() { + final Properties props = new Properties(); + props.put(StreamsConfig.APPLICATION_ID_CONFIG, "dummy-application"); + props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:2171"); + return props; + } +} diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyKeyValueStoreTest.java index 652694bbbace9..510ccfc8bd74b 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyKeyValueStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/CompositeReadOnlyKeyValueStoreTest.java @@ -80,7 +80,7 @@ private KeyValueStore newStoreInstance() { @SuppressWarnings("rawtypes") final InternalMockProcessorContext context = new InternalMockProcessorContext<>( new StateSerdes<>( - ProcessorStateManager.storeChangelogTopic("appId", storeName), + ProcessorStateManager.storeChangelogTopic("appId", storeName, null), Serdes.String(), Serdes.String() ), diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStoreTest.java index 3e5c3d78c5960..138359073b40f 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredKeyValueStoreTest.java @@ -93,7 +93,7 @@ public class MeteredKeyValueStoreTest { private static final KeyValue BYTE_KEY_VALUE_PAIR = KeyValue.pair(KEY_BYTES, VALUE_BYTES); private final String threadId = Thread.currentThread().getName(); - private final TaskId taskId = new TaskId(0, 0); + private final TaskId taskId = new TaskId(0, 0, "My-Topology"); @Mock(type = MockType.NICE) private KeyValueStore inner; @@ -178,7 +178,7 @@ public void shouldPassChangelogTopicNameToStateStoreSerde() { @Test public void shouldPassDefaultChangelogTopicNameToStateStoreSerdeIfLoggingDisabled() { - final String defaultChangelogTopicName = ProcessorStateManager.storeChangelogTopic(APPLICATION_ID, STORE_NAME); + final String defaultChangelogTopicName = ProcessorStateManager.storeChangelogTopic(APPLICATION_ID, STORE_NAME, taskId.namedTopology()); expect(context.changelogFor(STORE_NAME)).andReturn(null); doShouldPassChangelogTopicNameToStateStoreSerde(defaultChangelogTopicName); } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java index b3bea75f404cf..8f898d8bec5ca 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredSessionStoreTest.java @@ -97,7 +97,7 @@ public class MeteredSessionStoreTest { private static final long END_TIMESTAMP = 42L; private final String threadId = Thread.currentThread().getName(); - private final TaskId taskId = new TaskId(0, 0); + private final TaskId taskId = new TaskId(0, 0, "My-Topology"); private final Metrics metrics = new Metrics(); private MeteredSessionStore store; @Mock(type = MockType.NICE) @@ -181,7 +181,7 @@ public void shouldPassChangelogTopicNameToStateStoreSerde() { @Test public void shouldPassDefaultChangelogTopicNameToStateStoreSerdeIfLoggingDisabled() { final String defaultChangelogTopicName = - ProcessorStateManager.storeChangelogTopic(APPLICATION_ID, STORE_NAME); + ProcessorStateManager.storeChangelogTopic(APPLICATION_ID, STORE_NAME, taskId.namedTopology()); expect(context.changelogFor(STORE_NAME)).andReturn(null); doShouldPassChangelogTopicNameToStateStoreSerde(defaultChangelogTopicName); } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedKeyValueStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedKeyValueStoreTest.java index bce87ef2fbeec..af66098d88416 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedKeyValueStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedKeyValueStoreTest.java @@ -93,7 +93,7 @@ public class MeteredTimestampedKeyValueStoreTest { private final String threadId = Thread.currentThread().getName(); - private final TaskId taskId = new TaskId(0, 0); + private final TaskId taskId = new TaskId(0, 0, "My-Topology"); @Mock(type = MockType.NICE) private KeyValueStore inner; @Mock(type = MockType.NICE) @@ -187,7 +187,7 @@ public void shouldPassChangelogTopicNameToStateStoreSerde() { @Test public void shouldPassDefaultChangelogTopicNameToStateStoreSerdeIfLoggingDisabled() { - final String defaultChangelogTopicName = ProcessorStateManager.storeChangelogTopic(APPLICATION_ID, STORE_NAME); + final String defaultChangelogTopicName = ProcessorStateManager.storeChangelogTopic(APPLICATION_ID, STORE_NAME, taskId.namedTopology()); expect(context.changelogFor(STORE_NAME)).andReturn(null); doShouldPassChangelogTopicNameToStateStoreSerde(defaultChangelogTopicName); } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java index 426675184cb68..bc606919339d4 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredTimestampedWindowStoreTest.java @@ -31,6 +31,7 @@ import org.apache.kafka.streams.errors.StreamsException; import org.apache.kafka.streams.processor.ProcessorContext; import org.apache.kafka.streams.processor.StateStoreContext; +import org.apache.kafka.streams.processor.TaskId; import org.apache.kafka.streams.processor.internals.ProcessorStateManager; import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl; import org.apache.kafka.streams.state.ValueAndTimestamp; @@ -67,6 +68,7 @@ public class MeteredTimestampedWindowStoreTest { private static final int WINDOW_SIZE_MS = 10; private InternalMockProcessorContext context; + private final TaskId taskId = new TaskId(0, 0, "My-Topology"); private final WindowStore innerStoreMock = EasyMock.createNiceMock(WindowStore.class); private final Metrics metrics = new Metrics(new MetricConfig().recordLevel(Sensor.RecordingLevel.DEBUG)); private MeteredTimestampedWindowStore store = new MeteredTimestampedWindowStore<>( @@ -95,7 +97,8 @@ public void setUp() { new StreamsConfig(StreamsTestUtils.getStreamsConfig()), MockRecordCollector::new, new ThreadCache(new LogContext("testCache "), 0, streamsMetrics), - Time.SYSTEM + Time.SYSTEM, + taskId ); } @@ -147,7 +150,7 @@ public void shouldPassChangelogTopicNameToStateStoreSerde() { @Test public void shouldPassDefaultChangelogTopicNameToStateStoreSerdeIfLoggingDisabled() { final String defaultChangelogTopicName = - ProcessorStateManager.storeChangelogTopic(context.applicationId(), STORE_NAME); + ProcessorStateManager.storeChangelogTopic(context.applicationId(), STORE_NAME, taskId.namedTopology()); doShouldPassChangelogTopicNameToStateStoreSerde(defaultChangelogTopicName); } diff --git a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java index 461c50c617a5a..41676c9b98c75 100644 --- a/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/state/internals/MeteredWindowStoreTest.java @@ -171,7 +171,7 @@ public void shouldPassChangelogTopicNameToStateStoreSerde() { @Test public void shouldPassDefaultChangelogTopicNameToStateStoreSerdeIfLoggingDisabled() { final String defaultChangelogTopicName = - ProcessorStateManager.storeChangelogTopic(context.applicationId(), STORE_NAME); + ProcessorStateManager.storeChangelogTopic(context.applicationId(), STORE_NAME, context.taskId().namedTopology()); doShouldPassChangelogTopicNameToStateStoreSerde(defaultChangelogTopicName); } diff --git a/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java b/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java index b19fdf59a5314..c3cf64af7e966 100644 --- a/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java +++ b/streams/src/test/java/org/apache/kafka/test/InternalMockProcessorContext.java @@ -202,8 +202,20 @@ public InternalMockProcessorContext(final File stateDir, final RecordCollector.Supplier collectorSupplier, final ThreadCache cache, final Time time) { + this(stateDir, keySerde, valueSerde, metrics, config, collectorSupplier, cache, time, new TaskId(0, 0)); + } + + public InternalMockProcessorContext(final File stateDir, + final Serde keySerde, + final Serde valueSerde, + final StreamsMetricsImpl metrics, + final StreamsConfig config, + final RecordCollector.Supplier collectorSupplier, + final ThreadCache cache, + final Time time, + final TaskId taskId) { super( - new TaskId(0, 0), + taskId, config, metrics, cache diff --git a/streams/src/test/java/org/apache/kafka/test/NoOpProcessorContext.java b/streams/src/test/java/org/apache/kafka/test/NoOpProcessorContext.java index b3243cd8b0524..c42601191b432 100644 --- a/streams/src/test/java/org/apache/kafka/test/NoOpProcessorContext.java +++ b/streams/src/test/java/org/apache/kafka/test/NoOpProcessorContext.java @@ -147,6 +147,6 @@ public void registerCacheFlushListener(final String namespace, final DirtyEntryF @Override public String changelogFor(final String storeName) { - return ProcessorStateManager.storeChangelogTopic(applicationId(), storeName); + return ProcessorStateManager.storeChangelogTopic(applicationId(), storeName, taskId().namedTopology()); } } diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java index 78924b543be91..70ea243b0bac1 100644 --- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java +++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java @@ -417,9 +417,8 @@ private void setupTopology(final InternalTopologyBuilder builder, offsetsByTopicOrPatternPartition.put(tp, new AtomicLong()); } - final boolean createStateDirectory = processorTopology.hasPersistentLocalStore() || - (globalTopology != null && globalTopology.hasPersistentGlobalStore()); - stateDirectory = new StateDirectory(streamsConfig, mockWallClockTime, createStateDirectory, builder.hasNamedTopologies()); + // TODO KAFKA-12648: The TTD does not yet work with NamedTopologies, so just pass in false + stateDirectory = new StateDirectory(streamsConfig, mockWallClockTime, internalTopologyBuilder.hasPersistentStores(), false); } private void setupGlobalTask(final Time mockWallClockTime, @@ -879,7 +878,7 @@ final boolean isEmpty(final String topic) { */ public Map getAllStateStores() { final Map allStores = new HashMap<>(); - for (final String storeName : internalTopologyBuilder.allStateStoreName()) { + for (final String storeName : internalTopologyBuilder.allStateStoreNames()) { allStores.put(storeName, getStateStore(storeName, false)); } return allStores;