diff --git a/bin/copycat-distributed.sh b/bin/copycat-distributed.sh new file mode 100755 index 0000000000000..4d62300e00d13 --- /dev/null +++ b/bin/copycat-distributed.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +base_dir=$(dirname $0) + +if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then + export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/copycat-log4j.properties" +fi + +exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.copycat.cli.CopycatDistributed "$@" diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 2c5df0bec16e1..7b748eca56f8a 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -27,6 +27,9 @@ + + + @@ -138,6 +141,7 @@ + @@ -145,20 +149,18 @@ + - - - - + @@ -166,9 +168,6 @@ - - - @@ -184,6 +183,7 @@ + diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java index b21ec89ee0fdb..a3d8776083032 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/Consumer.java @@ -79,7 +79,7 @@ public interface Consumer extends Closeable { /** * @see KafkaConsumer#commitSync(Map) */ - public void commitSync(Map offsets); + public void commitSync(Map offsets); /** * @see KafkaConsumer#commitAsync() @@ -94,7 +94,7 @@ public interface Consumer extends Closeable { /** * @see KafkaConsumer#commitAsync(Map, OffsetCommitCallback) */ - public void commitAsync(Map offsets, OffsetCommitCallback callback); + public void commitAsync(Map offsets, OffsetCommitCallback callback); /** * @see KafkaConsumer#seek(TopicPartition, long) @@ -119,7 +119,7 @@ public interface Consumer extends Closeable { /** * @see KafkaConsumer#committed(TopicPartition) */ - public long committed(TopicPartition partition); + public OffsetAndMetadata committed(TopicPartition partition); /** * @see KafkaConsumer#metrics() diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index 93db34161f64b..68f61bff7b0f2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -48,6 +48,7 @@ import java.util.Collection; import java.util.Collections; import java.util.ConcurrentModificationException; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -776,6 +777,8 @@ public void assign(List partitions) { * * @throws NoOffsetForPartitionException If there is no stored offset for a subscribed partition and no automatic * offset reset policy has been configured. + * @throws org.apache.kafka.common.errors.OffsetOutOfRangeException If there is OffsetOutOfRange error in fetchResponse and + * the defaultResetPolicy is NONE */ @Override public ConsumerRecords poll(long timeout) { @@ -813,6 +816,8 @@ public ConsumerRecords poll(long timeout) { * heart-beating, auto-commits, and offset updates. * @param timeout The maximum time to block in the underlying poll * @return The fetched records (may be empty) + * @throws org.apache.kafka.common.errors.OffsetOutOfRangeException If there is OffsetOutOfRange error in fetchResponse and + * the defaultResetPolicy is NONE */ private Map>> pollOnce(long timeout) { // TODO: Sub-requests should take into account the poll timeout (KAFKA-1894) @@ -829,6 +834,12 @@ private Map>> pollOnce(long timeout) { // init any new fetches (won't resend pending fetches) Cluster cluster = this.metadata.fetch(); + Map>> records = fetcher.fetchedRecords(); + // Avoid block waiting for response if we already have data available, e.g. from another API call to commit. + if (!records.isEmpty()) { + client.poll(0); + return records; + } fetcher.initFetches(cluster); client.poll(timeout); return fetcher.fetchedRecords(); @@ -839,7 +850,7 @@ private void scheduleAutoCommitTask(final long interval) { public void run(long now) { commitAsync(new OffsetCommitCallback() { @Override - public void onComplete(Map offsets, Exception exception) { + public void onComplete(Map offsets, Exception exception) { if (exception != null) log.error("Auto offset commit failed.", exception); } @@ -880,10 +891,10 @@ public void commitSync() { * This is a synchronous commits and will block until either the commit succeeds or an unrecoverable error is * encountered (in which case it is thrown to the caller). * - * @param offsets The list of offsets per partition that should be committed to Kafka. + * @param offsets A map of offsets by partition with associated metadata */ @Override - public void commitSync(final Map offsets) { + public void commitSync(final Map offsets) { acquire(); try { coordinator.commitOffsetsSync(offsets); @@ -932,15 +943,16 @@ public void commitAsync(OffsetCommitCallback callback) { * This is an asynchronous call and will not block. Any errors encountered are either passed to the callback * (if provided) or discarded. * - * @param offsets The list of offsets per partition that should be committed to Kafka. + * @param offsets A map of offsets by partition with associate metadata. This map will be copied internally, so it + * is safe to mutate the map after returning. * @param callback Callback to invoke when the commit completes */ @Override - public void commitAsync(final Map offsets, OffsetCommitCallback callback) { + public void commitAsync(final Map offsets, OffsetCommitCallback callback) { acquire(); try { log.debug("Committing offsets: {} ", offsets); - coordinator.commitOffsetsAsync(offsets, callback); + coordinator.commitOffsetsAsync(new HashMap<>(offsets), callback); } finally { release(); } @@ -1013,10 +1025,9 @@ public long position(TopicPartition partition) { Long offset = this.subscriptions.consumed(partition); if (offset == null) { updateFetchPositions(Collections.singleton(partition)); - return this.subscriptions.consumed(partition); - } else { - return offset; + offset = this.subscriptions.consumed(partition); } + return offset; } finally { release(); } @@ -1030,15 +1041,13 @@ public long position(TopicPartition partition) { * consumer hasn't yet initialized its cache of committed offsets. * * @param partition The partition to check - * @return The last committed offset - * @throws NoOffsetForPartitionException If no offset has ever been committed by any process for the given - * partition. + * @return The last committed offset and metadata or null if there was no prior commit */ @Override - public long committed(TopicPartition partition) { + public OffsetAndMetadata committed(TopicPartition partition) { acquire(); try { - Long committed; + OffsetAndMetadata committed; if (subscriptions.isAssigned(partition)) { committed = this.subscriptions.committed(partition); if (committed == null) { @@ -1046,13 +1055,10 @@ public long committed(TopicPartition partition) { committed = this.subscriptions.committed(partition); } } else { - Map offsets = coordinator.fetchCommittedOffsets(Collections.singleton(partition)); + Map offsets = coordinator.fetchCommittedOffsets(Collections.singleton(partition)); committed = offsets.get(partition); } - if (committed == null) - throw new NoOffsetForPartitionException("No offset has been committed for partition " + partition); - return committed; } finally { release(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java index ba196dd55f196..3c0f26114d029 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java @@ -14,27 +14,32 @@ import org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener; import org.apache.kafka.clients.consumer.internals.SubscriptionState; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.TimeoutException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.regex.Pattern; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; /** * A mock of the {@link Consumer} interface you can use for testing code that uses Kafka. This class is not - * threadsafe - *

- * The consumer runs in the user thread and multiplexes I/O over TCP connections to each of the brokers it needs to - * communicate with. Failure to close the consumer after use will leak these resources. + * threadsafe . However, you can use the {@link #waitForPollThen(Runnable,long)} method to write multithreaded tests + * where a driver thread waits for {@link #poll(long)} to be called and then can safely perform operations during a + * callback. */ public class MockConsumer implements Consumer { @@ -43,6 +48,13 @@ public class MockConsumer implements Consumer { private Map>> records; private Set paused; private boolean closed; + private final Map beginningOffsets; + private final Map endOffsets; + + private AtomicReference pollLatch; + private KafkaException exception; + + private AtomicBoolean wakeup; public MockConsumer(OffsetResetStrategy offsetResetStrategy) { this.subscriptions = new SubscriptionState(offsetResetStrategy); @@ -50,20 +62,32 @@ public MockConsumer(OffsetResetStrategy offsetResetStrategy) { this.records = new HashMap<>(); this.paused = new HashSet<>(); this.closed = false; + this.beginningOffsets = new HashMap<>(); + this.endOffsets = new HashMap<>(); + this.pollLatch = new AtomicReference<>(); + this.exception = null; + this.wakeup = new AtomicBoolean(false); } @Override - public synchronized Set assignment() { + public Set assignment() { return this.subscriptions.assignedPartitions(); } + /** Simulate a rebalance event. */ + public void rebalance(Collection newAssignment) { + // TODO: Rebalance callbacks + this.records.clear(); + this.subscriptions.changePartitionAssignment(newAssignment); + } + @Override - public synchronized Set subscription() { + public Set subscription() { return this.subscriptions.subscription(); } @Override - public synchronized void subscribe(List topics) { + public void subscribe(List topics) { subscribe(topics, new NoOpConsumerRebalanceListener()); } @@ -82,13 +106,13 @@ public void subscribe(Pattern pattern, final ConsumerRebalanceListener listener) } @Override - public synchronized void subscribe(List topics, final ConsumerRebalanceListener listener) { + public void subscribe(List topics, final ConsumerRebalanceListener listener) { ensureNotClosed(); this.subscriptions.subscribe(topics, listener); } @Override - public synchronized void assign(List partitions) { + public void assign(List partitions) { ensureNotClosed(); this.subscriptions.assign(partitions); } @@ -100,14 +124,39 @@ public void unsubscribe() { } @Override - public synchronized ConsumerRecords poll(long timeout) { + public ConsumerRecords poll(long timeout) { ensureNotClosed(); + + CountDownLatch pollLatchCopy = pollLatch.get(); + if (pollLatchCopy != null) { + pollLatch.set(null); + pollLatchCopy.countDown(); + synchronized (pollLatchCopy) { + // Will block until caller of waitUntilPollThen() finishes their callback. + } + } + + if (wakeup.get()) { + wakeup.set(false); + throw new ConsumerWakeupException(); + } + + if (exception != null) { + RuntimeException exception = this.exception; + this.exception = null; + throw exception; + } + + // Handle seeks that need to wait for a poll() call to be processed + for (TopicPartition tp : subscriptions.missingFetchPositions()) + updateFetchPosition(tp); + // update the consumed offset for (Map.Entry>> entry : this.records.entrySet()) { if (!subscriptions.isPaused(entry.getKey())) { List> recs = entry.getValue(); if (!recs.isEmpty()) - this.subscriptions.consumed(entry.getKey(), recs.get(recs.size() - 1).offset()); + this.subscriptions.consumed(entry.getKey(), recs.get(recs.size() - 1).offset() + 1); } } @@ -116,15 +165,12 @@ public synchronized ConsumerRecords poll(long timeout) { return copy; } - public synchronized void addRecord(ConsumerRecord record) { + public void addRecord(ConsumerRecord record) { ensureNotClosed(); TopicPartition tp = new TopicPartition(record.topic(), record.partition()); - ArrayList currentAssigned = new ArrayList<>(this.subscriptions.assignedPartitions()); - if (!currentAssigned.contains(tp)) { - currentAssigned.add(tp); - this.subscriptions.changePartitionAssignment(currentAssigned); - } - subscriptions.seek(tp, record.offset()); + Set currentAssigned = new HashSet<>(this.subscriptions.assignedPartitions()); + if (!currentAssigned.contains(tp)) + throw new IllegalStateException("Cannot add records for a partition that is not assigned to the consumer"); List> recs = this.records.get(tp); if (recs == null) { recs = new ArrayList>(); @@ -133,10 +179,14 @@ public synchronized void addRecord(ConsumerRecord record) { recs.add(record); } + public void setException(KafkaException exception) { + this.exception = exception; + } + @Override - public synchronized void commitAsync(Map offsets, OffsetCommitCallback callback) { + public void commitAsync(Map offsets, OffsetCommitCallback callback) { ensureNotClosed(); - for (Entry entry : offsets.entrySet()) + for (Map.Entry entry : offsets.entrySet()) subscriptions.committed(entry.getKey(), entry.getValue()); if (callback != null) { callback.onComplete(offsets, null); @@ -144,54 +194,71 @@ public synchronized void commitAsync(Map offsets, OffsetCo } @Override - public synchronized void commitSync(Map offsets) { + public void commitSync(Map offsets) { commitAsync(offsets, null); } @Override - public synchronized void commitAsync() { + public void commitAsync() { commitAsync(null); } @Override - public synchronized void commitAsync(OffsetCommitCallback callback) { + public void commitAsync(OffsetCommitCallback callback) { ensureNotClosed(); commitAsync(this.subscriptions.allConsumed(), callback); } @Override - public synchronized void commitSync() { + public void commitSync() { commitSync(this.subscriptions.allConsumed()); } @Override - public synchronized void seek(TopicPartition partition, long offset) { + public void seek(TopicPartition partition, long offset) { ensureNotClosed(); subscriptions.seek(partition, offset); } @Override - public synchronized long committed(TopicPartition partition) { + public OffsetAndMetadata committed(TopicPartition partition) { ensureNotClosed(); return subscriptions.committed(partition); } @Override - public synchronized long position(TopicPartition partition) { + public long position(TopicPartition partition) { ensureNotClosed(); - return subscriptions.consumed(partition); + if (!this.subscriptions.isAssigned(partition)) + throw new IllegalArgumentException("You can only check the position for partitions assigned to this consumer."); + Long offset = this.subscriptions.consumed(partition); + if (offset == null) { + updateFetchPosition(partition); + offset = this.subscriptions.consumed(partition); + } + return offset; } @Override - public synchronized void seekToBeginning(TopicPartition... partitions) { + public void seekToBeginning(TopicPartition... partitions) { ensureNotClosed(); - throw new UnsupportedOperationException(); + for (TopicPartition tp : partitions) + subscriptions.needOffsetReset(tp, OffsetResetStrategy.EARLIEST); + } + + public void updateBeginningOffsets(Map newOffsets) { + beginningOffsets.putAll(newOffsets); } @Override - public synchronized void seekToEnd(TopicPartition... partitions) { + public void seekToEnd(TopicPartition... partitions) { ensureNotClosed(); - throw new UnsupportedOperationException(); + for (TopicPartition tp : partitions) + subscriptions.needOffsetReset(tp, OffsetResetStrategy.LATEST); + } + + public void updateEndOffsets(Map newOffsets) { + endOffsets.putAll(newOffsets); } @Override @@ -201,7 +268,7 @@ public synchronized void seekToEnd(TopicPartition... partitions) { } @Override - public synchronized List partitionsFor(String topic) { + public List partitionsFor(String topic) { ensureNotClosed(); List parts = this.partitions.get(topic); if (parts == null) @@ -216,7 +283,7 @@ public Map> listTopics() { return partitions; } - public synchronized void updatePartitions(String topic, List partitions) { + public void updatePartitions(String topic, List partitions) { ensureNotClosed(); this.partitions.put(topic, partitions); } @@ -238,14 +305,37 @@ public void resume(TopicPartition... partitions) { } @Override - public synchronized void close() { + public void close() { ensureNotClosed(); this.closed = true; } + public boolean closed() { + return this.closed; + } + @Override public void wakeup() { + wakeup.set(true); + } + public void waitForPoll(long timeoutMs) { + waitForPollThen(null, timeoutMs); + } + + public void waitForPollThen(Runnable task, long timeoutMs) { + CountDownLatch latch = new CountDownLatch(1); + synchronized (latch) { + pollLatch.set(latch); + try { + if (!latch.await(timeoutMs, TimeUnit.MILLISECONDS)) + throw new TimeoutException("Timed out waiting for consumer thread to call poll()."); + } catch (InterruptedException e) { + throw new IllegalStateException("MockConsumer waiting thread was interrupted.", e); + } + if (task != null) + task.run(); + } } public Set paused() { @@ -256,4 +346,32 @@ private void ensureNotClosed() { if (this.closed) throw new IllegalStateException("This consumer has already been closed."); } + + private void updateFetchPosition(TopicPartition tp) { + if (subscriptions.isOffsetResetNeeded(tp)) { + resetOffsetPosition(tp); + } else if (subscriptions.committed(tp) == null) { + subscriptions.needOffsetReset(tp); + resetOffsetPosition(tp); + } else { + subscriptions.seek(tp, subscriptions.committed(tp).offset()); + } + } + + private void resetOffsetPosition(TopicPartition tp) { + OffsetResetStrategy strategy = subscriptions.resetStrategy(tp); + Long offset; + if (strategy == OffsetResetStrategy.EARLIEST) { + offset = beginningOffsets.get(tp); + if (offset == null) + throw new IllegalStateException("MockConsumer didn't have beginning offset specified, but tried to seek to beginning"); + } else if (strategy == OffsetResetStrategy.LATEST) { + offset = endOffsets.get(tp); + if (offset == null) + throw new IllegalStateException("MockConsumer didn't have end offset specified, but tried to seek to end"); + } else { + throw new NoOffsetForPartitionException("No offset available"); + } + seek(tp, offset); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetAndMetadata.java b/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetAndMetadata.java new file mode 100644 index 0000000000000..1a9304798bd2f --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetAndMetadata.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.apache.kafka.clients.consumer; + +/** + * The Kafka offset commit API allows users to provide additional metadata (in the form of a string) + * when an offset is committed. This can be useful (for example) to store information about which + * node made the commit, what time the commit was made, etc. + */ +public class OffsetAndMetadata { + private final long offset; + private final String metadata; + + /** + * Construct a new OffsetAndMetadata object for committing through {@link KafkaConsumer}. + * @param offset The offset to be committed + * @param metadata Non-null metadata + */ + public OffsetAndMetadata(long offset, String metadata) { + if (metadata == null) + throw new IllegalArgumentException("Metadata cannot be null"); + + this.offset = offset; + this.metadata = metadata; + } + + /** + * Construct a new OffsetAndMetadata object for committing through {@link KafkaConsumer}. The metadata + * associated with the commit will be empty. + * @param offset The offset to be committed + */ + public OffsetAndMetadata(long offset) { + this(offset, ""); + } + + public long offset() { + return offset; + } + + public String metadata() { + return metadata; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + OffsetAndMetadata that = (OffsetAndMetadata) o; + + if (offset != that.offset) return false; + return metadata == null ? that.metadata == null : metadata.equals(that.metadata); + + } + + @Override + public int hashCode() { + int result = (int) (offset ^ (offset >>> 32)); + result = 31 * result + (metadata != null ? metadata.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "OffsetAndMetadata{" + + "offset=" + offset + + ", metadata='" + metadata + '\'' + + '}'; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetCommitCallback.java b/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetCommitCallback.java index bed82f6a54f46..97a06ad48015f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetCommitCallback.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/OffsetCommitCallback.java @@ -26,8 +26,8 @@ public interface OffsetCommitCallback { * A callback method the user can implement to provide asynchronous handling of commit request completion. * This method will be called when the commit request sent to the server has been acknowledged. * - * @param offsets A map of the offsets that this callback applies to + * @param offsets A map of the offsets and associated metadata that this callback applies to * @param exception The exception thrown during processing of the request, or null if the commit completed successfully */ - void onComplete(Map offsets, Exception exception); + void onComplete(Map offsets, Exception exception); } \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Coordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Coordinator.java index 5efe300cdbd0a..0b31c2275aa44 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Coordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Coordinator.java @@ -14,12 +14,15 @@ import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.DisconnectException; +import org.apache.kafka.common.errors.IllegalGenerationException; +import org.apache.kafka.common.errors.UnknownConsumerIdException; import org.apache.kafka.common.metrics.Measurable; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -114,8 +117,8 @@ public Coordinator(ConsumerNetworkClient client, */ public void refreshCommittedOffsetsIfNeeded() { if (subscriptions.refreshCommitsNeeded()) { - Map offsets = fetchCommittedOffsets(subscriptions.assignedPartitions()); - for (Map.Entry entry : offsets.entrySet()) { + Map offsets = fetchCommittedOffsets(subscriptions.assignedPartitions()); + for (Map.Entry entry : offsets.entrySet()) { TopicPartition tp = entry.getKey(); // verify assignment is still active if (subscriptions.isAssigned(tp)) @@ -130,13 +133,12 @@ public void refreshCommittedOffsetsIfNeeded() { * @param partitions The partitions to fetch offsets for * @return A map from partition to the committed offset */ - public Map fetchCommittedOffsets(Set partitions) { + public Map fetchCommittedOffsets(Set partitions) { while (true) { ensureCoordinatorKnown(); - ensurePartitionAssignment(); // contact coordinator to fetch committed offsets - RequestFuture> future = sendOffsetFetchRequest(partitions); + RequestFuture> future = sendOffsetFetchRequest(partitions); client.poll(future); if (future.succeeded()) @@ -196,7 +198,9 @@ private void reassignPartitions() { client.poll(future); if (future.failed()) { - if (!future.isRetriable()) + if (future.exception() instanceof UnknownConsumerIdException) + continue; + else if (!future.isRetriable()) throw future.exception(); Utils.sleep(retryBackoffMs); } @@ -349,7 +353,7 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture future) { } } - public void commitOffsetsAsync(final Map offsets, OffsetCommitCallback callback) { + public void commitOffsetsAsync(final Map offsets, OffsetCommitCallback callback) { this.subscriptions.needRefreshCommits(); RequestFuture future = sendOffsetCommitRequest(offsets); final OffsetCommitCallback cb = callback == null ? defaultOffsetCommitCallback : callback; @@ -366,10 +370,12 @@ public void onFailure(RuntimeException e) { }); } - public void commitOffsetsSync(Map offsets) { + public void commitOffsetsSync(Map offsets) { + if (offsets.isEmpty()) + return; + while (true) { ensureCoordinatorKnown(); - ensurePartitionAssignment(); RequestFuture future = sendOffsetCommitRequest(offsets); client.poll(future); @@ -394,7 +400,7 @@ public void commitOffsetsSync(Map offsets) { * @param offsets The list of offsets per partition that should be committed. * @return A request future whose value indicates whether the commit was successful or not */ - private RequestFuture sendOffsetCommitRequest(final Map offsets) { + private RequestFuture sendOffsetCommitRequest(final Map offsets) { if (coordinatorUnknown()) return RequestFuture.coordinatorNotAvailable(); @@ -402,10 +408,13 @@ private RequestFuture sendOffsetCommitRequest(final Map offsetData; - offsetData = new HashMap(offsets.size()); - for (Map.Entry entry : offsets.entrySet()) - offsetData.put(entry.getKey(), new OffsetCommitRequest.PartitionData(entry.getValue(), "")); + Map offsetData = new HashMap<>(offsets.size()); + for (Map.Entry entry : offsets.entrySet()) { + OffsetAndMetadata offsetAndMetadata = entry.getValue(); + offsetData.put(entry.getKey(), new OffsetCommitRequest.PartitionData( + offsetAndMetadata.offset(), offsetAndMetadata.metadata())); + } + OffsetCommitRequest req = new OffsetCommitRequest(this.groupId, this.generation, this.consumerId, @@ -418,7 +427,7 @@ private RequestFuture sendOffsetCommitRequest(final Map offsets, Exception exception) { + public void onComplete(Map offsets, Exception exception) { if (exception != null) log.error("Offset commit failed.", exception); } @@ -426,9 +435,9 @@ public void onComplete(Map offsets, Exception exception) { private class OffsetCommitResponseHandler extends CoordinatorResponseHandler { - private final Map offsets; + private final Map offsets; - public OffsetCommitResponseHandler(Map offsets) { + public OffsetCommitResponseHandler(Map offsets) { this.offsets = offsets; } @@ -442,38 +451,32 @@ public void handle(OffsetCommitResponse commitResponse, RequestFuture futu sensors.commitLatency.record(response.requestLatencyMs()); for (Map.Entry entry : commitResponse.responseData().entrySet()) { TopicPartition tp = entry.getKey(); - long offset = this.offsets.get(tp); + OffsetAndMetadata offsetAndMetadata = this.offsets.get(tp); + long offset = offsetAndMetadata.offset(); + short errorCode = entry.getValue(); if (errorCode == Errors.NONE.code()) { log.debug("Committed offset {} for partition {}", offset, tp); if (subscriptions.isAssigned(tp)) // update the local cache only if the partition is still assigned - subscriptions.committed(tp, offset); - } else if (errorCode == Errors.CONSUMER_COORDINATOR_NOT_AVAILABLE.code() - || errorCode == Errors.NOT_COORDINATOR_FOR_CONSUMER.code()) { - coordinatorDead(); - future.raise(Errors.forCode(errorCode)); - return; - } else if (errorCode == Errors.OFFSET_METADATA_TOO_LARGE.code() - || errorCode == Errors.INVALID_COMMIT_OFFSET_SIZE.code()) { - // do not need to throw the exception but just log the error + subscriptions.committed(tp, offsetAndMetadata); + } else { + if (errorCode == Errors.CONSUMER_COORDINATOR_NOT_AVAILABLE.code() + || errorCode == Errors.NOT_COORDINATOR_FOR_CONSUMER.code()) { + coordinatorDead(); + } else if (errorCode == Errors.UNKNOWN_CONSUMER_ID.code() + || errorCode == Errors.ILLEGAL_GENERATION.code()) { + // need to re-join group + subscriptions.needReassignment(); + } + log.error("Error committing partition {} at offset {}: {}", tp, offset, Errors.forCode(errorCode).exception().getMessage()); - } else if (errorCode == Errors.UNKNOWN_CONSUMER_ID.code() - || errorCode == Errors.ILLEGAL_GENERATION.code()) { - // need to re-join group - subscriptions.needReassignment(); + future.raise(Errors.forCode(errorCode)); return; - } else { - // do not need to throw the exception but just log the error - future.raise(Errors.forCode(errorCode)); - log.error("Error committing partition {} at offset {}: {}", - tp, - offset, - Errors.forCode(errorCode).exception().getMessage()); } } @@ -488,7 +491,7 @@ public void handle(OffsetCommitResponse commitResponse, RequestFuture futu * @param partitions The set of partitions to get offsets for. * @return A request future containing the committed offsets. */ - private RequestFuture> sendOffsetFetchRequest(Set partitions) { + private RequestFuture> sendOffsetFetchRequest(Set partitions) { if (coordinatorUnknown()) return RequestFuture.coordinatorNotAvailable(); @@ -501,7 +504,7 @@ private RequestFuture> sendOffsetFetchRequest(Set> { + private class OffsetFetchResponseHandler extends CoordinatorResponseHandler> { @Override public OffsetFetchResponse parse(ClientResponse response) { @@ -509,8 +512,8 @@ public OffsetFetchResponse parse(ClientResponse response) { } @Override - public void handle(OffsetFetchResponse response, RequestFuture> future) { - Map offsets = new HashMap(response.responseData().size()); + public void handle(OffsetFetchResponse response, RequestFuture> future) { + Map offsets = new HashMap<>(response.responseData().size()); for (Map.Entry entry : response.responseData().entrySet()) { TopicPartition tp = entry.getKey(); OffsetFetchResponse.PartitionData data = entry.getValue(); @@ -537,7 +540,7 @@ public void handle(OffsetFetchResponse response, RequestFuture= 0) { // record the position with the offset (-1 indicates no committed offset to fetch) - offsets.put(tp, data.offset); + offsets.put(tp, new OffsetAndMetadata(data.offset, data.metadata)); } else { log.debug("No committed offset for partition " + tp); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 0efd34d1c1294..4608959f01434 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -25,6 +25,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.DisconnectException; import org.apache.kafka.common.errors.InvalidMetadataException; +import org.apache.kafka.common.errors.OffsetOutOfRangeException; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Avg; @@ -78,6 +79,8 @@ public class Fetcher { private final Deserializer keyDeserializer; private final Deserializer valueDeserializer; + private final Map offsetOutOfRangePartitions; + public Fetcher(ConsumerNetworkClient client, int minBytes, int maxWaitMs, @@ -106,6 +109,7 @@ public Fetcher(ConsumerNetworkClient client, this.valueDeserializer = valueDeserializer; this.records = new LinkedList>(); + this.offsetOutOfRangePartitions = new HashMap<>(); this.sensors = new FetchManagerMetrics(metrics, metricGrpPrefix, metricTags); this.retryBackoffMs = retryBackoffMs; @@ -137,6 +141,7 @@ public void onFailure(RuntimeException e) { /** * Update the fetch positions for the provided partitions. * @param partitions + * @throws NoOffsetForPartitionException If no offset is stored for a given partition and no reset policy is available */ public void updateFetchPositions(Set partitions) { // reset the fetch position to the committed position @@ -152,9 +157,9 @@ public void updateFetchPositions(Set partitions) { subscriptions.needOffsetReset(tp); resetOffset(tp); } else { - log.debug("Resetting offset for partition {} to the committed offset {}", - tp, subscriptions.committed(tp)); - subscriptions.seek(tp, subscriptions.committed(tp)); + long committed = subscriptions.committed(tp).offset(); + log.debug("Resetting offset for partition {} to the committed offset {}", tp, committed); + subscriptions.seek(tp, committed); } } } @@ -257,16 +262,45 @@ private long listOffset(TopicPartition partition, long timestamp) { } } + /** + * If any partition from previous fetchResponse contains OffsetOutOfRange error and + * the defaultResetPolicy is NONE, throw OffsetOutOfRangeException + * + * @throws OffsetOutOfRangeException If there is OffsetOutOfRange error in fetchResponse + */ + private void throwIfOffsetOutOfRange() throws OffsetOutOfRangeException { + Map currentOutOfRangePartitions = new HashMap<>(); + + // filter offsetOutOfRangePartitions to retain only the fetchable partitions + for (Map.Entry entry: this.offsetOutOfRangePartitions.entrySet()) { + if (!subscriptions.isFetchable(entry.getKey())) { + log.debug("Ignoring fetched records for {} since it is no longer fetchable", entry.getKey()); + continue; + } + Long consumed = subscriptions.consumed(entry.getKey()); + // ignore partition if its consumed offset != offset in fetchResponse, e.g. after seek() + if (consumed != null && entry.getValue().equals(consumed)) + currentOutOfRangePartitions.put(entry.getKey(), entry.getValue()); + } + this.offsetOutOfRangePartitions.clear(); + if (!currentOutOfRangePartitions.isEmpty()) + throw new OffsetOutOfRangeException(currentOutOfRangePartitions); + } + /** * Return the fetched records, empty the record buffer and update the consumed position. * * @return The fetched records per partition + * @throws OffsetOutOfRangeException If there is OffsetOutOfRange error in fetchResponse and + * the defaultResetPolicy is NONE */ public Map>> fetchedRecords() { if (this.subscriptions.partitionAssignmentNeeded()) { return Collections.emptyMap(); } else { Map>> drained = new HashMap<>(); + throwIfOffsetOutOfRange(); + for (PartitionRecords part : this.records) { if (!subscriptions.isFetchable(part.partition)) { log.debug("Ignoring fetched records for {} since it is no longer fetchable", part.partition); @@ -378,8 +412,11 @@ private Map createFetchRequests(Cluster cluster) { fetchable.put(node, fetch); } - long offset = this.subscriptions.fetched(partition); - fetch.put(partition, new FetchRequest.PartitionData(offset, this.fetchSize)); + long fetched = this.subscriptions.fetched(partition); + long consumed = this.subscriptions.consumed(partition); + // Only fetch data for partitions whose previously fetched data has been consumed + if (consumed == fetched) + fetch.put(partition, new FetchRequest.PartitionData(fetched, this.fetchSize)); } } @@ -447,9 +484,12 @@ private void handleFetchResponse(ClientResponse resp, FetchRequest request) { || partition.errorCode == Errors.UNKNOWN_TOPIC_OR_PARTITION.code()) { this.metadata.requestUpdate(); } else if (partition.errorCode == Errors.OFFSET_OUT_OF_RANGE.code()) { - // TODO: this could be optimized by grouping all out-of-range partitions + long fetchOffset = request.fetchData().get(tp).offset; + if (subscriptions.hasDefaultOffsetResetPolicy()) + subscriptions.needOffsetReset(tp); + else + this.offsetOutOfRangePartitions.put(tp, fetchOffset); log.info("Fetch offset {} is out of range, resetting offset", subscriptions.fetched(tp)); - subscriptions.needOffsetReset(tp); } else if (partition.errorCode == Errors.UNKNOWN.code()) { log.warn("Unknown error fetching data for topic-partition {}", tp); } else { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestFuture.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestFuture.java index 5f0025144a6af..f5c1afcba49db 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestFuture.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestFuture.java @@ -100,6 +100,8 @@ public RuntimeException exception() { * @param value corresponding value (or null if there is none) */ public void complete(T value) { + if (isDone) + throw new IllegalStateException("Invalid attempt to complete a request future which is already complete"); this.value = value; this.isDone = true; fireSuccess(); @@ -111,6 +113,8 @@ public void complete(T value) { * @param e corresponding exception to be passed to caller */ public void raise(RuntimeException e) { + if (isDone) + throw new IllegalStateException("Invalid attempt to complete a request future which is already complete"); this.exception = e; this.isDone = true; fireFailure(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java index f976df4905069..25a0e9071f5d6 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java @@ -13,9 +13,11 @@ package org.apache.kafka.clients.consumer.internals; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.common.TopicPartition; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -41,7 +43,7 @@ * assignment is changed whether directly by the user or through a group rebalance. * * This class also maintains a cache of the latest commit position for each of the assigned - * partitions. This is updated through {@link #committed(TopicPartition, long)} and can be used + * partitions. This is updated through {@link #committed(TopicPartition, OffsetAndMetadata)} and can be used * to set the initial fetch position (e.g. {@link Fetcher#resetOffset(TopicPartition)}. */ public class SubscriptionState { @@ -176,11 +178,11 @@ private TopicPartitionState assignedState(TopicPartition tp) { return state; } - public void committed(TopicPartition tp, long offset) { + public void committed(TopicPartition tp, OffsetAndMetadata offset) { assignedState(tp).committed(offset); } - public Long committed(TopicPartition tp) { + public OffsetAndMetadata committed(TopicPartition tp) { return assignedState(tp).committed; } @@ -225,12 +227,12 @@ public Long consumed(TopicPartition tp) { return assignedState(tp).consumed; } - public Map allConsumed() { - Map allConsumed = new HashMap<>(); + public Map allConsumed() { + Map allConsumed = new HashMap<>(); for (Map.Entry entry : assignment.entrySet()) { TopicPartitionState state = entry.getValue(); if (state.hasValidPosition) - allConsumed.put(entry.getKey(), state.consumed); + allConsumed.put(entry.getKey(), new OffsetAndMetadata(state.consumed)); } return allConsumed; } @@ -243,6 +245,10 @@ public void needOffsetReset(TopicPartition partition) { needOffsetReset(partition, defaultResetStrategy); } + public boolean hasDefaultOffsetResetPolicy() { + return defaultResetStrategy != OffsetResetStrategy.NONE; + } + public boolean isOffsetResetNeeded(TopicPartition partition) { return assignedState(partition).awaitingReset; } @@ -259,7 +265,7 @@ public boolean hasAllFetchPositions() { } public Set missingFetchPositions() { - Set missing = new HashSet<>(this.assignment.keySet()); + Set missing = new HashSet<>(); for (Map.Entry entry : assignment.entrySet()) if (!entry.getValue().hasValidPosition) missing.add(entry.getKey()); @@ -270,7 +276,7 @@ public boolean partitionAssignmentNeeded() { return this.needsPartitionAssignment; } - public void changePartitionAssignment(List assignments) { + public void changePartitionAssignment(Collection assignments) { for (TopicPartition tp : assignments) if (!this.subscription.contains(tp.topic())) throw new IllegalArgumentException("Assigned partition " + tp + " for non-subscribed topic."); @@ -311,7 +317,7 @@ public ConsumerRebalanceListener listener() { private static class TopicPartitionState { private Long consumed; // offset exposed to the user private Long fetched; // current fetch position - private Long committed; // last committed position + private OffsetAndMetadata committed; // last committed position private boolean hasValidPosition; // whether we have valid consumed and fetched positions private boolean paused; // whether this partition has been paused by the user @@ -356,7 +362,7 @@ private void consumed(long offset) { this.consumed = offset; } - private void committed(Long offset) { + private void committed(OffsetAndMetadata offset) { this.committed = offset; } diff --git a/clients/src/main/java/org/apache/kafka/common/errors/IllegalGenerationException.java b/clients/src/main/java/org/apache/kafka/common/errors/IllegalGenerationException.java index d20b74a46fecf..fe8ba7a4bd38b 100644 --- a/clients/src/main/java/org/apache/kafka/common/errors/IllegalGenerationException.java +++ b/clients/src/main/java/org/apache/kafka/common/errors/IllegalGenerationException.java @@ -12,7 +12,7 @@ */ package org.apache.kafka.common.errors; -public class IllegalGenerationException extends RetriableException { +public class IllegalGenerationException extends ApiException { private static final long serialVersionUID = 1L; public IllegalGenerationException() { diff --git a/clients/src/main/java/org/apache/kafka/common/errors/OffsetOutOfRangeException.java b/clients/src/main/java/org/apache/kafka/common/errors/OffsetOutOfRangeException.java index fc7c6e3471b05..4983bc0002ce0 100644 --- a/clients/src/main/java/org/apache/kafka/common/errors/OffsetOutOfRangeException.java +++ b/clients/src/main/java/org/apache/kafka/common/errors/OffsetOutOfRangeException.java @@ -12,6 +12,9 @@ */ package org.apache.kafka.common.errors; +import org.apache.kafka.common.TopicPartition; +import java.util.Map; + /** * This offset is either larger or smaller than the range of offsets the server has for the given partition. * @@ -19,10 +22,15 @@ public class OffsetOutOfRangeException extends RetriableException { private static final long serialVersionUID = 1L; + private Map offsetOutOfRangePartitions = null; public OffsetOutOfRangeException() { } + public OffsetOutOfRangeException(Map offsetOutOfRangePartitions) { + this.offsetOutOfRangePartitions = offsetOutOfRangePartitions; + } + public OffsetOutOfRangeException(String message) { super(message); } @@ -35,4 +43,8 @@ public OffsetOutOfRangeException(String message, Throwable cause) { super(message, cause); } + public Map offsetOutOfRangePartitions() { + return offsetOutOfRangePartitions; + } + } diff --git a/clients/src/main/java/org/apache/kafka/common/errors/UnknownConsumerIdException.java b/clients/src/main/java/org/apache/kafka/common/errors/UnknownConsumerIdException.java index 9bcbd114e74e8..28bfd72fad8c5 100644 --- a/clients/src/main/java/org/apache/kafka/common/errors/UnknownConsumerIdException.java +++ b/clients/src/main/java/org/apache/kafka/common/errors/UnknownConsumerIdException.java @@ -12,7 +12,7 @@ */ package org.apache.kafka.common.errors; -public class UnknownConsumerIdException extends RetriableException { +public class UnknownConsumerIdException extends ApiException { private static final long serialVersionUID = 1L; public UnknownConsumerIdException() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java index d6e63861d02be..03df1e780c071 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java @@ -12,12 +12,6 @@ */ package org.apache.kafka.common.requests; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; @@ -26,6 +20,12 @@ import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.CollectionUtils; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + /** * This wrapper supports both v0 and v1 of OffsetCommitRequest. */ diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/MockConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/MockConsumerTest.java index 24e3d81c8dc41..fa06be9e2ba2a 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/MockConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/MockConsumerTest.java @@ -21,6 +21,7 @@ import org.junit.Test; import java.util.Arrays; +import java.util.HashMap; import java.util.Iterator; import static org.junit.Assert.assertEquals; @@ -34,6 +35,13 @@ public class MockConsumerTest { public void testSimpleMock() { consumer.subscribe(Arrays.asList("test"), new NoOpConsumerRebalanceListener()); assertEquals(0, consumer.poll(1000).count()); + consumer.rebalance(Arrays.asList(new TopicPartition("test", 0), new TopicPartition("test", 1))); + // Mock consumers need to seek manually since they cannot automatically reset offsets + HashMap beginningOffsets = new HashMap<>(); + beginningOffsets.put(new TopicPartition("test", 0), 0L); + beginningOffsets.put(new TopicPartition("test", 1), 0L); + consumer.updateBeginningOffsets(beginningOffsets); + consumer.seek(new TopicPartition("test", 0), 0); ConsumerRecord rec1 = new ConsumerRecord("test", 0, 0, "key1", "value1"); ConsumerRecord rec2 = new ConsumerRecord("test", 0, 1, "key2", "value2"); consumer.addRecord(rec1); @@ -43,9 +51,9 @@ public void testSimpleMock() { assertEquals(rec1, iter.next()); assertEquals(rec2, iter.next()); assertFalse(iter.hasNext()); - assertEquals(1L, consumer.position(new TopicPartition("test", 0))); + assertEquals(2L, consumer.position(new TopicPartition("test", 0))); consumer.commitSync(); - assertEquals(1L, consumer.committed(new TopicPartition("test", 0))); + assertEquals(2L, consumer.committed(new TopicPartition("test", 0)).offset()); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorTest.java index 490578e248711..82586ac8d9e2a 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorTest.java @@ -24,6 +24,7 @@ import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.MockClient; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.common.Cluster; @@ -31,6 +32,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.ApiException; import org.apache.kafka.common.errors.DisconnectException; +import org.apache.kafka.common.errors.OffsetMetadataTooLarge; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -302,16 +304,38 @@ public void testInvalidSessionTimeout() { } @Test - public void testCommitOffsetNormal() { + public void testCommitOffsetOnly() { + subscriptions.assign(Arrays.asList(tp)); + + client.prepareResponse(consumerMetadataResponse(node, Errors.NONE.code())); + coordinator.ensureCoordinatorKnown(); + + client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.NONE.code()))); + + AtomicBoolean success = new AtomicBoolean(false); + coordinator.commitOffsetsAsync(Collections.singletonMap(tp, new OffsetAndMetadata(100L)), callback(success)); + consumerClient.poll(0); + assertTrue(success.get()); + + assertEquals(100L, subscriptions.committed(tp).offset()); + } + + @Test + public void testCommitOffsetMetadata() { + subscriptions.assign(Arrays.asList(tp)); + client.prepareResponse(consumerMetadataResponse(node, Errors.NONE.code())); coordinator.ensureCoordinatorKnown(); client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.NONE.code()))); AtomicBoolean success = new AtomicBoolean(false); - coordinator.commitOffsetsAsync(Collections.singletonMap(tp, 100L), callback(success)); + coordinator.commitOffsetsAsync(Collections.singletonMap(tp, new OffsetAndMetadata(100L, "hello")), callback(success)); consumerClient.poll(0); assertTrue(success.get()); + + assertEquals(100L, subscriptions.committed(tp).offset()); + assertEquals("hello", subscriptions.committed(tp).metadata()); } @Test @@ -320,7 +344,7 @@ public void testCommitOffsetAsyncWithDefaultCallback() { client.prepareResponse(consumerMetadataResponse(node, Errors.NONE.code())); coordinator.ensureCoordinatorKnown(); client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.NONE.code()))); - coordinator.commitOffsetsAsync(Collections.singletonMap(tp, 100L), null); + coordinator.commitOffsetsAsync(Collections.singletonMap(tp, new OffsetAndMetadata(100L)), null); consumerClient.poll(0); assertEquals(invokedBeforeTest + 1, defaultOffsetCommitCallback.invoked); assertNull(defaultOffsetCommitCallback.exception); @@ -332,7 +356,7 @@ public void testCommitOffsetAsyncFailedWithDefaultCallback() { client.prepareResponse(consumerMetadataResponse(node, Errors.NONE.code())); coordinator.ensureCoordinatorKnown(); client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.CONSUMER_COORDINATOR_NOT_AVAILABLE.code()))); - coordinator.commitOffsetsAsync(Collections.singletonMap(tp, 100L), null); + coordinator.commitOffsetsAsync(Collections.singletonMap(tp, new OffsetAndMetadata(100L)), null); consumerClient.poll(0); assertEquals(invokedBeforeTest + 1, defaultOffsetCommitCallback.invoked); assertEquals(Errors.CONSUMER_COORDINATOR_NOT_AVAILABLE.exception(), defaultOffsetCommitCallback.exception); @@ -346,7 +370,7 @@ public void testCommitOffsetAsyncCoordinatorNotAvailable() { // async commit with coordinator not available MockCommitCallback cb = new MockCommitCallback(); client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.CONSUMER_COORDINATOR_NOT_AVAILABLE.code()))); - coordinator.commitOffsetsAsync(Collections.singletonMap(tp, 100L), cb); + coordinator.commitOffsetsAsync(Collections.singletonMap(tp, new OffsetAndMetadata(100L)), cb); consumerClient.poll(0); assertTrue(coordinator.coordinatorUnknown()); @@ -362,7 +386,7 @@ public void testCommitOffsetAsyncNotCoordinator() { // async commit with not coordinator MockCommitCallback cb = new MockCommitCallback(); client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.NOT_COORDINATOR_FOR_CONSUMER.code()))); - coordinator.commitOffsetsAsync(Collections.singletonMap(tp, 100L), cb); + coordinator.commitOffsetsAsync(Collections.singletonMap(tp, new OffsetAndMetadata(100L)), cb); consumerClient.poll(0); assertTrue(coordinator.coordinatorUnknown()); @@ -378,7 +402,7 @@ public void testCommitOffsetAsyncDisconnected() { // async commit with coordinator disconnected MockCommitCallback cb = new MockCommitCallback(); client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.NONE.code())), true); - coordinator.commitOffsetsAsync(Collections.singletonMap(tp, 100L), cb); + coordinator.commitOffsetsAsync(Collections.singletonMap(tp, new OffsetAndMetadata(100L)), cb); consumerClient.poll(0); assertTrue(coordinator.coordinatorUnknown()); @@ -395,7 +419,7 @@ public void testCommitOffsetSyncNotCoordinator() { client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.NOT_COORDINATOR_FOR_CONSUMER.code()))); client.prepareResponse(consumerMetadataResponse(node, Errors.NONE.code())); client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.NONE.code()))); - coordinator.commitOffsetsSync(Collections.singletonMap(tp, 100L)); + coordinator.commitOffsetsSync(Collections.singletonMap(tp, new OffsetAndMetadata(100L))); } @Test @@ -407,7 +431,7 @@ public void testCommitOffsetSyncCoordinatorNotAvailable() { client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.CONSUMER_COORDINATOR_NOT_AVAILABLE.code()))); client.prepareResponse(consumerMetadataResponse(node, Errors.NONE.code())); client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.NONE.code()))); - coordinator.commitOffsetsSync(Collections.singletonMap(tp, 100L)); + coordinator.commitOffsetsSync(Collections.singletonMap(tp, new OffsetAndMetadata(100L))); } @Test @@ -419,7 +443,17 @@ public void testCommitOffsetSyncCoordinatorDisconnected() { client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.NONE.code())), true); client.prepareResponse(consumerMetadataResponse(node, Errors.NONE.code())); client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.NONE.code()))); - coordinator.commitOffsetsSync(Collections.singletonMap(tp, 100L)); + coordinator.commitOffsetsSync(Collections.singletonMap(tp, new OffsetAndMetadata(100L))); + } + + @Test(expected = OffsetMetadataTooLarge.class) + public void testCommitOffsetMetadataTooLarge() { + // since offset metadata is provided by the user, we have to propagate the exception so they can handle it + client.prepareResponse(consumerMetadataResponse(node, Errors.NONE.code())); + coordinator.ensureCoordinatorKnown(); + + client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.OFFSET_METADATA_TOO_LARGE.code()))); + coordinator.commitOffsetsSync(Collections.singletonMap(tp, new OffsetAndMetadata(100L, "metadata"))); } @Test(expected = ApiException.class) @@ -429,7 +463,7 @@ public void testCommitOffsetSyncCallbackWithNonRetriableException() { // sync commit with invalid partitions should throw if we have no callback client.prepareResponse(offsetCommitResponse(Collections.singletonMap(tp, Errors.COMMITTING_PARTITIONS_NOT_ASSIGNED.code())), false); - coordinator.commitOffsetsSync(Collections.singletonMap(tp, 100L)); + coordinator.commitOffsetsSync(Collections.singletonMap(tp, new OffsetAndMetadata(100L))); } @Test @@ -442,7 +476,7 @@ public void testRefreshOffset() { client.prepareResponse(offsetFetchResponse(tp, Errors.NONE.code(), "", 100L)); coordinator.refreshCommittedOffsetsIfNeeded(); assertFalse(subscriptions.refreshCommitsNeeded()); - assertEquals(100L, (long) subscriptions.committed(tp)); + assertEquals(100L, subscriptions.committed(tp).offset()); } @Test @@ -456,7 +490,7 @@ public void testRefreshOffsetLoadInProgress() { client.prepareResponse(offsetFetchResponse(tp, Errors.NONE.code(), "", 100L)); coordinator.refreshCommittedOffsetsIfNeeded(); assertFalse(subscriptions.refreshCommitsNeeded()); - assertEquals(100L, (long) subscriptions.committed(tp)); + assertEquals(100L, subscriptions.committed(tp).offset()); } @Test @@ -471,7 +505,7 @@ public void testRefreshOffsetNotCoordinatorForConsumer() { client.prepareResponse(offsetFetchResponse(tp, Errors.NONE.code(), "", 100L)); coordinator.refreshCommittedOffsetsIfNeeded(); assertFalse(subscriptions.refreshCommitsNeeded()); - assertEquals(100L, (long) subscriptions.committed(tp)); + assertEquals(100L, subscriptions.committed(tp).offset()); } @Test @@ -516,7 +550,7 @@ private Struct offsetFetchResponse(TopicPartition tp, Short error, String metada private OffsetCommitCallback callback(final AtomicBoolean success) { return new OffsetCommitCallback() { @Override - public void onComplete(Map offsets, Exception exception) { + public void onComplete(Map offsets, Exception exception) { if (exception == null) success.set(true); } @@ -528,7 +562,7 @@ private static class MockCommitCallback implements OffsetCommitCallback { public Exception exception = null; @Override - public void onComplete(Map offsets, Exception exception) { + public void onComplete(Map offsets, Exception exception) { invoked++; this.exception = exception; } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index a4f6c19496c2f..b3169d87bbde0 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -21,12 +21,14 @@ import org.apache.kafka.clients.MockClient; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.Node; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.OffsetOutOfRangeException; import org.apache.kafka.common.metrics.KafkaMetric; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.Errors; @@ -55,6 +57,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class FetcherTest { private ConsumerRebalanceListener listener = new NoOpConsumerRebalanceListener(); @@ -72,27 +75,15 @@ public class FetcherTest { private Cluster cluster = TestUtils.singletonCluster(topicName, 1); private Node node = cluster.nodes().get(0); private SubscriptionState subscriptions = new SubscriptionState(OffsetResetStrategy.EARLIEST); + private SubscriptionState subscriptionsNoAutoReset = new SubscriptionState(OffsetResetStrategy.NONE); private Metrics metrics = new Metrics(time); private Map metricTags = new LinkedHashMap(); private static final double EPSILON = 0.0001; private ConsumerNetworkClient consumerClient = new ConsumerNetworkClient(client, metadata, time, 100); private MemoryRecords records = MemoryRecords.emptyRecords(ByteBuffer.allocate(1024), CompressionType.NONE); - - private Fetcher fetcher = new Fetcher(consumerClient, - minBytes, - maxWaitMs, - fetchSize, - true, // check crc - new ByteArrayDeserializer(), - new ByteArrayDeserializer(), - metadata, - subscriptions, - metrics, - "consumer" + groupId, - metricTags, - time, - retryBackoffMs); + private Fetcher fetcher = createFetcher(subscriptions, metrics); + private Fetcher fetcherNoAutoReset = createFetcher(subscriptionsNoAutoReset, new Metrics(time)); @Before public void setup() throws Exception { @@ -205,6 +196,38 @@ public void testFetchOffsetOutOfRange() { assertEquals(null, subscriptions.consumed(tp)); } + @Test + public void testFetchedRecordsAfterSeek() { + subscriptionsNoAutoReset.assign(Arrays.asList(tp)); + subscriptionsNoAutoReset.seek(tp, 0); + + fetcherNoAutoReset.initFetches(cluster); + client.prepareResponse(fetchResponse(this.records.buffer(), Errors.OFFSET_OUT_OF_RANGE.code(), 100L, 0)); + consumerClient.poll(0); + assertFalse(subscriptionsNoAutoReset.isOffsetResetNeeded(tp)); + subscriptionsNoAutoReset.seek(tp, 2); + assertEquals(0, fetcherNoAutoReset.fetchedRecords().size()); + } + + @Test + public void testFetchOffsetOutOfRangeException() { + subscriptionsNoAutoReset.assign(Arrays.asList(tp)); + subscriptionsNoAutoReset.seek(tp, 0); + + fetcherNoAutoReset.initFetches(cluster); + client.prepareResponse(fetchResponse(this.records.buffer(), Errors.OFFSET_OUT_OF_RANGE.code(), 100L, 0)); + consumerClient.poll(0); + assertFalse(subscriptionsNoAutoReset.isOffsetResetNeeded(tp)); + try { + fetcherNoAutoReset.fetchedRecords(); + fail("Should have thrown OffsetOutOfRangeException"); + } catch (OffsetOutOfRangeException e) { + assertTrue(e.offsetOutOfRangePartitions().containsKey(tp)); + assertEquals(e.offsetOutOfRangePartitions().size(), 1); + } + assertEquals(0, fetcherNoAutoReset.fetchedRecords().size()); + } + @Test public void testFetchDisconnected() { subscriptions.assign(Arrays.asList(tp)); @@ -227,7 +250,7 @@ public void testUpdateFetchPositionToCommitted() { // unless a specific reset is expected, the default behavior is to reset to the committed // position if one is present subscriptions.assign(Arrays.asList(tp)); - subscriptions.committed(tp, 5); + subscriptions.committed(tp, new OffsetAndMetadata(5)); fetcher.updateFetchPositions(Collections.singleton(tp)); assertTrue(subscriptions.isFetchable(tp)); @@ -357,4 +380,21 @@ private Struct fetchResponse(ByteBuffer buffer, short error, long hw, int thrott FetchResponse response = new FetchResponse(Collections.singletonMap(tp, new FetchResponse.PartitionData(error, hw, buffer)), throttleTime); return response.toStruct(); } + + private Fetcher createFetcher(SubscriptionState subscriptions, Metrics metrics) { + return new Fetcher(consumerClient, + minBytes, + maxWaitMs, + fetchSize, + true, // check crc + new ByteArrayDeserializer(), + new ByteArrayDeserializer(), + metadata, + subscriptions, + metrics, + "consumer" + groupId, + metricTags, + time, + retryBackoffMs); + } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java index a279faa00b522..a0568add53cd6 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java @@ -27,6 +27,7 @@ import java.util.regex.Pattern; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.common.TopicPartition; import org.junit.Test; @@ -42,7 +43,7 @@ public class SubscriptionStateTest { public void partitionAssignment() { state.assign(Arrays.asList(tp0)); assertEquals(Collections.singleton(tp0), state.assignedPartitions()); - state.committed(tp0, 1); + state.committed(tp0, new OffsetAndMetadata(1)); state.seek(tp0, 1); assertTrue(state.isFetchable(tp0)); assertAllPositions(tp0, 1L); @@ -78,7 +79,7 @@ public void topicSubscription() { assertTrue(state.partitionsAutoAssigned()); state.changePartitionAssignment(asList(tp0)); state.seek(tp0, 1); - state.committed(tp0, 1); + state.committed(tp0, new OffsetAndMetadata(1)); assertAllPositions(tp0, 1L); state.changePartitionAssignment(asList(tp1)); assertTrue(state.isAssigned(tp1)); @@ -98,6 +99,15 @@ public void partitionPause() { assertTrue(state.isFetchable(tp0)); } + @Test + public void commitOffsetMetadata() { + state.assign(Arrays.asList(tp0)); + state.committed(tp0, new OffsetAndMetadata(5, "hi")); + + assertEquals(5, state.committed(tp0).offset()); + assertEquals("hi", state.committed(tp0).metadata()); + } + @Test public void topicUnsubscription() { final String topic = "test"; @@ -106,7 +116,7 @@ public void topicUnsubscription() { assertTrue(state.assignedPartitions().isEmpty()); assertTrue(state.partitionsAutoAssigned()); state.changePartitionAssignment(asList(tp0)); - state.committed(tp0, 1); + state.committed(tp0, new OffsetAndMetadata(1)); state.seek(tp0, 1); assertAllPositions(tp0, 1L); state.changePartitionAssignment(asList(tp1)); @@ -143,7 +153,7 @@ public void cantChangeConsumedPositionForNonAssignedPartition() { } public void assertAllPositions(TopicPartition tp, Long offset) { - assertEquals(offset, state.committed(tp)); + assertEquals(offset.longValue(), state.committed(tp).offset()); assertEquals(offset, state.fetched(tp)); assertEquals(offset, state.consumed(tp)); } diff --git a/config/copycat-distributed.properties b/config/copycat-distributed.properties new file mode 100644 index 0000000000000..654ed24dad2ef --- /dev/null +++ b/config/copycat-distributed.properties @@ -0,0 +1,39 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +# These are defaults. This file just demonstrates how to override some settings. +bootstrap.servers=localhost:9092 + +# The converters specify the format of data in Kafka and how to translate it into Copycat data. Every Copycat user will +# need to configure these based on the format they want their data in when loaded from or stored into Kafka +key.converter=org.apache.kafka.copycat.json.JsonConverter +value.converter=org.apache.kafka.copycat.json.JsonConverter +# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply +# it to +key.converter.schemas.enable=true +value.converter.schemas.enable=true + +# The offset converter is configurable and must be specified, but most users will always want to use the built-in default. +# Offset data is never visible outside of Copcyat. +offset.key.converter=org.apache.kafka.copycat.json.JsonConverter +offset.value.converter=org.apache.kafka.copycat.json.JsonConverter +offset.key.converter.schemas.enable=false +offset.value.converter.schemas.enable=false + +offset.storage.topic=copycat-offsets +# Flush much faster than normal, which is useful for testing/debugging +offset.flush.interval.ms=10000 diff --git a/copycat/api/src/main/java/org/apache/kafka/copycat/sink/SinkTask.java b/copycat/api/src/main/java/org/apache/kafka/copycat/sink/SinkTask.java index 49fbbe937ee78..bf5d152100d04 100644 --- a/copycat/api/src/main/java/org/apache/kafka/copycat/sink/SinkTask.java +++ b/copycat/api/src/main/java/org/apache/kafka/copycat/sink/SinkTask.java @@ -16,6 +16,7 @@ **/ package org.apache.kafka.copycat.sink; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.annotation.InterfaceStability; import org.apache.kafka.copycat.connector.Task; @@ -60,5 +61,5 @@ public void initialize(SinkTaskContext context) { * * @param offsets mapping of TopicPartition to committed offset */ - public abstract void flush(Map offsets); + public abstract void flush(Map offsets); } diff --git a/copycat/file/src/main/java/org/apache/kafka/copycat/file/FileStreamSinkTask.java b/copycat/file/src/main/java/org/apache/kafka/copycat/file/FileStreamSinkTask.java index 870a95222742a..9ea459c728013 100644 --- a/copycat/file/src/main/java/org/apache/kafka/copycat/file/FileStreamSinkTask.java +++ b/copycat/file/src/main/java/org/apache/kafka/copycat/file/FileStreamSinkTask.java @@ -17,6 +17,7 @@ package org.apache.kafka.copycat.file; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.copycat.errors.CopycatException; import org.apache.kafka.copycat.sink.SinkRecord; @@ -69,7 +70,7 @@ public void put(Collection sinkRecords) { } @Override - public void flush(Map offsets) { + public void flush(Map offsets) { outputStream.flush(); } diff --git a/copycat/file/src/test/java/org/apache/kafka/copycat/file/FileStreamSinkTaskTest.java b/copycat/file/src/test/java/org/apache/kafka/copycat/file/FileStreamSinkTaskTest.java index 1dfb5d88d5133..ac8b5f19d2018 100644 --- a/copycat/file/src/test/java/org/apache/kafka/copycat/file/FileStreamSinkTaskTest.java +++ b/copycat/file/src/test/java/org/apache/kafka/copycat/file/FileStreamSinkTaskTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.copycat.file; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.copycat.data.Schema; import org.apache.kafka.copycat.sink.SinkRecord; @@ -45,14 +46,14 @@ public void setup() { @Test public void testPutFlush() { - HashMap offsets = new HashMap<>(); + HashMap offsets = new HashMap<>(); // We do not call task.start() since it would override the output stream task.put(Arrays.asList( new SinkRecord("topic1", 0, null, null, Schema.STRING_SCHEMA, "line1", 1) )); - offsets.put(new TopicPartition("topic1", 0), 1L); + offsets.put(new TopicPartition("topic1", 0), new OffsetAndMetadata(1L)); task.flush(offsets); assertEquals("line1\n", os.toString()); @@ -60,8 +61,8 @@ public void testPutFlush() { new SinkRecord("topic1", 0, null, null, Schema.STRING_SCHEMA, "line2", 2), new SinkRecord("topic2", 0, null, null, Schema.STRING_SCHEMA, "line3", 1) )); - offsets.put(new TopicPartition("topic1", 0), 2L); - offsets.put(new TopicPartition("topic2", 0), 1L); + offsets.put(new TopicPartition("topic1", 0), new OffsetAndMetadata(2L)); + offsets.put(new TopicPartition("topic2", 0), new OffsetAndMetadata(1L)); task.flush(offsets); assertEquals("line1\nline2\nline3\n", os.toString()); } diff --git a/copycat/runtime/src/main/java/org/apache/kafka/copycat/cli/CopycatDistributed.java b/copycat/runtime/src/main/java/org/apache/kafka/copycat/cli/CopycatDistributed.java new file mode 100644 index 0000000000000..b5e88964b559e --- /dev/null +++ b/copycat/runtime/src/main/java/org/apache/kafka/copycat/cli/CopycatDistributed.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +package org.apache.kafka.copycat.cli; + +import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.copycat.runtime.Copycat; +import org.apache.kafka.copycat.runtime.Herder; +import org.apache.kafka.copycat.runtime.Worker; +import org.apache.kafka.copycat.runtime.standalone.StandaloneHerder; +import org.apache.kafka.copycat.storage.KafkaOffsetBackingStore; +import org.apache.kafka.copycat.util.Callback; +import org.apache.kafka.copycat.util.FutureCallback; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Properties; + +/** + *

+ * Command line utility that runs Copycat in distributed mode. In this mode, the process joints a group of other workers + * and work is distributed among them. This is useful for running Copycat as a service, where connectors can be + * submitted to the cluster to be automatically executed in a scalable, distributed fashion. This also allows you to + * easily scale out horizontally, elastically adding or removing capacity simply by starting or stopping worker + * instances. + *

+ */ +@InterfaceStability.Unstable +public class CopycatDistributed { + private static final Logger log = LoggerFactory.getLogger(CopycatDistributed.class); + + public static void main(String[] args) throws Exception { + Properties workerProps; + Properties connectorProps; + + if (args.length < 2) { + log.info("Usage: CopycatDistributed worker.properties connector1.properties [connector2.properties ...]"); + System.exit(1); + } + + String workerPropsFile = args[0]; + workerProps = !workerPropsFile.isEmpty() ? Utils.loadProps(workerPropsFile) : new Properties(); + + WorkerConfig workerConfig = new WorkerConfig(workerProps); + Worker worker = new Worker(workerConfig, new KafkaOffsetBackingStore()); + Herder herder = new StandaloneHerder(worker); + final Copycat copycat = new Copycat(worker, herder); + copycat.start(); + + try { + for (final String connectorPropsFile : Arrays.copyOfRange(args, 1, args.length)) { + connectorProps = Utils.loadProps(connectorPropsFile); + FutureCallback cb = new FutureCallback<>(new Callback() { + @Override + public void onCompletion(Throwable error, String id) { + if (error != null) + log.error("Failed to create job for {}", connectorPropsFile); + } + }); + herder.addConnector(connectorProps, cb); + cb.get(); + } + } catch (Throwable t) { + log.error("Stopping after connector error", t); + copycat.stop(); + } + + // Shutdown will be triggered by Ctrl-C or via HTTP shutdown request + copycat.awaitStop(); + } +} diff --git a/copycat/runtime/src/main/java/org/apache/kafka/copycat/cli/CopycatStandalone.java b/copycat/runtime/src/main/java/org/apache/kafka/copycat/cli/CopycatStandalone.java index 130a5294c3e19..12ec154b6a8a6 100644 --- a/copycat/runtime/src/main/java/org/apache/kafka/copycat/cli/CopycatStandalone.java +++ b/copycat/runtime/src/main/java/org/apache/kafka/copycat/cli/CopycatStandalone.java @@ -19,9 +19,11 @@ import org.apache.kafka.common.annotation.InterfaceStability; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.copycat.runtime.Copycat; import org.apache.kafka.copycat.runtime.Herder; import org.apache.kafka.copycat.runtime.Worker; import org.apache.kafka.copycat.runtime.standalone.StandaloneHerder; +import org.apache.kafka.copycat.storage.FileOffsetBackingStore; import org.apache.kafka.copycat.util.Callback; import org.apache.kafka.copycat.util.FutureCallback; import org.slf4j.Logger; @@ -58,9 +60,9 @@ public static void main(String[] args) throws Exception { workerProps = !workerPropsFile.isEmpty() ? Utils.loadProps(workerPropsFile) : new Properties(); WorkerConfig workerConfig = new WorkerConfig(workerProps); - Worker worker = new Worker(workerConfig); + Worker worker = new Worker(workerConfig, new FileOffsetBackingStore()); Herder herder = new StandaloneHerder(worker); - final org.apache.kafka.copycat.runtime.Copycat copycat = new org.apache.kafka.copycat.runtime.Copycat(worker, herder); + final Copycat copycat = new Copycat(worker, herder); copycat.start(); try { diff --git a/copycat/runtime/src/main/java/org/apache/kafka/copycat/runtime/Worker.java b/copycat/runtime/src/main/java/org/apache/kafka/copycat/runtime/Worker.java index 6cbce0b251e3e..a34a014ce93d1 100644 --- a/copycat/runtime/src/main/java/org/apache/kafka/copycat/runtime/Worker.java +++ b/copycat/runtime/src/main/java/org/apache/kafka/copycat/runtime/Worker.java @@ -59,8 +59,8 @@ public class Worker { private KafkaProducer producer; private SourceTaskOffsetCommitter sourceTaskOffsetCommitter; - public Worker(WorkerConfig config) { - this(new SystemTime(), config, null); + public Worker(WorkerConfig config, OffsetBackingStore offsetBackingStore) { + this(new SystemTime(), config, offsetBackingStore); } @SuppressWarnings("unchecked") @@ -76,12 +76,8 @@ public Worker(Time time, WorkerConfig config, OffsetBackingStore offsetBackingSt this.offsetValueConverter = config.getConfiguredInstance(WorkerConfig.OFFSET_VALUE_CONVERTER_CLASS_CONFIG, Converter.class); this.offsetValueConverter.configure(config.originalsWithPrefix("offset.value.converter."), false); - if (offsetBackingStore != null) { - this.offsetBackingStore = offsetBackingStore; - } else { - this.offsetBackingStore = new FileOffsetBackingStore(); - this.offsetBackingStore.configure(config.originals()); - } + this.offsetBackingStore = offsetBackingStore; + this.offsetBackingStore.configure(config.originals()); } public void start() { @@ -132,7 +128,7 @@ public void stop() { long timeoutMs = limit - time.milliseconds(); sourceTaskOffsetCommitter.close(timeoutMs); - offsetBackingStore.start(); + offsetBackingStore.stop(); log.info("Worker stopped"); } diff --git a/copycat/runtime/src/main/java/org/apache/kafka/copycat/runtime/WorkerSinkTask.java b/copycat/runtime/src/main/java/org/apache/kafka/copycat/runtime/WorkerSinkTask.java index 1047213c90def..cbda20121d37e 100644 --- a/copycat/runtime/src/main/java/org/apache/kafka/copycat/runtime/WorkerSinkTask.java +++ b/copycat/runtime/src/main/java/org/apache/kafka/copycat/runtime/WorkerSinkTask.java @@ -122,9 +122,9 @@ public void poll(long timeoutMs) { **/ public void commitOffsets(long now, boolean sync, final int seqno, boolean flush) { log.info("{} Committing offsets", this); - HashMap offsets = new HashMap<>(); + HashMap offsets = new HashMap<>(); for (TopicPartition tp : consumer.assignment()) { - offsets.put(tp, consumer.position(tp)); + offsets.put(tp, new OffsetAndMetadata(consumer.position(tp))); } // We only don't flush the task in one case: when shutting down, the task has already been // stopped and all data should have already been flushed @@ -147,7 +147,7 @@ public void commitOffsets(long now, boolean sync, final int seqno, boolean flush } else { OffsetCommitCallback cb = new OffsetCommitCallback() { @Override - public void onComplete(Map offsets, Exception error) { + public void onComplete(Map offsets, Exception error) { workThread.onCommitCompleted(error, seqno); } }; diff --git a/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/FileOffsetBackingStore.java b/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/FileOffsetBackingStore.java index dfa9e7808e09d..f707fd6f53753 100644 --- a/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/FileOffsetBackingStore.java +++ b/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/FileOffsetBackingStore.java @@ -68,17 +68,12 @@ private void load() { Object obj = is.readObject(); if (!(obj instanceof HashMap)) throw new CopycatException("Expected HashMap but found " + obj.getClass()); - HashMap> raw = (HashMap>) obj; + Map raw = (Map) obj; data = new HashMap<>(); - for (Map.Entry> entry : raw.entrySet()) { - HashMap converted = new HashMap<>(); - for (Map.Entry mapEntry : entry.getValue().entrySet()) { - ByteBuffer key = (mapEntry.getKey() != null) ? ByteBuffer.wrap(mapEntry.getKey()) : null; - ByteBuffer value = (mapEntry.getValue() != null) ? ByteBuffer.wrap(mapEntry.getValue()) : - null; - converted.put(key, value); - } - data.put(entry.getKey(), converted); + for (Map.Entry mapEntry : raw.entrySet()) { + ByteBuffer key = (mapEntry.getKey() != null) ? ByteBuffer.wrap(mapEntry.getKey()) : null; + ByteBuffer value = (mapEntry.getValue() != null) ? ByteBuffer.wrap(mapEntry.getValue()) : null; + data.put(key, value); } is.close(); } catch (FileNotFoundException | EOFException e) { @@ -92,15 +87,11 @@ private void load() { protected void save() { try { ObjectOutputStream os = new ObjectOutputStream(new FileOutputStream(file)); - HashMap> raw = new HashMap<>(); - for (Map.Entry> entry : data.entrySet()) { - HashMap converted = new HashMap<>(); - for (Map.Entry mapEntry : entry.getValue().entrySet()) { - byte[] key = (mapEntry.getKey() != null) ? mapEntry.getKey().array() : null; - byte[] value = (mapEntry.getValue() != null) ? mapEntry.getValue().array() : null; - converted.put(key, value); - } - raw.put(entry.getKey(), converted); + Map raw = new HashMap<>(); + for (Map.Entry mapEntry : data.entrySet()) { + byte[] key = (mapEntry.getKey() != null) ? mapEntry.getKey().array() : null; + byte[] value = (mapEntry.getValue() != null) ? mapEntry.getValue().array() : null; + raw.put(key, value); } os.writeObject(raw); os.close(); diff --git a/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/KafkaOffsetBackingStore.java b/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/KafkaOffsetBackingStore.java new file mode 100644 index 0000000000000..a74b39cfed1e1 --- /dev/null +++ b/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/KafkaOffsetBackingStore.java @@ -0,0 +1,393 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +package org.apache.kafka.copycat.storage; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.ConsumerWakeupException; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.utils.SystemTime; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.copycat.errors.CopycatException; +import org.apache.kafka.copycat.util.Callback; +import org.apache.kafka.copycat.util.ConvertingFutureCallback; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + *

+ * Implementation of OffsetBackingStore that uses a Kafka topic to store offset data. + *

+ *

+ * Internally, this implementation both produces to and consumes from a Kafka topic which stores the offsets. + * It accepts producer and consumer overrides via its configuration but forces some settings to specific values + * to ensure correct behavior (e.g. acks, auto.offset.reset). + *

+ */ +public class KafkaOffsetBackingStore implements OffsetBackingStore { + private static final Logger log = LoggerFactory.getLogger(KafkaOffsetBackingStore.class); + + public final static String OFFSET_STORAGE_TOPIC_CONFIG = "offset.storage.topic"; + + private final static long CREATE_TOPIC_TIMEOUT_MS = 30000; + + private Time time; + private Map configs; + private String topic; + private Consumer consumer; + private Producer producer; + private HashMap data; + + private Thread thread; + private boolean stopRequested; + private Queue> readLogEndOffsetCallbacks; + + public KafkaOffsetBackingStore() { + this(new SystemTime()); + } + + public KafkaOffsetBackingStore(Time time) { + this.time = time; + } + + @Override + public void configure(Map configs) { + this.configs = configs; + topic = (String) configs.get(OFFSET_STORAGE_TOPIC_CONFIG); + if (topic == null) + throw new CopycatException("Offset storage topic must be specified"); + + data = new HashMap<>(); + stopRequested = false; + readLogEndOffsetCallbacks = new ArrayDeque<>(); + } + + @Override + public void start() { + log.info("Starting KafkaOffsetBackingStore with topic " + topic); + + producer = createProducer(); + consumer = createConsumer(); + List partitions = new ArrayList<>(); + + // Until we have admin utilities we can use to check for the existence of this topic and create it if it is missing, + // we rely on topic auto-creation + List partitionInfos = null; + long started = time.milliseconds(); + while (partitionInfos == null && time.milliseconds() - started < CREATE_TOPIC_TIMEOUT_MS) { + partitionInfos = consumer.partitionsFor(topic); + Utils.sleep(Math.min(time.milliseconds() - started, 1000)); + } + if (partitionInfos == null) + throw new CopycatException("Could not look up partition metadata for offset backing store topic in" + + " allotted period. This could indicate a connectivity issue, unavailable topic partitions, or if" + + " this is your first use of the topic it may have taken too long to create."); + + for (PartitionInfo partition : partitionInfos) + partitions.add(new TopicPartition(partition.topic(), partition.partition())); + consumer.assign(partitions); + + readToLogEnd(); + + thread = new WorkThread(); + thread.start(); + + log.info("Finished reading offsets topic and starting KafkaOffsetBackingStore"); + } + + @Override + public void stop() { + log.info("Stopping KafkaOffsetBackingStore"); + + synchronized (this) { + stopRequested = true; + consumer.wakeup(); + } + + try { + thread.join(); + } catch (InterruptedException e) { + throw new CopycatException("Failed to stop KafkaOffsetBackingStore. Exiting without cleanly shutting " + + "down it's producer and consumer.", e); + } + + try { + producer.close(); + } catch (KafkaException e) { + log.error("Failed to close KafkaOffsetBackingStore producer", e); + } + + try { + consumer.close(); + } catch (KafkaException e) { + log.error("Failed to close KafkaOffsetBackingStore consumer", e); + } + } + + @Override + public Future> get(final Collection keys, + final Callback> callback) { + ConvertingFutureCallback> future = new ConvertingFutureCallback>(callback) { + @Override + public Map convert(Void result) { + Map values = new HashMap<>(); + synchronized (KafkaOffsetBackingStore.this) { + for (ByteBuffer key : keys) + values.put(key, data.get(key)); + } + return values; + } + }; + readLogEndOffsetCallbacks.add(future); + consumer.wakeup(); + return future; + } + + @Override + public Future set(final Map values, final Callback callback) { + SetCallbackFuture producerCallback = new SetCallbackFuture(values.size(), callback); + + for (Map.Entry entry : values.entrySet()) { + producer.send(new ProducerRecord<>(topic, entry.getKey().array(), entry.getValue().array()), producerCallback); + } + + return producerCallback; + } + + + + private Producer createProducer() { + Map producerProps = new HashMap<>(); + producerProps.putAll(configs); + producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); + producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); + producerProps.put(ProducerConfig.ACKS_CONFIG, "all"); + return new KafkaProducer<>(producerProps); + } + + private Consumer createConsumer() { + Map consumerConfig = new HashMap<>(); + consumerConfig.putAll(configs); + consumerConfig.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); + consumerConfig.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + consumerConfig.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + consumerConfig.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + return new KafkaConsumer<>(consumerConfig); + } + + private void poll(long timeoutMs) { + try { + ConsumerRecords records = consumer.poll(timeoutMs); + for (ConsumerRecord record : records) { + ByteBuffer key = record.key() != null ? ByteBuffer.wrap((byte[]) record.key()) : null; + ByteBuffer value = record.value() != null ? ByteBuffer.wrap((byte[]) record.value()) : null; + data.put(key, value); + } + } catch (ConsumerWakeupException e) { + // Expected on get() or stop(). The calling code should handle this + throw e; + } catch (KafkaException e) { + log.error("Error polling: " + e); + } + } + + private void readToLogEnd() { + log.trace("Reading to end of offset log"); + + Set assignment = consumer.assignment(); + + // This approach to getting the current end offset is hacky until we have an API for looking these up directly + Map offsets = new HashMap<>(); + for (TopicPartition tp : assignment) { + long offset = consumer.position(tp); + offsets.put(tp, offset); + consumer.seekToEnd(tp); + } + + Map endOffsets = new HashMap<>(); + try { + poll(0); + } finally { + // If there is an exception, even a possibly expected one like ConsumerWakeupException, we need to make sure + // the consumers position is reset or it'll get into an inconsistent state. + for (TopicPartition tp : assignment) { + long startOffset = offsets.get(tp); + long endOffset = consumer.position(tp); + if (endOffset > startOffset) { + endOffsets.put(tp, endOffset); + consumer.seek(tp, startOffset); + } + log.trace("Reading to end of log for {}: starting offset {} to ending offset {}", tp, startOffset, endOffset); + } + } + + while (!endOffsets.isEmpty()) { + poll(Integer.MAX_VALUE); + + Iterator> it = endOffsets.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + if (consumer.position(entry.getKey()) >= entry.getValue()) + it.remove(); + else + break; + } + } + } + + + private static class SetCallbackFuture implements org.apache.kafka.clients.producer.Callback, Future { + private int numLeft; + private boolean completed = false; + private Throwable exception = null; + private final Callback callback; + + public SetCallbackFuture(int numRecords, Callback callback) { + numLeft = numRecords; + this.callback = callback; + } + + @Override + public synchronized void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + if (!completed) { + this.exception = exception; + callback.onCompletion(exception, null); + completed = true; + this.notify(); + } + return; + } + + numLeft -= 1; + if (numLeft == 0) { + callback.onCompletion(null, null); + completed = true; + this.notify(); + } + } + + @Override + public synchronized boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public synchronized boolean isCancelled() { + return false; + } + + @Override + public synchronized boolean isDone() { + return completed; + } + + @Override + public synchronized Void get() throws InterruptedException, ExecutionException { + while (!completed) { + this.wait(); + } + if (exception != null) + throw new ExecutionException(exception); + return null; + } + + @Override + public synchronized Void get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + long started = System.currentTimeMillis(); + long limit = started + unit.toMillis(timeout); + while (!completed) { + long leftMs = limit - System.currentTimeMillis(); + if (leftMs < 0) + throw new TimeoutException("KafkaOffsetBackingStore Future timed out."); + this.wait(leftMs); + } + if (exception != null) + throw new ExecutionException(exception); + return null; + } + } + + private class WorkThread extends Thread { + @Override + public void run() { + try { + while (true) { + int numCallbacks; + synchronized (KafkaOffsetBackingStore.this) { + if (stopRequested) + break; + numCallbacks = readLogEndOffsetCallbacks.size(); + } + + if (numCallbacks > 0) { + try { + readToLogEnd(); + } catch (ConsumerWakeupException e) { + // Either received another get() call and need to retry reading to end of log or stop() was + // called. Both are handled by restarting this loop. + continue; + } + } + + synchronized (KafkaOffsetBackingStore.this) { + for (int i = 0; i < numCallbacks; i++) { + Callback cb = readLogEndOffsetCallbacks.poll(); + cb.onCompletion(null, null); + } + } + + try { + poll(Integer.MAX_VALUE); + } catch (ConsumerWakeupException e) { + // See previous comment, both possible causes of this wakeup are handled by starting this loop again + continue; + } + } + } catch (Throwable t) { + log.error("Unexpected exception in KafkaOffsetBackingStore's work thread", t); + } + } + } +} diff --git a/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/MemoryOffsetBackingStore.java b/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/MemoryOffsetBackingStore.java index 6ffba58293ab5..11a1b891ffd0e 100644 --- a/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/MemoryOffsetBackingStore.java +++ b/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/MemoryOffsetBackingStore.java @@ -38,7 +38,7 @@ public class MemoryOffsetBackingStore implements OffsetBackingStore { private static final Logger log = LoggerFactory.getLogger(MemoryOffsetBackingStore.class); - protected HashMap> data = new HashMap<>(); + protected Map data = new HashMap<>(); protected ExecutorService executor = Executors.newSingleThreadExecutor(); public MemoryOffsetBackingStore() { @@ -60,18 +60,15 @@ public synchronized void stop() { @Override public Future> get( - final String namespace, final Collection keys, + final Collection keys, final Callback> callback) { return executor.submit(new Callable>() { @Override public Map call() throws Exception { Map result = new HashMap<>(); synchronized (MemoryOffsetBackingStore.this) { - Map namespaceData = data.get(namespace); - if (namespaceData == null) - return result; for (ByteBuffer key : keys) { - result.put(key, namespaceData.get(key)); + result.put(key, data.get(key)); } } if (callback != null) @@ -83,19 +80,14 @@ public Map call() throws Exception { } @Override - public Future set(final String namespace, final Map values, + public Future set(final Map values, final Callback callback) { return executor.submit(new Callable() { @Override public Void call() throws Exception { synchronized (MemoryOffsetBackingStore.this) { - Map namespaceData = data.get(namespace); - if (namespaceData == null) { - namespaceData = new HashMap<>(); - data.put(namespace, namespaceData); - } for (Map.Entry entry : values.entrySet()) { - namespaceData.put(entry.getKey(), entry.getValue()); + data.put(entry.getKey(), entry.getValue()); } save(); } diff --git a/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/OffsetBackingStore.java b/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/OffsetBackingStore.java index e8cb2aeb83742..239d9a8d73b76 100644 --- a/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/OffsetBackingStore.java +++ b/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/OffsetBackingStore.java @@ -34,8 +34,8 @@ *

*

* Since OffsetBackingStore is a shared resource that may be used by many OffsetStorage instances - * that are associated with individual tasks, all operations include a namespace which should be - * used to isolate different key spaces. + * that are associated with individual tasks, the caller must be sure keys include information about the + * connector so that the shared namespace does not result in conflicting keys. *

*/ public interface OffsetBackingStore extends Configurable { @@ -53,22 +53,20 @@ public interface OffsetBackingStore extends Configurable { /** * Get the values for the specified keys - * @param namespace prefix for the keys in this request * @param keys list of keys to look up * @param callback callback to invoke on completion * @return future for the resulting map from key to value */ public Future> get( - String namespace, Collection keys, + Collection keys, Callback> callback); /** * Set the specified keys and values. - * @param namespace prefix for the keys in this request * @param values map from key to value * @param callback callback to invoke on completion * @return void future for the operation */ - public Future set(String namespace, Map values, + public Future set(Map values, Callback callback); } diff --git a/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/OffsetStorageReaderImpl.java b/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/OffsetStorageReaderImpl.java index 7521955abc077..dbb3d0d087304 100644 --- a/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/OffsetStorageReaderImpl.java +++ b/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/OffsetStorageReaderImpl.java @@ -62,7 +62,7 @@ public Map, Map> offsets(Collection Map, Map> offsets(Collection serialized value from backing store Map raw; try { - raw = backingStore.get(namespace, serializedToOriginal.keySet(), null).get(); + raw = backingStore.get(serializedToOriginal.keySet(), null).get(); } catch (Exception e) { log.error("Failed to fetch offsets from namespace {}: ", namespace, e); throw new CopycatException("Failed to fetch offsets.", e); diff --git a/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/OffsetStorageWriter.java b/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/OffsetStorageWriter.java index be8c7187308d7..59c12a7c9ade3 100644 --- a/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/OffsetStorageWriter.java +++ b/copycat/runtime/src/main/java/org/apache/kafka/copycat/storage/OffsetStorageWriter.java @@ -139,7 +139,8 @@ public Future doFlush(final Callback callback) { // for that data. The only enforcement of the format is here. OffsetUtils.validateFormat(entry.getKey()); OffsetUtils.validateFormat(entry.getValue()); - byte[] key = keyConverter.fromCopycatData(namespace, null, entry.getKey()); + // When serializing the key, we add in the namespace information so the key is [namespace, real key] + byte[] key = keyConverter.fromCopycatData(namespace, null, Arrays.asList(namespace, entry.getKey())); ByteBuffer keyBuffer = (key != null) ? ByteBuffer.wrap(key) : null; byte[] value = valueConverter.fromCopycatData(namespace, null, entry.getValue()); ByteBuffer valueBuffer = (value != null) ? ByteBuffer.wrap(value) : null; @@ -158,7 +159,7 @@ public Future doFlush(final Callback callback) { // And submit the data log.debug("Submitting {} entries to backing store", offsetsSerialized.size()); - return backingStore.set(namespace, offsetsSerialized, new Callback() { + return backingStore.set(offsetsSerialized, new Callback() { @Override public void onCompletion(Throwable error, Void result) { boolean isCurrent = handleFinishWrite(flushId, error, result); diff --git a/copycat/runtime/src/main/java/org/apache/kafka/copycat/util/ConvertingFutureCallback.java b/copycat/runtime/src/main/java/org/apache/kafka/copycat/util/ConvertingFutureCallback.java new file mode 100644 index 0000000000000..6bf38857933e5 --- /dev/null +++ b/copycat/runtime/src/main/java/org/apache/kafka/copycat/util/ConvertingFutureCallback.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +package org.apache.kafka.copycat.util; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public abstract class ConvertingFutureCallback implements Callback, Future { + + private Callback underlying; + private CountDownLatch finishedLatch; + private T result = null; + private Throwable exception = null; + + public ConvertingFutureCallback(Callback underlying) { + this.underlying = underlying; + this.finishedLatch = new CountDownLatch(1); + } + + public abstract T convert(U result); + + @Override + public void onCompletion(Throwable error, U result) { + this.exception = error; + this.result = convert(result); + if (underlying != null) + underlying.onCompletion(error, this.result); + finishedLatch.countDown(); + } + + @Override + public boolean cancel(boolean b) { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return finishedLatch.getCount() == 0; + } + + @Override + public T get() throws InterruptedException, ExecutionException { + finishedLatch.await(); + return result(); + } + + @Override + public T get(long l, TimeUnit timeUnit) + throws InterruptedException, ExecutionException, TimeoutException { + finishedLatch.await(l, timeUnit); + return result(); + } + + private T result() throws ExecutionException { + if (exception != null) { + throw new ExecutionException(exception); + } + return result; + } +} + diff --git a/copycat/runtime/src/main/java/org/apache/kafka/copycat/util/FutureCallback.java b/copycat/runtime/src/main/java/org/apache/kafka/copycat/util/FutureCallback.java index db9f2c457621b..61e04b6d53441 100644 --- a/copycat/runtime/src/main/java/org/apache/kafka/copycat/util/FutureCallback.java +++ b/copycat/runtime/src/main/java/org/apache/kafka/copycat/util/FutureCallback.java @@ -17,60 +17,14 @@ package org.apache.kafka.copycat.util; -import java.util.concurrent.*; - -public class FutureCallback implements Callback, Future { - - private Callback underlying; - private CountDownLatch finishedLatch; - private T result = null; - private Throwable exception = null; +public class FutureCallback extends ConvertingFutureCallback { public FutureCallback(Callback underlying) { - this.underlying = underlying; - this.finishedLatch = new CountDownLatch(1); - } - - @Override - public void onCompletion(Throwable error, T result) { - underlying.onCompletion(error, result); - this.exception = error; - this.result = result; - finishedLatch.countDown(); + super(underlying); } @Override - public boolean cancel(boolean b) { - return false; - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return finishedLatch.getCount() == 0; - } - - @Override - public T get() throws InterruptedException, ExecutionException { - finishedLatch.await(); - return result(); - } - - @Override - public T get(long l, TimeUnit timeUnit) - throws InterruptedException, ExecutionException, TimeoutException { - finishedLatch.await(l, timeUnit); - return result(); - } - - private T result() throws ExecutionException { - if (exception != null) { - throw new ExecutionException(exception); - } + public T convert(T result) { return result; } } diff --git a/copycat/runtime/src/test/java/org/apache/kafka/copycat/runtime/WorkerSinkTaskTest.java b/copycat/runtime/src/test/java/org/apache/kafka/copycat/runtime/WorkerSinkTaskTest.java index 91469d366658c..687ed8fd56d2d 100644 --- a/copycat/runtime/src/test/java/org/apache/kafka/copycat/runtime/WorkerSinkTaskTest.java +++ b/copycat/runtime/src/test/java/org/apache/kafka/copycat/runtime/WorkerSinkTaskTest.java @@ -346,7 +346,7 @@ public Long answer() throws Throwable { } ); - sinkTask.flush(Collections.singletonMap(TOPIC_PARTITION, finalOffset)); + sinkTask.flush(Collections.singletonMap(TOPIC_PARTITION, new OffsetAndMetadata(finalOffset))); IExpectationSetters flushExpectation = PowerMock.expectLastCall(); if (flushError != null) { flushExpectation.andThrow(flushError).once(); @@ -354,7 +354,7 @@ public Long answer() throws Throwable { } final Capture capturedCallback = EasyMock.newCapture(); - final Map offsets = Collections.singletonMap(TOPIC_PARTITION, finalOffset); + final Map offsets = Collections.singletonMap(TOPIC_PARTITION, new OffsetAndMetadata(finalOffset)); consumer.commitAsync(EasyMock.eq(offsets), EasyMock.capture(capturedCallback)); PowerMock.expectLastCall().andAnswer(new IAnswer() { diff --git a/copycat/runtime/src/test/java/org/apache/kafka/copycat/runtime/WorkerTest.java b/copycat/runtime/src/test/java/org/apache/kafka/copycat/runtime/WorkerTest.java index 701e23023f2aa..e75d2f90d1ad4 100644 --- a/copycat/runtime/src/test/java/org/apache/kafka/copycat/runtime/WorkerTest.java +++ b/copycat/runtime/src/test/java/org/apache/kafka/copycat/runtime/WorkerTest.java @@ -37,6 +37,7 @@ import org.powermock.modules.junit4.PowerMockRunner; import java.util.List; +import java.util.Map; import java.util.Properties; @RunWith(PowerMockRunner.class) @@ -45,6 +46,7 @@ public class WorkerTest extends ThreadedTest { private ConnectorTaskId taskId = new ConnectorTaskId("job", 0); + private WorkerConfig config; private Worker worker; private OffsetBackingStore offsetBackingStore = PowerMock.createMock(OffsetBackingStore.class); @@ -59,13 +61,16 @@ public void setup() { workerProps.setProperty("offset.value.converter", "org.apache.kafka.copycat.json.JsonConverter"); workerProps.setProperty("offset.key.converter.schemas.enable", "false"); workerProps.setProperty("offset.value.converter.schemas.enable", "false"); - WorkerConfig config = new WorkerConfig(workerProps); - worker = new Worker(new MockTime(), config, offsetBackingStore); - worker.start(); + config = new WorkerConfig(workerProps); } @Test public void testAddRemoveTask() throws Exception { + offsetBackingStore.configure(EasyMock.anyObject(Map.class)); + EasyMock.expectLastCall(); + offsetBackingStore.start(); + EasyMock.expectLastCall(); + ConnectorTaskId taskId = new ConnectorTaskId("job", 0); // Create @@ -96,8 +101,13 @@ public void testAddRemoveTask() throws Exception { workerTask.close(); EasyMock.expectLastCall(); + offsetBackingStore.stop(); + EasyMock.expectLastCall(); + PowerMock.replayAll(); + worker = new Worker(new MockTime(), config, offsetBackingStore); + worker.start(); worker.addTask(taskId, TestSourceTask.class.getName(), origProps); worker.stopTask(taskId); // Nothing should be left, so this should effectively be a nop @@ -108,11 +118,26 @@ public void testAddRemoveTask() throws Exception { @Test(expected = CopycatException.class) public void testStopInvalidTask() { + offsetBackingStore.configure(EasyMock.anyObject(Map.class)); + EasyMock.expectLastCall(); + offsetBackingStore.start(); + EasyMock.expectLastCall(); + + PowerMock.replayAll(); + + worker = new Worker(new MockTime(), config, offsetBackingStore); + worker.start(); + worker.stopTask(taskId); } @Test public void testCleanupTasksOnStop() throws Exception { + offsetBackingStore.configure(EasyMock.anyObject(Map.class)); + EasyMock.expectLastCall(); + offsetBackingStore.start(); + EasyMock.expectLastCall(); + // Create TestSourceTask task = PowerMock.createMock(TestSourceTask.class); WorkerSourceTask workerTask = PowerMock.createMock(WorkerSourceTask.class); @@ -142,8 +167,13 @@ public void testCleanupTasksOnStop() throws Exception { workerTask.close(); EasyMock.expectLastCall(); + offsetBackingStore.stop(); + EasyMock.expectLastCall(); + PowerMock.replayAll(); + worker = new Worker(new MockTime(), config, offsetBackingStore); + worker.start(); worker.addTask(taskId, TestSourceTask.class.getName(), origProps); worker.stop(); diff --git a/copycat/runtime/src/test/java/org/apache/kafka/copycat/storage/FileOffsetBackingStoreTest.java b/copycat/runtime/src/test/java/org/apache/kafka/copycat/storage/FileOffsetBackingStoreTest.java index bbcbdc9c6064a..2976c0a7398df 100644 --- a/copycat/runtime/src/test/java/org/apache/kafka/copycat/storage/FileOffsetBackingStoreTest.java +++ b/copycat/runtime/src/test/java/org/apache/kafka/copycat/storage/FileOffsetBackingStoreTest.java @@ -67,9 +67,9 @@ public void testGetSet() throws Exception { Callback> getCallback = expectSuccessfulGetCallback(); PowerMock.replayAll(); - store.set("namespace", firstSet, setCallback).get(); + store.set(firstSet, setCallback).get(); - Map values = store.get("namespace", Arrays.asList(buffer("key"), buffer("bad")), getCallback).get(); + Map values = store.get(Arrays.asList(buffer("key"), buffer("bad")), getCallback).get(); assertEquals(buffer("value"), values.get(buffer("key"))); assertEquals(null, values.get(buffer("bad"))); @@ -82,14 +82,14 @@ public void testSaveRestore() throws Exception { Callback> getCallback = expectSuccessfulGetCallback(); PowerMock.replayAll(); - store.set("namespace", firstSet, setCallback).get(); + store.set(firstSet, setCallback).get(); store.stop(); // Restore into a new store to ensure correct reload from scratch FileOffsetBackingStore restore = new FileOffsetBackingStore(); restore.configure(props); restore.start(); - Map values = restore.get("namespace", Arrays.asList(buffer("key")), getCallback).get(); + Map values = restore.get(Arrays.asList(buffer("key")), getCallback).get(); assertEquals(buffer("value"), values.get(buffer("key"))); PowerMock.verifyAll(); diff --git a/copycat/runtime/src/test/java/org/apache/kafka/copycat/storage/KafkaOffsetBackingStoreTest.java b/copycat/runtime/src/test/java/org/apache/kafka/copycat/storage/KafkaOffsetBackingStoreTest.java new file mode 100644 index 0000000000000..6a3eec3e48469 --- /dev/null +++ b/copycat/runtime/src/test/java/org/apache/kafka/copycat/storage/KafkaOffsetBackingStoreTest.java @@ -0,0 +1,458 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +package org.apache.kafka.copycat.storage; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.MockConsumer; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.LeaderNotAvailableException; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.copycat.errors.CopycatException; +import org.apache.kafka.copycat.util.Callback; +import org.apache.kafka.copycat.util.TestFuture; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.api.easymock.annotation.Mock; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.apache.kafka.copycat.util.ByteArrayProducerRecordEquals.eqProducerRecord; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(KafkaOffsetBackingStore.class) +@PowerMockIgnore("javax.management.*") +public class KafkaOffsetBackingStoreTest { + private static final String TOPIC = "copycat-offsets"; + private static final TopicPartition TP0 = new TopicPartition(TOPIC, 0); + private static final TopicPartition TP1 = new TopicPartition(TOPIC, 1); + private static final Map DEFAULT_PROPS = new HashMap<>(); + static { + DEFAULT_PROPS.put(KafkaOffsetBackingStore.OFFSET_STORAGE_TOPIC_CONFIG, TOPIC); + DEFAULT_PROPS.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "broker1:9092,broker2:9093"); + } + private static final Set CONSUMER_ASSIGNMENT = new HashSet<>(Arrays.asList(TP0, TP1)); + private static final Map FIRST_SET = new HashMap<>(); + static { + FIRST_SET.put(buffer("key"), buffer("value")); + FIRST_SET.put(null, null); + } + + + private static final Node LEADER = new Node(1, "broker1", 9092); + private static final Node REPLICA = new Node(1, "broker2", 9093); + + private static final PartitionInfo TPINFO0 = new PartitionInfo(TOPIC, 0, LEADER, new Node[]{REPLICA}, new Node[]{REPLICA}); + private static final PartitionInfo TPINFO1 = new PartitionInfo(TOPIC, 1, LEADER, new Node[]{REPLICA}, new Node[]{REPLICA}); + + private static final ByteBuffer TP0_KEY = buffer("TP0KEY"); + private static final ByteBuffer TP1_KEY = buffer("TP1KEY"); + private static final ByteBuffer TP0_VALUE = buffer("VAL0"); + private static final ByteBuffer TP1_VALUE = buffer("VAL1"); + private static final ByteBuffer TP0_VALUE_NEW = buffer("VAL0_NEW"); + private static final ByteBuffer TP1_VALUE_NEW = buffer("VAL1_NEW"); + + private KafkaOffsetBackingStore store; + + @Mock private KafkaProducer producer; + private MockConsumer consumer; + + @Before + public void setUp() throws Exception { + store = PowerMock.createPartialMockAndInvokeDefaultConstructor(KafkaOffsetBackingStore.class, new String[]{"createConsumer", "createProducer"}); + consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST); + consumer.updatePartitions(TOPIC, Arrays.asList(TPINFO0, TPINFO1)); + Map beginningOffsets = new HashMap<>(); + beginningOffsets.put(TP0, 0L); + beginningOffsets.put(TP1, 0L); + consumer.updateBeginningOffsets(beginningOffsets); + } + + @Test(expected = CopycatException.class) + public void testMissingTopic() { + store = new KafkaOffsetBackingStore(); + store.configure(Collections.emptyMap()); + } + + @Test + public void testStartStop() throws Exception { + expectStart(); + expectStop(); + + PowerMock.replayAll(); + + store.configure(DEFAULT_PROPS); + + Map endOffsets = new HashMap<>(); + endOffsets.put(TP0, 0L); + endOffsets.put(TP1, 0L); + consumer.updateEndOffsets(endOffsets); + store.start(); + assertEquals(CONSUMER_ASSIGNMENT, consumer.assignment()); + + store.stop(); + + assertFalse(Whitebox.getInternalState(store, "thread").isAlive()); + assertTrue(consumer.closed()); + PowerMock.verifyAll(); + } + + @Test + public void testReloadOnStart() throws Exception { + expectStart(); + expectStop(); + + PowerMock.replayAll(); + + store.configure(DEFAULT_PROPS); + + Map endOffsets = new HashMap<>(); + endOffsets.put(TP0, 1L); + endOffsets.put(TP1, 1L); + consumer.updateEndOffsets(endOffsets); + Thread startConsumerOpsThread = new Thread("start-consumer-ops-thread") { + @Override + public void run() { + // Needs to seek to end to find end offsets + consumer.waitForPoll(10000); + + // Should keep polling until it reaches current log end offset for all partitions + consumer.waitForPollThen(new Runnable() { + @Override + public void run() { + consumer.addRecord(new ConsumerRecord<>(TOPIC, 0, 0, TP0_KEY.array(), TP0_VALUE.array())); + } + }, 10000); + + consumer.waitForPollThen(new Runnable() { + @Override + public void run() { + consumer.addRecord(new ConsumerRecord<>(TOPIC, 1, 0, TP1_KEY.array(), TP1_VALUE.array())); + } + }, 10000); + } + }; + startConsumerOpsThread.start(); + store.start(); + startConsumerOpsThread.join(10000); + assertFalse(startConsumerOpsThread.isAlive()); + assertEquals(CONSUMER_ASSIGNMENT, consumer.assignment()); + HashMap data = Whitebox.getInternalState(store, "data"); + assertEquals(TP0_VALUE, data.get(TP0_KEY)); + assertEquals(TP1_VALUE, data.get(TP1_KEY)); + + store.stop(); + + assertFalse(Whitebox.getInternalState(store, "thread").isAlive()); + assertTrue(consumer.closed()); + PowerMock.verifyAll(); + } + + @Test + public void testGetSet() throws Exception { + expectStart(); + TestFuture tp0Future = new TestFuture<>(); + ProducerRecord tp0Record = new ProducerRecord<>(TOPIC, TP0_KEY.array(), TP0_VALUE.array()); + Capture callback1 = EasyMock.newCapture(); + EasyMock.expect(producer.send(eqProducerRecord(tp0Record), EasyMock.capture(callback1))).andReturn(tp0Future); + TestFuture tp1Future = new TestFuture<>(); + ProducerRecord tp1Record = new ProducerRecord<>(TOPIC, TP1_KEY.array(), TP1_VALUE.array()); + Capture callback2 = EasyMock.newCapture(); + EasyMock.expect(producer.send(eqProducerRecord(tp1Record), EasyMock.capture(callback2))).andReturn(tp1Future); + + expectStop(); + + PowerMock.replayAll(); + + store.configure(DEFAULT_PROPS); + + Map endOffsets = new HashMap<>(); + endOffsets.put(TP0, 0L); + endOffsets.put(TP1, 0L); + consumer.updateEndOffsets(endOffsets); + Thread startConsumerOpsThread = new Thread("start-consumer-ops-thread") { + @Override + public void run() { + // Should keep polling until it has partition info + consumer.waitForPollThen(new Runnable() { + @Override + public void run() { + consumer.seek(TP0, 0); + consumer.seek(TP1, 0); + } + }, 10000); + } + }; + startConsumerOpsThread.start(); + store.start(); + startConsumerOpsThread.join(10000); + assertFalse(startConsumerOpsThread.isAlive()); + assertEquals(CONSUMER_ASSIGNMENT, consumer.assignment()); + + Map toSet = new HashMap<>(); + toSet.put(TP0_KEY, TP0_VALUE); + toSet.put(TP1_KEY, TP1_VALUE); + final AtomicBoolean invoked = new AtomicBoolean(false); + Future setFuture = store.set(toSet, new Callback() { + @Override + public void onCompletion(Throwable error, Void result) { + invoked.set(true); + } + }); + assertFalse(setFuture.isDone()); + tp1Future.resolve((RecordMetadata) null); // Output not used, so safe to not return a real value for testing + assertFalse(setFuture.isDone()); + tp0Future.resolve((RecordMetadata) null); + // Out of order callbacks shouldn't matter, should still require all to be invoked before invoking the callback + // for the store's set callback + callback2.getValue().onCompletion(null, null); + assertFalse(invoked.get()); + callback1.getValue().onCompletion(null, null); + setFuture.get(10000, TimeUnit.MILLISECONDS); + assertTrue(invoked.get()); + + // Getting data should continue to return old data... + final AtomicBoolean getInvokedAndPassed = new AtomicBoolean(false); + store.get(Arrays.asList(TP0_KEY, TP1_KEY), new Callback>() { + @Override + public void onCompletion(Throwable error, Map result) { + // Since we didn't read them yet, these will be null + assertEquals(null, result.get(TP0_KEY)); + assertEquals(null, result.get(TP1_KEY)); + getInvokedAndPassed.set(true); + } + }).get(10000, TimeUnit.MILLISECONDS); + assertTrue(getInvokedAndPassed.get()); + + // Until the consumer gets the new data + Thread readNewDataThread = new Thread("read-new-data-thread") { + @Override + public void run() { + // Should keep polling until it reaches current log end offset for all partitions + consumer.waitForPollThen(new Runnable() { + @Override + public void run() { + consumer.addRecord(new ConsumerRecord<>(TOPIC, 0, 0, TP0_KEY.array(), TP0_VALUE_NEW.array())); + } + }, 10000); + + consumer.waitForPollThen(new Runnable() { + @Override + public void run() { + consumer.addRecord(new ConsumerRecord<>(TOPIC, 1, 0, TP1_KEY.array(), TP1_VALUE_NEW.array())); + } + }, 10000); + } + }; + readNewDataThread.start(); + readNewDataThread.join(10000); + assertFalse(readNewDataThread.isAlive()); + + // And now the new data should be returned + final AtomicBoolean finalGetInvokedAndPassed = new AtomicBoolean(false); + store.get(Arrays.asList(TP0_KEY, TP1_KEY), new Callback>() { + @Override + public void onCompletion(Throwable error, Map result) { + assertEquals(TP0_VALUE_NEW, result.get(TP0_KEY)); + assertEquals(TP1_VALUE_NEW, result.get(TP1_KEY)); + finalGetInvokedAndPassed.set(true); + } + }).get(10000, TimeUnit.MILLISECONDS); + assertTrue(finalGetInvokedAndPassed.get()); + + store.stop(); + + assertFalse(Whitebox.getInternalState(store, "thread").isAlive()); + assertTrue(consumer.closed()); + PowerMock.verifyAll(); + } + + @Test + public void testConsumerError() throws Exception { + expectStart(); + expectStop(); + + PowerMock.replayAll(); + + store.configure(DEFAULT_PROPS); + + Map endOffsets = new HashMap<>(); + endOffsets.put(TP0, 1L); + endOffsets.put(TP1, 1L); + consumer.updateEndOffsets(endOffsets); + Thread startConsumerOpsThread = new Thread("start-consumer-ops-thread") { + @Override + public void run() { + // Trigger exception + consumer.waitForPollThen(new Runnable() { + @Override + public void run() { + consumer.setException(Errors.CONSUMER_COORDINATOR_NOT_AVAILABLE.exception()); + } + }, 10000); + + // Should keep polling until it reaches current log end offset for all partitions + consumer.waitForPollThen(new Runnable() { + @Override + public void run() { + consumer.addRecord(new ConsumerRecord<>(TOPIC, 0, 0, TP0_KEY.array(), TP0_VALUE_NEW.array())); + consumer.addRecord(new ConsumerRecord<>(TOPIC, 1, 0, TP0_KEY.array(), TP0_VALUE_NEW.array())); + } + }, 10000); + } + }; + startConsumerOpsThread.start(); + store.start(); + startConsumerOpsThread.join(10000); + assertFalse(startConsumerOpsThread.isAlive()); + assertEquals(CONSUMER_ASSIGNMENT, consumer.assignment()); + + store.stop(); + + assertFalse(Whitebox.getInternalState(store, "thread").isAlive()); + assertTrue(consumer.closed()); + PowerMock.verifyAll(); + } + + @Test + public void testProducerError() throws Exception { + expectStart(); + TestFuture tp0Future = new TestFuture<>(); + ProducerRecord tp0Record = new ProducerRecord<>(TOPIC, TP0_KEY.array(), TP0_VALUE.array()); + Capture callback1 = EasyMock.newCapture(); + EasyMock.expect(producer.send(eqProducerRecord(tp0Record), EasyMock.capture(callback1))).andReturn(tp0Future); + TestFuture tp1Future = new TestFuture<>(); + ProducerRecord tp1Record = new ProducerRecord<>(TOPIC, TP1_KEY.array(), TP1_VALUE.array()); + Capture callback2 = EasyMock.newCapture(); + EasyMock.expect(producer.send(eqProducerRecord(tp1Record), EasyMock.capture(callback2))).andReturn(tp1Future); + + expectStop(); + + PowerMock.replayAll(); + + store.configure(DEFAULT_PROPS); + + Map endOffsets = new HashMap<>(); + endOffsets.put(TP0, 0L); + endOffsets.put(TP1, 0L); + consumer.updateEndOffsets(endOffsets); + Thread startConsumerOpsThread = new Thread("start-consumer-ops-thread") { + @Override + public void run() { + // Should keep polling until it has partition info + consumer.waitForPollThen(new Runnable() { + @Override + public void run() { + consumer.seek(TP0, 0); + consumer.seek(TP1, 0); + } + }, 10000); + } + }; + startConsumerOpsThread.start(); + store.start(); + startConsumerOpsThread.join(10000); + assertFalse(startConsumerOpsThread.isAlive()); + assertEquals(CONSUMER_ASSIGNMENT, consumer.assignment()); + + Map toSet = new HashMap<>(); + toSet.put(TP0_KEY, TP0_VALUE); + toSet.put(TP1_KEY, TP1_VALUE); + final AtomicReference setException = new AtomicReference<>(); + Future setFuture = store.set(toSet, new Callback() { + @Override + public void onCompletion(Throwable error, Void result) { + assertNull(setException.get()); // Should only be invoked once + setException.set(error); + } + }); + assertFalse(setFuture.isDone()); + KafkaException exc = new LeaderNotAvailableException("Error"); + tp1Future.resolve(exc); + callback2.getValue().onCompletion(null, exc); + // One failure should resolve the future immediately + try { + setFuture.get(10000, TimeUnit.MILLISECONDS); + fail("Should have see ExecutionException"); + } catch (ExecutionException e) { + // expected + } + assertNotNull(setException.get()); + + // Callbacks can continue to arrive + tp0Future.resolve((RecordMetadata) null); + callback1.getValue().onCompletion(null, null); + + store.stop(); + + assertFalse(Whitebox.getInternalState(store, "thread").isAlive()); + assertTrue(consumer.closed()); + PowerMock.verifyAll(); + } + + + private void expectStart() throws Exception { + PowerMock.expectPrivate(store, "createProducer") + .andReturn(producer); + PowerMock.expectPrivate(store, "createConsumer") + .andReturn(consumer); + } + + private void expectStop() { + producer.close(); + PowerMock.expectLastCall(); + // MockConsumer close is checked after test. + } + + private static ByteBuffer buffer(String v) { + return ByteBuffer.wrap(v.getBytes()); + } + +} diff --git a/copycat/runtime/src/test/java/org/apache/kafka/copycat/storage/OffsetStorageWriterTest.java b/copycat/runtime/src/test/java/org/apache/kafka/copycat/storage/OffsetStorageWriterTest.java index 956d06447289a..e33ecd0d99476 100644 --- a/copycat/runtime/src/test/java/org/apache/kafka/copycat/storage/OffsetStorageWriterTest.java +++ b/copycat/runtime/src/test/java/org/apache/kafka/copycat/storage/OffsetStorageWriterTest.java @@ -31,7 +31,9 @@ import org.powermock.modules.junit4.PowerMockRunner; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.*; @@ -43,6 +45,7 @@ public class OffsetStorageWriterTest { private static final String NAMESPACE = "namespace"; // Copycat format - any types should be accepted here private static final Map OFFSET_KEY = Collections.singletonMap("key", "key"); + private static final List OFFSET_KEY_WRAPPED = Arrays.asList(NAMESPACE, OFFSET_KEY); private static final Map OFFSET_VALUE = Collections.singletonMap("key", 12); // Serialized @@ -195,12 +198,11 @@ private void expectStore(final Callback callback, final boolean fail) { private void expectStore(final Callback callback, final boolean fail, final CountDownLatch waitForCompletion) { - EasyMock.expect(keyConverter.fromCopycatData(NAMESPACE, null, OFFSET_KEY)).andReturn(OFFSET_KEY_SERIALIZED); + EasyMock.expect(keyConverter.fromCopycatData(NAMESPACE, null, OFFSET_KEY_WRAPPED)).andReturn(OFFSET_KEY_SERIALIZED); EasyMock.expect(valueConverter.fromCopycatData(NAMESPACE, null, OFFSET_VALUE)).andReturn(OFFSET_VALUE_SERIALIZED); final Capture> storeCallback = Capture.newInstance(); - EasyMock.expect(store.set(EasyMock.eq(NAMESPACE), EasyMock.eq(OFFSETS_SERIALIZED), - EasyMock.capture(storeCallback))) + EasyMock.expect(store.set(EasyMock.eq(OFFSETS_SERIALIZED), EasyMock.capture(storeCallback))) .andAnswer(new IAnswer>() { @Override public Future answer() throws Throwable { diff --git a/copycat/runtime/src/test/java/org/apache/kafka/copycat/util/ByteArrayProducerRecordEquals.java b/copycat/runtime/src/test/java/org/apache/kafka/copycat/util/ByteArrayProducerRecordEquals.java new file mode 100644 index 0000000000000..929ea859bb14b --- /dev/null +++ b/copycat/runtime/src/test/java/org/apache/kafka/copycat/util/ByteArrayProducerRecordEquals.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +package org.apache.kafka.copycat.util; + +import org.apache.kafka.clients.producer.ProducerRecord; +import org.easymock.EasyMock; +import org.easymock.IArgumentMatcher; + +import java.util.Arrays; + +public class ByteArrayProducerRecordEquals implements IArgumentMatcher { + private ProducerRecord record; + + public static ProducerRecord eqProducerRecord(ProducerRecord in) { + EasyMock.reportMatcher(new ByteArrayProducerRecordEquals(in)); + return null; + } + + public ByteArrayProducerRecordEquals(ProducerRecord record) { + this.record = record; + } + + @Override + public boolean matches(Object argument) { + if (!(argument instanceof ProducerRecord)) + return false; + ProducerRecord other = (ProducerRecord) argument; + return record.topic().equals(other.topic()) && + record.partition() != null ? record.partition().equals(other.partition()) : other.partition() == null && + record.key() != null ? Arrays.equals(record.key(), other.key()) : other.key() == null && + record.value() != null ? Arrays.equals(record.value(), other.value()) : other.value() == null; + } + + @Override + public void appendTo(StringBuffer buffer) { + buffer.append(record.toString()); + } +} diff --git a/copycat/runtime/src/test/java/org/apache/kafka/copycat/util/TestFuture.java b/copycat/runtime/src/test/java/org/apache/kafka/copycat/util/TestFuture.java new file mode 100644 index 0000000000000..8143c44a6148f --- /dev/null +++ b/copycat/runtime/src/test/java/org/apache/kafka/copycat/util/TestFuture.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +package org.apache.kafka.copycat.util; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class TestFuture implements Future { + private volatile boolean resolved; + private T result; + private Throwable exception; + + public TestFuture() { + resolved = false; + } + + public void resolve(T val) { + this.result = val; + resolved = true; + + } + + public void resolve(Throwable t) { + exception = t; + resolved = true; + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return resolved; + } + + @Override + public T get() throws InterruptedException, ExecutionException { + while (true) { + try { + return get(Integer.MAX_VALUE, TimeUnit.DAYS); + } catch (TimeoutException e) { + // ignore + } + } + } + + @Override + public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + while (!resolved) { + this.wait(TimeUnit.MILLISECONDS.convert(timeout, unit)); + } + + if (exception != null) + throw new ExecutionException(exception); + return result; + } +} diff --git a/copycat/runtime/src/test/resources/log4j.properties b/copycat/runtime/src/test/resources/log4j.properties new file mode 100644 index 0000000000000..d5e90fe788f76 --- /dev/null +++ b/copycat/runtime/src/test/resources/log4j.properties @@ -0,0 +1,23 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## +log4j.rootLogger=OFF, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n + +log4j.logger.org.apache.kafka=ERROR diff --git a/core/src/main/scala/kafka/api/ApiUtils.scala b/core/src/main/scala/kafka/api/ApiUtils.scala index 1f80de1638978..ca0a63f8423e2 100644 --- a/core/src/main/scala/kafka/api/ApiUtils.scala +++ b/core/src/main/scala/kafka/api/ApiUtils.scala @@ -14,10 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package kafka.api +package kafka.api import java.nio._ +import java.nio.channels.GatheringByteChannel import kafka.common._ +import org.apache.kafka.common.network.TransportLayer /** * Helper functions specific to parsing or serializing requests and responses @@ -107,5 +109,10 @@ object ApiUtils { throw new KafkaException(name + " has value " + value + " which is not in the range " + range + ".") else value } + + private[api] def hasPendingWrites(channel: GatheringByteChannel): Boolean = channel match { + case t: TransportLayer => t.hasPendingWrites + case _ => false + } } diff --git a/core/src/main/scala/kafka/api/ApiVersion.scala b/core/src/main/scala/kafka/api/ApiVersion.scala index a7c15f3781afa..c9c19761ec288 100644 --- a/core/src/main/scala/kafka/api/ApiVersion.scala +++ b/core/src/main/scala/kafka/api/ApiVersion.scala @@ -33,7 +33,7 @@ object ApiVersion { "0.8.0" -> KAFKA_080, "0.8.1" -> KAFKA_081, "0.8.2" -> KAFKA_082, - "0.8.3" -> KAFKA_083 + "0.9.0" -> KAFKA_090 ) def apply(version: String): ApiVersion = versionNameMap(version.split("\\.").slice(0,3).mkString(".")) @@ -72,7 +72,7 @@ case object KAFKA_082 extends ApiVersion { val id: Int = 2 } -case object KAFKA_083 extends ApiVersion { - val version: String = "0.8.3.X" +case object KAFKA_090 extends ApiVersion { + val version: String = "0.9.0.X" val id: Int = 3 -} \ No newline at end of file +} diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala index 2c0703357661b..aa15612bd5d7f 100644 --- a/core/src/main/scala/kafka/api/FetchResponse.scala +++ b/core/src/main/scala/kafka/api/FetchResponse.scala @@ -24,9 +24,7 @@ import kafka.common.{TopicAndPartition, ErrorMapping} import kafka.message.{MessageSet, ByteBufferMessageSet} import kafka.api.ApiUtils._ import org.apache.kafka.common.KafkaException -import org.apache.kafka.common.network.TransportLayer -import org.apache.kafka.common.network.Send -import org.apache.kafka.common.network.MultiSend +import org.apache.kafka.common.network.{SSLTransportLayer, TransportLayer, Send, MultiSend} import scala.collection._ @@ -55,6 +53,7 @@ case class FetchResponsePartitionData(error: Short = ErrorMapping.NoError, hw: L class PartitionDataSend(val partitionId: Int, val partitionData: FetchResponsePartitionData) extends Send { + private val emptyBuffer = ByteBuffer.allocate(0) private val messageSize = partitionData.messages.sizeInBytes private var messagesSentSize = 0 private var pending = false @@ -71,15 +70,20 @@ class PartitionDataSend(val partitionId: Int, override def writeTo(channel: GatheringByteChannel): Long = { var written = 0L - if(buffer.hasRemaining) + if (buffer.hasRemaining) written += channel.write(buffer) - if (!buffer.hasRemaining && messagesSentSize < messageSize) { - val bytesSent = partitionData.messages.writeTo(channel, messagesSentSize, messageSize - messagesSentSize) - messagesSentSize += bytesSent - written += bytesSent + if (!buffer.hasRemaining) { + if (messagesSentSize < messageSize) { + val bytesSent = partitionData.messages.writeTo(channel, messagesSentSize, messageSize - messagesSentSize) + messagesSentSize += bytesSent + written += bytesSent + } + if (messagesSentSize >= messageSize && hasPendingWrites(channel)) + channel.write(emptyBuffer) } - if (channel.isInstanceOf[TransportLayer]) - pending = channel.asInstanceOf[TransportLayer].hasPendingWrites + + pending = hasPendingWrites(channel) + written } @@ -112,6 +116,8 @@ case class TopicData(topic: String, partitionData: Map[Int, FetchResponsePartiti class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { + private val emptyBuffer = ByteBuffer.allocate(0) + private var sent = 0L private var pending = false @@ -135,14 +141,16 @@ class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { throw new KafkaException("This operation cannot be completed on a complete request.") var written = 0L - if(buffer.hasRemaining) + if (buffer.hasRemaining) written += channel.write(buffer) - if(!buffer.hasRemaining && !sends.completed) { - written += sends.writeTo(channel) + if (!buffer.hasRemaining) { + if (!sends.completed) + written += sends.writeTo(channel) + if (sends.completed && hasPendingWrites(channel)) + written += channel.write(emptyBuffer) } - if (channel.isInstanceOf[TransportLayer]) - pending = channel.asInstanceOf[TransportLayer].hasPendingWrites + pending = hasPendingWrites(channel) sent += written written @@ -245,6 +253,9 @@ case class FetchResponse(correlationId: Int, class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) extends Send { + + private val emptyBuffer = ByteBuffer.allocate(0) + private val payloadSize = fetchResponse.sizeInBytes private var sent = 0L @@ -272,15 +283,18 @@ class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) exte throw new KafkaException("This operation cannot be completed on a complete request.") var written = 0L - if(buffer.hasRemaining) + + if (buffer.hasRemaining) written += channel.write(buffer) - if(!buffer.hasRemaining && !sends.completed) { - written += sends.writeTo(channel) + if (!buffer.hasRemaining) { + if (!sends.completed) + written += sends.writeTo(channel) + if (sends.completed && hasPendingWrites(channel)) + written += channel.write(emptyBuffer) } - sent += written - if (channel.isInstanceOf[TransportLayer]) - pending = channel.asInstanceOf[TransportLayer].hasPendingWrites + sent += written + pending = hasPendingWrites(channel) written } diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index e42d10488f8df..2027ec8aadadf 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -36,13 +36,12 @@ import kafka.utils.CoreUtils.inLock import kafka.utils.ZkUtils._ import kafka.utils._ import org.I0Itec.zkclient.exception.ZkNodeExistsException -import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient} +import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient, ZkConnection} import org.apache.zookeeper.Watcher.Event.KeeperState import scala.collection._ import scala.collection.JavaConversions._ - /** * This class handles the consumers interaction with zookeeper * @@ -90,6 +89,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, private val rebalanceLock = new Object private var fetcher: Option[ConsumerFetcherManager] = None private var zkClient: ZkClient = null + private var zkConnection : ZkConnection = null private var topicRegistry = new Pool[String, Pool[Int, PartitionTopicInfo]] private val checkpointedZkOffsets = new Pool[TopicAndPartition, Long] private val topicThreadIdAndQueues = new Pool[(String, ConsumerThreadId), BlockingQueue[FetchedDataChunk]] @@ -178,7 +178,9 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, private def connectZk() { info("Connecting to zookeeper instance at " + config.zkConnect) - zkClient = ZkUtils.createZkClient(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + val (client, connection) = ZkUtils.createZkClientAndConnection(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + zkClient = client + zkConnection = connection } // Blocks until the offset manager is located and a channel is established to it. @@ -261,9 +263,12 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, val timestamp = SystemTime.milliseconds.toString val consumerRegistrationInfo = Json.encode(Map("version" -> 1, "subscription" -> topicCount.getTopicCountMap, "pattern" -> topicCount.pattern, "timestamp" -> timestamp)) + val zkWatchedEphemeral = new ZKCheckedEphemeral(dirs. + consumerRegistryDir + "/" + consumerIdString, + consumerRegistrationInfo, + zkConnection.getZookeeper) + zkWatchedEphemeral.create() - createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, - (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) info("end registering consumer " + consumerIdString + " in ZK") } diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala index 0d62c96663519..1220c3829da26 100755 --- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala +++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala @@ -16,7 +16,7 @@ */ package kafka.controller -import kafka.api.{LeaderAndIsr, KAFKA_083, PartitionStateInfo} +import kafka.api.{LeaderAndIsr, KAFKA_090, PartitionStateInfo} import kafka.utils._ import org.apache.kafka.clients.{ClientResponse, ClientRequest, ManualMetadataUpdater, NetworkClient} import org.apache.kafka.common.{TopicPartition, Node} @@ -379,7 +379,7 @@ class ControllerBrokerRequestBatch(controller: KafkaController) extends Logging topicPartition -> partitionState } - val version = if (controller.config.interBrokerProtocolVersion.onOrAfter(KAFKA_083)) (1: Short) else (0: Short) + val version = if (controller.config.interBrokerProtocolVersion.onOrAfter(KAFKA_090)) (1: Short) else (0: Short) val updateMetadataRequest = if (version == 0) { diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala index 54a31c6cffd54..a7b44cab501a9 100755 --- a/core/src/main/scala/kafka/controller/KafkaController.scala +++ b/core/src/main/scala/kafka/controller/KafkaController.scala @@ -37,13 +37,14 @@ import kafka.utils.CoreUtils._ import org.apache.zookeeper.Watcher.Event.KeeperState import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.utils.Time -import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient} +import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient, ZkConnection} import org.I0Itec.zkclient.exception.{ZkNodeExistsException, ZkNoNodeException} import java.util.concurrent.locks.ReentrantLock import kafka.server._ import kafka.common.TopicAndPartition class ControllerContext(val zkClient: ZkClient, + val zkConnection: ZkConnection, val zkSessionTimeout: Int) { var controllerChannelManager: ControllerChannelManager = null val controllerLock: ReentrantLock = new ReentrantLock() @@ -153,11 +154,11 @@ object KafkaController extends Logging { } } -class KafkaController(val config : KafkaConfig, zkClient: ZkClient, val brokerState: BrokerState, time: Time, metrics: Metrics, threadNamePrefix: Option[String] = None) extends Logging with KafkaMetricsGroup { +class KafkaController(val config : KafkaConfig, zkClient: ZkClient, zkConnection: ZkConnection, val brokerState: BrokerState, time: Time, metrics: Metrics, threadNamePrefix: Option[String] = None) extends Logging with KafkaMetricsGroup { this.logIdent = "[Controller " + config.brokerId + "]: " private var isRunning = true private val stateChangeLogger = KafkaController.stateChangeLogger - val controllerContext = new ControllerContext(zkClient, config.zkSessionTimeoutMs) + val controllerContext = new ControllerContext(zkClient, zkConnection, config.zkSessionTimeoutMs) val partitionStateMachine = new PartitionStateMachine(this) val replicaStateMachine = new ReplicaStateMachine(this) private val controllerElector = new ZookeeperLeaderElector(controllerContext, ZkUtils.ControllerPath, onControllerFailover, diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 2b9531b1ddda9..78bc57644463b 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -689,35 +689,35 @@ class KafkaApis(val requestChannel: RequestChannel, val joinGroupRequest = request.body.asInstanceOf[JoinGroupRequest] val respHeader = new ResponseHeader(request.header.correlationId) - val (authorizedTopics, unauthorizedTopics) = joinGroupRequest.topics().partition { topic => - authorize(request.session, Read, new Resource(Topic, topic)) && - authorize(request.session, Read, new Resource(ConsumerGroup, joinGroupRequest.groupId())) - } - // the callback for sending a join-group response def sendResponseCallback(partitions: Set[TopicAndPartition], consumerId: String, generationId: Int, errorCode: Short) { - val error = if (errorCode == ErrorMapping.NoError && unauthorizedTopics.nonEmpty) ErrorMapping.AuthorizationCode else errorCode - - val partitionList = if (error == ErrorMapping.NoError) + val partitionList = if (errorCode == ErrorMapping.NoError) partitions.map(tp => new TopicPartition(tp.topic, tp.partition)).toBuffer else List.empty.toBuffer - val responseBody = new JoinGroupResponse(error, generationId, consumerId, partitionList) + val responseBody = new JoinGroupResponse(errorCode, generationId, consumerId, partitionList) trace("Sending join group response %s for correlation id %d to client %s." - .format(responseBody, request.header.correlationId, request.header.clientId)) - requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, respHeader, responseBody))) + .format(responseBody, request.header.correlationId, request.header.clientId)) + requestChannel.sendResponse(new Response(request, new ResponseSend(request.connectionId, respHeader, responseBody))) } - // let the coordinator to handle join-group - coordinator.handleJoinGroup( - joinGroupRequest.groupId(), - joinGroupRequest.consumerId(), - authorizedTopics.toSet, - joinGroupRequest.sessionTimeout(), - joinGroupRequest.strategy(), - sendResponseCallback) + // ensure that the client is authorized to join the group and read from all subscribed topics + if (!authorize(request.session, Read, new Resource(ConsumerGroup, joinGroupRequest.groupId())) || + joinGroupRequest.topics().exists(topic => !authorize(request.session, Read, new Resource(Topic, topic)))) { + val responseBody = new JoinGroupResponse(ErrorMapping.AuthorizationCode, 0, joinGroupRequest.consumerId(), List.empty[TopicPartition]) + requestChannel.sendResponse(new Response(request, new ResponseSend(request.connectionId, respHeader, responseBody))) + } else { + // let the coordinator to handle join-group + coordinator.handleJoinGroup( + joinGroupRequest.groupId(), + joinGroupRequest.consumerId(), + joinGroupRequest.topics().toSet, + joinGroupRequest.sessionTimeout(), + joinGroupRequest.strategy(), + sendResponseCallback) + } } def handleHeartbeatRequest(request: RequestChannel.Request) { diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala index 1e8b2331486ff..1c594f6fa0d3d 100755 --- a/core/src/main/scala/kafka/server/KafkaConfig.scala +++ b/core/src/main/scala/kafka/server/KafkaConfig.scala @@ -431,7 +431,7 @@ object KafkaConfig { val InterBrokerSecurityProtocolDoc = "Security protocol used to communicate between brokers. Defaults to plain text." val InterBrokerProtocolVersionDoc = "Specify which version of the inter-broker protocol will be used.\n" + " This is typically bumped after all brokers were upgraded to a new version.\n" + - " Example of some valid values are: 0.8.0, 0.8.1, 0.8.1.1, 0.8.2, 0.8.2.0, 0.8.2.1, 0.8.3, 0.8.3.0. Check ApiVersion for the full list." + " Example of some valid values are: 0.8.0, 0.8.1, 0.8.1.1, 0.8.2, 0.8.2.0, 0.8.2.1, 0.9.0.0, 0.9.0.1 Check ApiVersion for the full list." /** ********* Controlled shutdown configuration ***********/ val ControlledShutdownMaxRetriesDoc = "Controlled shutdown can fail for multiple reasons. This determines the number of retries when such failure happens" val ControlledShutdownRetryBackoffMsDoc = "Before each retry, the system needs time to recover from the state that caused the previous failure (Controller fail over, replica lag etc). This config determines the amount of time to wait before retrying." diff --git a/core/src/main/scala/kafka/server/KafkaHealthcheck.scala b/core/src/main/scala/kafka/server/KafkaHealthcheck.scala index e6e270bbdea6c..16760d4f6d3e6 100644 --- a/core/src/main/scala/kafka/server/KafkaHealthcheck.scala +++ b/core/src/main/scala/kafka/server/KafkaHealthcheck.scala @@ -21,7 +21,7 @@ import kafka.cluster.EndPoint import kafka.utils._ import org.apache.kafka.common.protocol.SecurityProtocol import org.apache.zookeeper.Watcher.Event.KeeperState -import org.I0Itec.zkclient.{IZkStateListener, ZkClient} +import org.I0Itec.zkclient.{IZkStateListener, ZkClient, ZkConnection} import java.net.InetAddress @@ -35,8 +35,8 @@ import java.net.InetAddress */ class KafkaHealthcheck(private val brokerId: Int, private val advertisedEndpoints: Map[SecurityProtocol, EndPoint], - private val zkSessionTimeoutMs: Int, - private val zkClient: ZkClient) extends Logging { + private val zkClient: ZkClient, + private val zkConnection: ZkConnection) extends Logging { val brokerIdPath = ZkUtils.BrokerIdsPath + "/" + brokerId val sessionExpireListener = new SessionExpireListener @@ -62,7 +62,7 @@ class KafkaHealthcheck(private val brokerId: Int, // only PLAINTEXT is supported as default // if the broker doesn't listen on PLAINTEXT protocol, an empty endpoint will be registered and older clients will be unable to connect val plaintextEndpoint = updatedEndpoints.getOrElse(SecurityProtocol.PLAINTEXT, new EndPoint(null,-1,null)) - ZkUtils.registerBrokerInZk(zkClient, brokerId, plaintextEndpoint.host, plaintextEndpoint.port, updatedEndpoints, zkSessionTimeoutMs, jmxPort) + ZkUtils.registerBrokerInZk(zkClient, zkConnection, brokerId, plaintextEndpoint.host, plaintextEndpoint.port, updatedEndpoints, jmxPort) } /** @@ -71,9 +71,7 @@ class KafkaHealthcheck(private val brokerId: Int, */ class SessionExpireListener() extends IZkStateListener { @throws(classOf[Exception]) - def handleStateChanged(state: KeeperState) { - // do nothing, since zkclient will do reconnect for us. - } + def handleStateChanged(state: KeeperState) {} /** * Called after the zookeeper session has expired and a new session has been created. You would have to re-create diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 5cc9c5d5a1eb1..0a60efbfd8df0 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -21,7 +21,7 @@ import java.net.{SocketTimeoutException} import java.util import kafka.admin._ -import kafka.api.{KAFKA_083, ApiVersion} +import kafka.api.{KAFKA_090, ApiVersion} import kafka.log.LogConfig import kafka.log.CleanerConfig import kafka.log.LogManager @@ -43,7 +43,7 @@ import org.apache.kafka.common.utils.AppInfoParser import scala.collection.mutable import scala.collection.JavaConverters._ -import org.I0Itec.zkclient.ZkClient +import org.I0Itec.zkclient.{ZkClient, ZkConnection} import kafka.controller.{ControllerStats, KafkaController} import kafka.cluster.{EndPoint, Broker} import kafka.common.{ErrorMapping, InconsistentBrokerIdException, GenerateBrokerIdException} @@ -129,6 +129,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime, threadNamePr val metadataCache: MetadataCache = new MetadataCache(config.brokerId) var zkClient: ZkClient = null + var zkConnection: ZkConnection = null val correlationId: AtomicInteger = new AtomicInteger(0) val brokerMetaPropsFile = "meta.properties" val brokerMetadataCheckpoints = config.logDirs.map(logDir => (logDir, new BrokerMetadataCheckpoint(new File(logDir + File.separator +brokerMetaPropsFile)))).toMap @@ -164,7 +165,9 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime, threadNamePr kafkaScheduler.startup() /* setup zookeeper */ - zkClient = initZk() + val (client, connection) = initZk() + zkClient = client + zkConnection = connection /* start log manager */ logManager = createLogManager(zkClient, brokerState) @@ -183,7 +186,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime, threadNamePr replicaManager.startup() /* start kafka controller */ - kafkaController = new KafkaController(config, zkClient, brokerState, kafkaMetricsTime, metrics, threadNamePrefix) + kafkaController = new KafkaController(config, zkClient, zkConnection, brokerState, kafkaMetricsTime, metrics, threadNamePrefix) kafkaController.startup() /* start kafka coordinator */ @@ -220,7 +223,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime, threadNamePr else (protocol, endpoint) } - kafkaHealthcheck = new KafkaHealthcheck(config.brokerId, listeners, config.zkSessionTimeoutMs, zkClient) + kafkaHealthcheck = new KafkaHealthcheck(config.brokerId, listeners, zkClient, zkConnection) kafkaHealthcheck.startup() /* register broker metrics */ @@ -242,7 +245,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime, threadNamePr } } - private def initZk(): ZkClient = { + private def initZk(): (ZkClient, ZkConnection) = { info("Connecting to zookeeper on " + config.zkConnect) val chroot = { @@ -260,9 +263,9 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime, threadNamePr zkClientForChrootCreation.close() } - val zkClient = ZkUtils.createZkClient(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + val (zkClient, zkConnection) = ZkUtils.createZkClientAndConnection(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) ZkUtils.setupCommonPaths(zkClient) - zkClient + (zkClient, zkConnection) } @@ -478,9 +481,9 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime, threadNamePr brokerState.newState(PendingControlledShutdown) val shutdownSucceeded = - // Before 0.8.3, `ControlledShutdownRequest` did not contain `client_id` and it's a mandatory field in + // Before 0.9.0.0, `ControlledShutdownRequest` did not contain `client_id` and it's a mandatory field in // `RequestHeader`, which is used by `NetworkClient` - if (config.interBrokerProtocolVersion.onOrAfter(KAFKA_083)) + if (config.interBrokerProtocolVersion.onOrAfter(KAFKA_090)) networkClientControlledShutdown(config.controlledShutdownMaxRetries.intValue) else blockingChannelControlledShutdown(config.controlledShutdownMaxRetries.intValue) diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala index 6c85e52e172bf..12698c7bb50db 100644 --- a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala +++ b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala @@ -23,7 +23,7 @@ import kafka.admin.AdminUtils import kafka.cluster.BrokerEndPoint import kafka.log.LogConfig import kafka.message.ByteBufferMessageSet -import kafka.api.KAFKA_083 +import kafka.api.KAFKA_090 import kafka.common.{KafkaStorageException, TopicAndPartition} import ReplicaFetcherThread._ import org.apache.kafka.clients.{ManualMetadataUpdater, NetworkClient, ClientRequest, ClientResponse} @@ -54,7 +54,7 @@ class ReplicaFetcherThread(name: String, type REQ = FetchRequest type PD = PartitionData - private val fetchRequestVersion: Short = if (brokerConfig.interBrokerProtocolVersion.onOrAfter(KAFKA_083)) 1 else 0 + private val fetchRequestVersion: Short = if (brokerConfig.interBrokerProtocolVersion.onOrAfter(KAFKA_090)) 1 else 0 private val socketTimeout: Int = brokerConfig.replicaSocketTimeoutMs private val replicaId = brokerConfig.brokerId private val maxWait = brokerConfig.replicaFetchWaitMaxMs diff --git a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala index 2b36b2de3e7ea..b283e0a37b410 100755 --- a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala +++ b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala @@ -18,7 +18,7 @@ package kafka.server import kafka.utils.ZkUtils._ import kafka.utils.CoreUtils._ -import kafka.utils.{Json, SystemTime, Logging} +import kafka.utils.{Json, SystemTime, Logging, ZKCheckedEphemeral} import org.I0Itec.zkclient.exception.ZkNodeExistsException import org.I0Itec.zkclient.IZkDataListener import kafka.controller.ControllerContext @@ -56,7 +56,7 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, case None => -1 } } - + def elect: Boolean = { val timestamp = SystemTime.milliseconds.toString val electString = Json.encode(Map("version" -> 1, "brokerid" -> brokerId, "timestamp" -> timestamp)) @@ -73,9 +73,10 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, } try { - createEphemeralPathExpectConflictHandleZKBug(controllerContext.zkClient, electionPath, electString, brokerId, - (controllerString : String, leaderId : Any) => KafkaController.parseControllerId(controllerString) == leaderId.asInstanceOf[Int], - controllerContext.zkSessionTimeout) + val zkCheckedEphemeral = new ZKCheckedEphemeral(electionPath, + electString, + controllerContext.zkConnection.getZookeeper) + zkCheckedEphemeral.create() info(brokerId + " successfully elected as leader") leaderId = brokerId onBecomingLeader() diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 74b587e04cdd6..e1cfa2e795964 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -17,11 +17,12 @@ package kafka.utils +import java.util.concurrent.CountDownLatch import kafka.cluster._ import kafka.consumer.{ConsumerThreadId, TopicCount} import kafka.server.ConfigType -import org.I0Itec.zkclient.ZkClient -import org.I0Itec.zkclient.exception.{ZkNodeExistsException, ZkNoNodeException, +import org.I0Itec.zkclient.{ZkClient,ZkConnection} +import org.I0Itec.zkclient.exception.{ZkException, ZkNodeExistsException, ZkNoNodeException, ZkMarshallingError, ZkBadVersionException} import org.I0Itec.zkclient.serialize.ZkSerializer import org.apache.kafka.common.config.ConfigException @@ -36,6 +37,14 @@ import kafka.controller.KafkaController import kafka.controller.LeaderIsrAndControllerEpoch import kafka.common.TopicAndPartition +import org.apache.zookeeper.AsyncCallback.{DataCallback,StringCallback} +import org.apache.zookeeper.CreateMode +import org.apache.zookeeper.KeeperException +import org.apache.zookeeper.KeeperException.Code +import org.apache.zookeeper.ZooDefs.Ids +import org.apache.zookeeper.ZooKeeper + + object ZkUtils extends Logging { val ConsumersPath = "/consumers" val BrokerIdsPath = "/brokers/ids" @@ -195,24 +204,22 @@ object ZkUtils extends Logging { * @param timeout * @param jmxPort */ - def registerBrokerInZk(zkClient: ZkClient, id: Int, host: String, port: Int, advertisedEndpoints: immutable.Map[SecurityProtocol, EndPoint], timeout: Int, jmxPort: Int) { + def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, id: Int, host: String, port: Int, advertisedEndpoints: immutable.Map[SecurityProtocol, EndPoint], jmxPort: Int) { val brokerIdPath = ZkUtils.BrokerIdsPath + "/" + id val timestamp = SystemTime.milliseconds.toString val brokerInfo = Json.encode(Map("version" -> 2, "host" -> host, "port" -> port, "endpoints"->advertisedEndpoints.values.map(_.connectionString).toArray, "jmx_port" -> jmxPort, "timestamp" -> timestamp)) - val expectedBroker = new Broker(id, advertisedEndpoints) - - registerBrokerInZk(zkClient, brokerIdPath, brokerInfo, expectedBroker, timeout) + registerBrokerInZk(zkClient, zkConnection, brokerIdPath, brokerInfo) info("Registered broker %d at path %s with addresses: %s".format(id, brokerIdPath, advertisedEndpoints.mkString(","))) } - private def registerBrokerInZk(zkClient: ZkClient, brokerIdPath: String, brokerInfo: String, expectedBroker: Broker, timeout: Int) { + private def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, brokerIdPath: String, brokerInfo: String) { try { - createEphemeralPathExpectConflictHandleZKBug(zkClient, brokerIdPath, brokerInfo, expectedBroker, - (brokerString: String, broker: Any) => Broker.createBroker(broker.asInstanceOf[Broker].id, brokerString).equals(broker.asInstanceOf[Broker]), - timeout) - + val zkCheckedEphemeral = new ZKCheckedEphemeral(brokerIdPath, + brokerInfo, + zkConnection.getZookeeper) + zkCheckedEphemeral.create() } catch { case e: ZkNodeExistsException => throw new RuntimeException("A broker is already registered on the path " + brokerIdPath @@ -301,47 +308,6 @@ object ZkUtils extends Logging { } } - /** - * Create an ephemeral node with the given path and data. - * Throw NodeExistsException if node already exists. - * Handles the following ZK session timeout bug: - * - * https://issues.apache.org/jira/browse/ZOOKEEPER-1740 - * - * Upon receiving a NodeExistsException, read the data from the conflicted path and - * trigger the checker function comparing the read data and the expected data, - * If the checker function returns true then the above bug might be encountered, back off and retry; - * otherwise re-throw the exception - */ - def createEphemeralPathExpectConflictHandleZKBug(zkClient: ZkClient, path: String, data: String, expectedCallerData: Any, checker: (String, Any) => Boolean, backoffTime: Int): Unit = { - while (true) { - try { - createEphemeralPathExpectConflict(zkClient, path, data) - return - } catch { - case e: ZkNodeExistsException => { - // An ephemeral node may still exist even after its corresponding session has expired - // due to a Zookeeper bug, in this case we need to retry writing until the previous node is deleted - // and hence the write succeeds without ZkNodeExistsException - ZkUtils.readDataMaybeNull(zkClient, path)._1 match { - case Some(writtenData) => { - if (checker(writtenData, expectedCallerData)) { - info("I wrote this conflicted ephemeral node [%s] at %s a while back in a different session, ".format(data, path) - + "hence I will backoff for this node to be deleted by Zookeeper and retry") - - Thread.sleep(backoffTime) - } else { - throw e - } - } - case None => // the node disappeared; retry creating the ephemeral node immediately - } - } - case e2: Throwable => throw e2 - } - } - } - /** * Create an persistent node with the given path and data. Create parents if necessary. */ @@ -809,7 +775,13 @@ object ZkUtils extends Logging { def createZkClient(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): ZkClient = { val zkClient = new ZkClient(zkUrl, sessionTimeout, connectionTimeout, ZKStringSerializer) - zkClient + zkClient + } + + def createZkClientAndConnection(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): (ZkClient, ZkConnection) = { + val zkConnection = new ZkConnection(zkUrl, sessionTimeout) + val zkClient = new ZkClient(zkConnection, connectionTimeout, ZKStringSerializer) + (zkClient, zkConnection) } } @@ -892,3 +864,157 @@ object ZkPath { client.createPersistentSequential(path, data) } } + +/** + * Creates an ephemeral znode checking the session owner + * in the case of conflict. In the regular case, the + * znode is created and the create call returns OK. If + * the call receives a node exists event, then it checks + * if the session matches. If it does, then it returns OK, + * and otherwise it fails the operation. + */ + +class ZKCheckedEphemeral(path: String, + data: String, + zkHandle: ZooKeeper) extends Logging { + private val createCallback = new CreateCallback + private val getDataCallback = new GetDataCallback + val latch: CountDownLatch = new CountDownLatch(1) + var result: Code = Code.OK + + private class CreateCallback extends StringCallback { + def processResult(rc: Int, + path: String, + ctx: Object, + name: String) { + Code.get(rc) match { + case Code.OK => + setResult(Code.OK) + case Code.CONNECTIONLOSS => + // try again + createEphemeral + case Code.NONODE => + error("No node for path %s (could be the parent missing)".format(path)) + setResult(Code.NONODE) + case Code.NODEEXISTS => + zkHandle.getData(path, false, getDataCallback, null) + case Code.SESSIONEXPIRED => + error("Session has expired while creating %s".format(path)) + setResult(Code.SESSIONEXPIRED) + case _ => + warn("ZooKeeper event while creating registration node: %s %s".format(path, Code.get(rc))) + setResult(Code.get(rc)) + } + } + } + + private class GetDataCallback extends DataCallback { + def processResult(rc: Int, + path: String, + ctx: Object, + readData: Array[Byte], + stat: Stat) { + Code.get(rc) match { + case Code.OK => + if (stat.getEphemeralOwner != zkHandle.getSessionId) + setResult(Code.NODEEXISTS) + else + setResult(Code.OK) + case Code.NONODE => + info("The ephemeral node [%s] at %s has gone away while reading it, ".format(data, path)) + createEphemeral + case Code.SESSIONEXPIRED => + error("Session has expired while reading znode %s".format(path)) + setResult(Code.SESSIONEXPIRED) + case _ => + warn("ZooKeeper event while getting znode data: %s %s".format(path, Code.get(rc))) + setResult(Code.get(rc)) + } + } + } + + private def createEphemeral() { + zkHandle.create(path, + ZKStringSerializer.serialize(data), + Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL, + createCallback, + null) + } + + private def createRecursive(prefix: String, suffix: String) { + debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + if(suffix.isEmpty()) { + createEphemeral + } else { + zkHandle.create(prefix, + new Array[Byte](0), + Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT, + new StringCallback() { + def processResult(rc : Int, + path : String, + ctx : Object, + name : String) { + Code.get(rc) match { + case Code.OK | Code.NODEEXISTS => + // Nothing to do + case Code.CONNECTIONLOSS => + // try again + val suffix = ctx.asInstanceOf[String] + createRecursive(path, suffix) + case Code.NONODE => + error("No node for path %s (could be the parent missing)".format(path)) + setResult(Code.get(rc)) + case Code.SESSIONEXPIRED => + error("Session has expired while creating %s".format(path)) + setResult(Code.get(rc)) + case _ => + warn("ZooKeeper event while creating registration node: %s %s".format(path, Code.get(rc))) + setResult(Code.get(rc)) + } + } + }, + suffix) + // Update prefix and suffix + val index = suffix.indexOf('/', 1) match { + case -1 => suffix.length + case x : Int => x + } + // Get new prefix + val newPrefix = prefix + suffix.substring(0, index) + // Get new suffix + val newSuffix = suffix.substring(index, suffix.length) + createRecursive(newPrefix, newSuffix) + } + } + + private def setResult(code: Code) { + result = code + latch.countDown() + } + + private def waitUntilResolved(): Code = { + latch.await() + result + } + + def create() { + val index = path.indexOf('/', 1) match { + case -1 => path.length + case x : Int => x + } + val prefix = path.substring(0, index) + val suffix = path.substring(index, path.length) + debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + createRecursive(prefix, suffix) + val result = waitUntilResolved() + info("Result of znode creation is: %s".format(result)) + result match { + case Code.OK => + // Nothing to do + case _ => + throw ZkException.create(KeeperException.create(result)) + } + } +} diff --git a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala index 28a3e90da7a43..3f17e796aac3e 100644 --- a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala +++ b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala @@ -87,7 +87,7 @@ class ConsumerBounceTest extends IntegrationTestHarness with Logging { } consumer.commitSync() - assertEquals(consumer.position(tp), consumer.committed(tp)) + assertEquals(consumer.position(tp), consumer.committed(tp).offset) if (consumer.position(tp) == numRecords) { consumer.seekToBeginning() @@ -131,7 +131,7 @@ class ConsumerBounceTest extends IntegrationTestHarness with Logging { } else if (coin == 2) { info("Committing offset.") consumer.commitSync() - assertEquals(consumer.position(tp), consumer.committed(tp)) + assertEquals(consumer.position(tp), consumer.committed(tp).offset) } } } diff --git a/core/src/test/scala/integration/kafka/api/ConsumerTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerTest.scala index 1b1973f9f554e..af81a8330b417 100644 --- a/core/src/test/scala/integration/kafka/api/ConsumerTest.scala +++ b/core/src/test/scala/integration/kafka/api/ConsumerTest.scala @@ -186,23 +186,22 @@ class ConsumerTest extends IntegrationTestHarness with Logging { this.consumers(0).poll(50) val pos1 = this.consumers(0).position(tp) val pos2 = this.consumers(0).position(tp2) - this.consumers(0).commitSync(Map[TopicPartition,java.lang.Long]((tp, 3L)).asJava) - assertEquals(3, this.consumers(0).committed(tp)) - intercept[NoOffsetForPartitionException] { - this.consumers(0).committed(tp2) - } + this.consumers(0).commitSync(Map[TopicPartition,OffsetAndMetadata]((tp, new OffsetAndMetadata(3L))).asJava) + assertEquals(3, this.consumers(0).committed(tp).offset) + assertNull(this.consumers(0).committed(tp2)) + // positions should not change assertEquals(pos1, this.consumers(0).position(tp)) assertEquals(pos2, this.consumers(0).position(tp2)) - this.consumers(0).commitSync(Map[TopicPartition,java.lang.Long]((tp2, 5L)).asJava) - assertEquals(3, this.consumers(0).committed(tp)) - assertEquals(5, this.consumers(0).committed(tp2)) + this.consumers(0).commitSync(Map[TopicPartition,OffsetAndMetadata]((tp2, new OffsetAndMetadata(5L))).asJava) + assertEquals(3, this.consumers(0).committed(tp).offset) + assertEquals(5, this.consumers(0).committed(tp2).offset) // Using async should pick up the committed changes after commit completes val commitCallback = new CountConsumerCommitCallback() - this.consumers(0).commitAsync(Map[TopicPartition,java.lang.Long]((tp2, 7L)).asJava, commitCallback) + this.consumers(0).commitAsync(Map[TopicPartition,OffsetAndMetadata]((tp2, new OffsetAndMetadata(7L))).asJava, commitCallback) awaitCommitCallback(this.consumers(0), commitCallback) - assertEquals(7, this.consumers(0).committed(tp2)) + assertEquals(7, this.consumers(0).committed(tp2).offset) } @Test @@ -240,7 +239,25 @@ class ConsumerTest extends IntegrationTestHarness with Logging { consumeRecords(this.consumers(0), numRecords = 1, startingOffset = 0) } + @Test + def testCommitMetadata() { + this.consumers(0).assign(List(tp)) + + // sync commit + val syncMetadata = new OffsetAndMetadata(5, "foo") + this.consumers(0).commitSync(Map((tp, syncMetadata))) + assertEquals(syncMetadata, this.consumers(0).committed(tp)) + + // async commit + val asyncMetadata = new OffsetAndMetadata(10, "bar") + val callback = new CountConsumerCommitCallback + this.consumers(0).commitAsync(Map((tp, asyncMetadata)), callback) + awaitCommitCallback(this.consumers(0), callback) + + assertEquals(asyncMetadata, this.consumers(0).committed(tp)) + } + def testPositionAndCommit() { sendRecords(5) @@ -258,12 +275,12 @@ class ConsumerTest extends IntegrationTestHarness with Logging { assertEquals("position() on a partition that we are subscribed to should reset the offset", 0L, this.consumers(0).position(tp)) this.consumers(0).commitSync() - assertEquals(0L, this.consumers(0).committed(tp)) + assertEquals(0L, this.consumers(0).committed(tp).offset) consumeRecords(this.consumers(0), 5, 0) assertEquals("After consuming 5 records, position should be 5", 5L, this.consumers(0).position(tp)) this.consumers(0).commitSync() - assertEquals("Committed offset should be returned", 5L, this.consumers(0).committed(tp)) + assertEquals("Committed offset should be returned", 5L, this.consumers(0).committed(tp).offset) sendRecords(1) @@ -473,14 +490,14 @@ class ConsumerTest extends IntegrationTestHarness with Logging { val startCount = commitCallback.count val started = System.currentTimeMillis() while (commitCallback.count == startCount && System.currentTimeMillis() - started < 10000) - this.consumers(0).poll(10000) + this.consumers(0).poll(50) assertEquals(startCount + 1, commitCallback.count) } private class CountConsumerCommitCallback extends OffsetCommitCallback { var count = 0 - override def onComplete(offsets: util.Map[TopicPartition, lang.Long], exception: Exception): Unit = count += 1 + override def onComplete(offsets: util.Map[TopicPartition, OffsetAndMetadata], exception: Exception): Unit = count += 1 } } \ No newline at end of file diff --git a/core/src/test/scala/integration/kafka/api/SSLConsumerTest.scala b/core/src/test/scala/integration/kafka/api/SSLConsumerTest.scala index df9b3d8dad696..b2fc0575bf962 100644 --- a/core/src/test/scala/integration/kafka/api/SSLConsumerTest.scala +++ b/core/src/test/scala/integration/kafka/api/SSLConsumerTest.scala @@ -169,10 +169,8 @@ class SSLConsumerTest extends KafkaServerTestHarness with Logging { def testPositionAndCommit() { sendRecords(5) - // committed() on a partition with no committed offset throws an exception - intercept[NoOffsetForPartitionException] { - this.consumers(0).committed(new TopicPartition(topic, 15)) - } + // committed() on a partition with no committed offset returns null + assertNull(this.consumers(0).committed(new TopicPartition(topic, 15))) // position() on a partition that we aren't subscribed to throws an exception intercept[IllegalArgumentException] { @@ -183,12 +181,12 @@ class SSLConsumerTest extends KafkaServerTestHarness with Logging { assertEquals("position() on a partition that we are subscribed to should reset the offset", 0L, this.consumers(0).position(tp)) this.consumers(0).commitSync() - assertEquals(0L, this.consumers(0).committed(tp)) + assertEquals(0L, this.consumers(0).committed(tp).offset) consumeRecords(this.consumers(0), 5, 0) assertEquals("After consuming 5 records, position should be 5", 5L, this.consumers(0).position(tp)) this.consumers(0).commitSync() - assertEquals("Committed offset should be returned", 5L, this.consumers(0).committed(tp)) + assertEquals("Committed offset should be returned", 5L, this.consumers(0).committed(tp).offset) sendRecords(1) diff --git a/core/src/test/scala/unit/kafka/admin/AdminTest.scala b/core/src/test/scala/unit/kafka/admin/AdminTest.scala index 9bd8171f484c1..2d18069884d18 100755 --- a/core/src/test/scala/unit/kafka/admin/AdminTest.scala +++ b/core/src/test/scala/unit/kafka/admin/AdminTest.scala @@ -66,7 +66,7 @@ class AdminTest extends ZooKeeperTestHarness with Logging { @Test def testManualReplicaAssignment() { val brokers = List(0, 1, 2, 3, 4) - TestUtils.createBrokersInZk(zkClient, brokers) + TestUtils.createBrokersInZk(zkClient, zkConnection, brokers) // duplicate brokers intercept[IllegalArgumentException] { @@ -117,7 +117,7 @@ class AdminTest extends ZooKeeperTestHarness with Logging { 11 -> 1 ) val topic = "test" - TestUtils.createBrokersInZk(zkClient, List(0, 1, 2, 3, 4)) + TestUtils.createBrokersInZk(zkClient, zkConnection, List(0, 1, 2, 3, 4)) // create the topic AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK(zkClient, topic, expectedReplicaAssignment) // create leaders for all partitions @@ -137,7 +137,7 @@ class AdminTest extends ZooKeeperTestHarness with Logging { def testTopicCreationWithCollision() { val topic = "test.topic" val collidingTopic = "test_topic" - TestUtils.createBrokersInZk(zkClient, List(0, 1, 2, 3, 4)) + TestUtils.createBrokersInZk(zkClient, zkConnection, List(0, 1, 2, 3, 4)) // create the topic AdminUtils.createTopic(zkClient, topic, 3, 1) diff --git a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala index 9bfec72261eaf..d4fa0d5c1b58a 100644 --- a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala @@ -36,7 +36,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging { val cleanupVal = "compact" // create brokers val brokers = List(0, 1, 2) - TestUtils.createBrokersInZk(zkClient, brokers) + TestUtils.createBrokersInZk(zkClient, zkConnection, brokers) // create the topic val createOpts = new TopicCommandOptions(Array("--partitions", numPartitionsOriginal.toString, "--replication-factor", "1", @@ -67,7 +67,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging { // create brokers val brokers = List(0, 1, 2) - TestUtils.createBrokersInZk(zkClient, brokers) + TestUtils.createBrokersInZk(zkClient, zkConnection, brokers) // create the NormalTopic val createOpts = new TopicCommandOptions(Array("--partitions", numPartitionsOriginal.toString, diff --git a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala index ff1783096ef88..ac347ef307663 100755 --- a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala +++ b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala @@ -136,7 +136,7 @@ class LeaderElectionTest extends ZooKeeperTestHarness { new LeaderAndIsrRequest.EndPoint(brokerEndPoint.id, brokerEndPoint.host, brokerEndPoint.port) } - val controllerContext = new ControllerContext(zkClient, 6000) + val controllerContext = new ControllerContext(zkClient, zkConnection, 6000) controllerContext.liveBrokers = brokers.toSet val controllerChannelManager = new ControllerChannelManager(controllerContext, controllerConfig, new SystemTime, new Metrics) controllerChannelManager.startup() diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index b01adc860b416..7f482fb61adcf 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -31,7 +31,7 @@ import org.apache.kafka.common.utils.Utils._ import collection.mutable.ListBuffer -import org.I0Itec.zkclient.ZkClient +import org.I0Itec.zkclient.{ZkClient, ZkConnection} import kafka.server._ import kafka.producer._ @@ -500,9 +500,9 @@ object TestUtils extends Logging { } } - def createBrokersInZk(zkClient: ZkClient, ids: Seq[Int]): Seq[Broker] = { + def createBrokersInZk(zkClient: ZkClient, zkConnection: ZkConnection, ids: Seq[Int]): Seq[Broker] = { val brokers = ids.map(id => new Broker(id, "localhost", 6667, SecurityProtocol.PLAINTEXT)) - brokers.foreach(b => ZkUtils.registerBrokerInZk(zkClient, b.id, "localhost", 6667, b.endPoints, 6000, jmxPort = -1)) + brokers.foreach(b => ZkUtils.registerBrokerInZk(zkClient, zkConnection, b.id, "localhost", 6667, b.endPoints, jmxPort = -1)) brokers } diff --git a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala index f240e89b67d3f..2bf658c5f244c 100644 --- a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala +++ b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala @@ -19,7 +19,13 @@ package kafka.zk import kafka.consumer.ConsumerConfig import kafka.utils.ZkUtils +import kafka.utils.ZKCheckedEphemeral import kafka.utils.TestUtils +import org.apache.zookeeper.CreateMode +import org.apache.zookeeper.WatchedEvent +import org.apache.zookeeper.Watcher +import org.apache.zookeeper.ZooDefs.Ids +import org.I0Itec.zkclient.exception.{ZkException,ZkNodeExistsException} import org.junit.{Test, Assert} class ZKEphemeralTest extends ZooKeeperTestHarness { @@ -44,4 +50,96 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { val nodeExists = ZkUtils.pathExists(zkClient, "/tmp/zktest") Assert.assertFalse(nodeExists) } + + /***** + ***** Tests for ZkWatchedEphemeral + *****/ + + /** + * Tests basic creation + */ + @Test + def testZkWatchedEphemeral = { + val path = "/zwe-test" + testCreation(path) + } + + /** + * Tests recursive creation + */ + @Test + def testZkWatchedEphemeralRecursive = { + val path = "/zwe-test-parent/zwe-test" + testCreation(path) + } + + private def testCreation(path: String) { + val zk = zkConnection.getZookeeper + val zwe = new ZKCheckedEphemeral(path, "", zk) + var created = false + var counter = 10 + + zk.exists(path, new Watcher() { + def process(event: WatchedEvent) { + if(event.getType == Watcher.Event.EventType.NodeCreated) { + created = true + } + } + }) + zwe.create() + // Waits until the znode is created + TestUtils.waitUntilTrue(() => ZkUtils.pathExists(zkClient, path), + "Znode %s wasn't created".format(path)) + } + + /** + * Tests that it fails in the presence of an overlapping + * session. + */ + @Test + def testOverlappingSessions = { + val path = "/zwe-test" + val zk1 = zkConnection.getZookeeper + + //Creates a second session + val (_, zkConnection2) = ZkUtils.createZkClientAndConnection(zkConnect, zkSessionTimeoutMs, zkConnectionTimeout) + val zk2 = zkConnection2.getZookeeper + var zwe = new ZKCheckedEphemeral(path, "", zk2) + + // Creates znode for path in the first session + zk1.create(path, Array[Byte](), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL) + + //Bootstraps the ZKWatchedEphemeral object + var gotException = false; + try { + zwe.create() + } catch { + case e: ZkNodeExistsException => + gotException = true + } + Assert.assertTrue(gotException) + } + + /** + * Tests if succeeds with znode from the same session + * + */ + @Test + def testSameSession = { + val path = "/zwe-test" + val zk = zkConnection.getZookeeper + // Creates znode for path in the first session + zk.create(path, Array[Byte](), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL) + + var zwe = new ZKCheckedEphemeral(path, "", zk) + //Bootstraps the ZKWatchedEphemeral object + var gotException = false; + try { + zwe.create() + } catch { + case e: ZkNodeExistsException => + gotException = true + } + Assert.assertFalse(gotException) + } } diff --git a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala index e4bfb48c2602d..3e1c6e0ceecc1 100755 --- a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala +++ b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala @@ -17,7 +17,7 @@ package kafka.zk -import org.I0Itec.zkclient.ZkClient +import org.I0Itec.zkclient.{ZkClient, ZkConnection} import kafka.utils.{ZkUtils, CoreUtils} import org.junit.{After, Before} import org.scalatest.junit.JUnitSuite @@ -26,6 +26,7 @@ trait ZooKeeperTestHarness extends JUnitSuite { var zkPort: Int = -1 var zookeeper: EmbeddedZookeeper = null var zkClient: ZkClient = null + var zkConnection : ZkConnection = null val zkConnectionTimeout = 6000 val zkSessionTimeout = 6000 @@ -35,7 +36,9 @@ trait ZooKeeperTestHarness extends JUnitSuite { def setUp() { zookeeper = new EmbeddedZookeeper() zkPort = zookeeper.port - zkClient = ZkUtils.createZkClient(zkConnect, zkSessionTimeout, zkConnectionTimeout) + val (client, connection) = ZkUtils.createZkClientAndConnection(zkConnect, zkSessionTimeout, zkConnectionTimeout) + zkClient = client + zkConnection = connection } @After diff --git a/gradle.properties b/gradle.properties index 6f18a50902ea9..faeaf885efed0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,7 +14,7 @@ # limitations under the License. group=org.apache.kafka -version=0.8.3-SNAPSHOT +version=0.9.0.0-SNAPSHOT scalaVersion=2.10.5 task=build org.gradle.jvmargs=-XX:MaxPermSize=512m -Xmx1024m -Xss2m diff --git a/kafka-merge-pr.py b/kafka-merge-pr.py index 7f48bf97730d8..078708e0590a1 100644 --- a/kafka-merge-pr.py +++ b/kafka-merge-pr.py @@ -65,7 +65,9 @@ # TODO Introduce a convention as this is too brittle RELEASE_BRANCH_PREFIX = "0." -DEV_BRANCH_NAME="trunk" +DEV_BRANCH_NAME = "trunk" + +DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "0.9.0.0") def get_json(url): try: @@ -238,9 +240,17 @@ def cherry_pick(pr_num, merge_hash, default_branch): def fix_version_from_branch(branch, versions): # Note: Assumes this is a sorted (newest->oldest) list of un-released versions if branch == DEV_BRANCH_NAME: - return versions[0] + versions = filter(lambda x: x == DEFAULT_FIX_VERSION, versions) + if len(versions) > 0: + return versions[0] + else: + return None else: - return filter(lambda x: x.name.startswith(branch), versions)[-1] + versions = filter(lambda x: x.startswith(branch), versions) + if len(versions) > 0: + return versions[-1] + else: + return None def resolve_jira_issue(merge_branches, comment, default_jira_id=""): @@ -273,20 +283,10 @@ def resolve_jira_issue(merge_branches, comment, default_jira_id=""): versions = asf_jira.project_versions(CAPITALIZED_PROJECT_NAME) versions = sorted(versions, key=lambda x: x.name, reverse=True) versions = filter(lambda x: x.raw['released'] is False, versions) - # Consider only x.y.z versions - versions = filter(lambda x: re.match('\d+\.\d+\.\d+', x.name), versions) - - default_fix_versions = map(lambda x: fix_version_from_branch(x, versions).name, merge_branches) - for v in default_fix_versions: - # Handles the case where we have forked a release branch but not yet made the release. - # In this case, if the PR is committed to the master branch and the release branch, we - # only consider the release branch to be the fix version. E.g. it is not valid to have - # both 1.1.0 and 1.0.0 as fix versions. - (major, minor, patch) = v.split(".") - if patch == "0": - previous = "%s.%s.%s" % (major, int(minor) - 1, 0) - if previous in default_fix_versions: - default_fix_versions = filter(lambda x: x != v, default_fix_versions) + + version_names = map(lambda x: x.name, versions) + default_fix_versions = map(lambda x: fix_version_from_branch(x, version_names), merge_branches) + default_fix_versions = filter(lambda x: x != None, default_fix_versions) default_fix_versions = ",".join(default_fix_versions) fix_versions = raw_input("Enter comma-separated fix version(s) [%s]: " % default_fix_versions) diff --git a/log4j-appender/src/main/java/org/apache/kafka/log4jappender/KafkaLog4jAppender.java b/log4j-appender/src/main/java/org/apache/kafka/log4jappender/KafkaLog4jAppender.java index 628ff53ff70b8..2baef06e1a8e3 100644 --- a/log4j-appender/src/main/java/org/apache/kafka/log4jappender/KafkaLog4jAppender.java +++ b/log4j-appender/src/main/java/org/apache/kafka/log4jappender/KafkaLog4jAppender.java @@ -117,7 +117,7 @@ public void activateOptions() { if (compressionType != null) props.put(COMPRESSION_TYPE_CONFIG, compressionType); if (requiredNumAcks != Integer.MAX_VALUE) - props.put(ACKS_CONFIG, requiredNumAcks); + props.put(ACKS_CONFIG, Integer.toString(requiredNumAcks)); if (retries > 0) props.put(RETRIES_CONFIG, retries); diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java index 97dbb3bf89387..2f1fb35894170 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessorStateManager.java @@ -143,6 +143,7 @@ public void register(StateStore store, RestoreFunc restoreFunc) { if (checkpointedOffsets.containsKey(storePartition)) { restoreConsumer.seek(storePartition, checkpointedOffsets.get(storePartition)); } else { + // TODO: in this case, we need to ignore the preciously flushed state restoreConsumer.seekToBeginning(storePartition); } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java index 6ff60b46f99b3..40fb723577175 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java @@ -19,6 +19,7 @@ import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.TopicPartition; @@ -250,7 +251,11 @@ public void commit() { // 3) commit consumed offsets if it is dirty already if (commitOffsetNeeded) { - consumer.commitSync(consumedOffsets); + Map consumedOffsetsAndMetadata = new HashMap<>(consumedOffsets.size()); + for (Map.Entry entry : consumedOffsets.entrySet()) { + consumedOffsetsAndMetadata.put(entry.getKey(), new OffsetAndMetadata(entry.getValue())); + } + consumer.commitSync(consumedOffsetsAndMetadata); commitOffsetNeeded = false; } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorStateManagerTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorStateManagerTest.java index c983c6b1a3a6e..343ed52b7ef54 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorStateManagerTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorStateManagerTest.java @@ -92,7 +92,7 @@ public void apply(byte[] key, byte[] value) { }; } - private static class MockRestoreConsumer extends MockConsumer { + private class MockRestoreConsumer extends MockConsumer { private final Serializer serializer = new IntegerSerializer(); public TopicPartition assignedPartition = null; @@ -100,32 +100,36 @@ private static class MockRestoreConsumer extends MockConsumer { public long seekOffset = -1L; public boolean seekToBeginingCalled = false; public boolean seekToEndCalled = false; - private long endOffset = -1L; - private long currentOffset = -1L; + private long endOffset = 0L; + private long currentOffset = 0L; private ArrayList> recordBuffer = new ArrayList<>(); MockRestoreConsumer() { super(OffsetResetStrategy.EARLIEST); + + reset(); } - // prepares this mock restore consumer for a state store registration - public void prepare() { + // reset this mock restore consumer for a state store registration + public void reset() { assignedPartition = null; seekOffset = -1L; seekToBeginingCalled = false; seekToEndCalled = false; - endOffset = -1L; + endOffset = 0L; recordBuffer.clear(); } // buffer a record (we cannot use addRecord because we need to add records before asigning a partition) public void bufferRecord(ConsumerRecord record) { recordBuffer.add( - new ConsumerRecord<>(record.topic(), record.partition(), record.offset(), - serializer.serialize(record.topic(), record.key()), - serializer.serialize(record.topic(), record.value()))); + new ConsumerRecord<>(record.topic(), record.partition(), record.offset(), + serializer.serialize(record.topic(), record.key()), + serializer.serialize(record.topic(), record.value()))); endOffset = record.offset(); + + super.updateEndOffsets(Collections.singletonMap(assignedPartition, endOffset)); } @Override @@ -138,6 +142,10 @@ public synchronized void assign(List partitions) { if (assignedPartition != null) throw new IllegalStateException("RestoreConsumer: partition already assigned"); assignedPartition = partitions.get(0); + + // set the beginning offset to 0 + // NOTE: this is users responsible to set the initial lEO. + super.updateBeginningOffsets(Collections.singletonMap(assignedPartition, 0L)); } super.assign(partitions); @@ -186,16 +194,27 @@ public synchronized void seek(TopicPartition partition, long offset) { @Override public synchronized void seekToBeginning(TopicPartition... partitions) { - if (seekToBeginingCalled) - throw new IllegalStateException("RestoreConsumer: offset already seeked to beginning"); + if (partitions.length != 1) + throw new IllegalStateException("RestoreConsumer: other than one partition specified"); + + for (TopicPartition partition : partitions) { + if (!partition.equals(assignedPartition)) + throw new IllegalStateException("RestoreConsumer: seek-to-end not on the assigned partition"); + } seekToBeginingCalled = true; + currentOffset = 0L; } @Override public synchronized void seekToEnd(TopicPartition... partitions) { - if (seekToEndCalled) - throw new IllegalStateException("RestoreConsumer: offset already seeked to beginning"); + if (partitions.length != 1) + throw new IllegalStateException("RestoreConsumer: other than one partition specified"); + + for (TopicPartition partition : partitions) { + if (!partition.equals(assignedPartition)) + throw new IllegalStateException("RestoreConsumer: seek-to-end not on the assigned partition"); + } seekToEndCalled = true; currentOffset = endOffset; @@ -262,12 +281,13 @@ public void testRegisterPersistentStore() throws IOException { new PartitionInfo("persistentStore", 1, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("persistentStore", 2, Node.noNode(), new Node[0], new Node[0]) )); + restoreConsumer.updateEndOffsets(Collections.singletonMap(new TopicPartition("persistentStore", 2), 13L)); MockStateStore persistentStore = new MockStateStore("persistentStore", false); // non persistent store ProcessorStateManager stateMgr = new ProcessorStateManager(2, baseDir, restoreConsumer); try { - restoreConsumer.prepare(); + restoreConsumer.reset(); ArrayList expectedKeys = new ArrayList<>(); for (int i = 1; i <= 3; i++) { @@ -295,6 +315,7 @@ public void testRegisterPersistentStore() throws IOException { Utils.delete(baseDir); } } + @Test public void testRegisterNonPersistentStore() throws IOException { File baseDir = Files.createTempDirectory("test").toFile(); @@ -308,12 +329,13 @@ public void testRegisterNonPersistentStore() throws IOException { new PartitionInfo("nonPersistentStore", 1, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("nonPersistentStore", 2, Node.noNode(), new Node[0], new Node[0]) )); + restoreConsumer.updateEndOffsets(Collections.singletonMap(new TopicPartition("persistentStore", 2), 13L)); MockStateStore nonPersistentStore = new MockStateStore("nonPersistentStore", true); // persistent store ProcessorStateManager stateMgr = new ProcessorStateManager(2, baseDir, restoreConsumer); try { - restoreConsumer.prepare(); + restoreConsumer.reset(); ArrayList expectedKeys = new ArrayList<>(); for (int i = 1; i <= 3; i++) { @@ -328,7 +350,7 @@ public void testRegisterNonPersistentStore() throws IOException { stateMgr.register(nonPersistentStore, nonPersistentStore.restoreFunc); assertEquals(new TopicPartition("nonPersistentStore", 2), restoreConsumer.assignedPartition); - assertEquals(-1L, restoreConsumer.seekOffset); + assertEquals(0L, restoreConsumer.seekOffset); assertTrue(restoreConsumer.seekToBeginingCalled); assertTrue(restoreConsumer.seekToEndCalled); assertEquals(expectedKeys, nonPersistentStore.keys); @@ -397,10 +419,10 @@ public void testClose() throws IOException { // make sure the checkpoint file is deleted assertFalse(checkpointFile.exists()); - restoreConsumer.prepare(); + restoreConsumer.reset(); stateMgr.register(persistentStore, persistentStore.restoreFunc); - restoreConsumer.prepare(); + restoreConsumer.reset(); stateMgr.register(nonPersistentStore, nonPersistentStore.restoreFunc); } finally { // close the state manager with the ack'ed offsets diff --git a/streams/src/test/java/org/apache/kafka/test/ProcessorTopologyTestDriver.java b/streams/src/test/java/org/apache/kafka/test/ProcessorTopologyTestDriver.java index 88b7a54748885..75f8b4c1e4943 100644 --- a/streams/src/test/java/org/apache/kafka/test/ProcessorTopologyTestDriver.java +++ b/streams/src/test/java/org/apache/kafka/test/ProcessorTopologyTestDriver.java @@ -310,6 +310,7 @@ public synchronized long position(TopicPartition partition) { List partitionInfos = new ArrayList<>(); partitionInfos.add(new PartitionInfo(topicName, id, null, null, null)); consumer.updatePartitions(topicName, partitionInfos); + consumer.updateEndOffsets(Collections.singletonMap(new TopicPartition(topicName, id), 0L)); } return consumer; } diff --git a/tests/kafkatest/services/copycat.py b/tests/kafkatest/services/copycat.py index 7452b858fc319..ea5753bc5c500 100644 --- a/tests/kafkatest/services/copycat.py +++ b/tests/kafkatest/services/copycat.py @@ -18,8 +18,8 @@ import subprocess, signal -class CopycatStandaloneService(Service): - """Runs Copycat in standalone mode.""" +class CopycatServiceBase(Service): + """Base class for Copycat services providing some common settings and functionality""" logs = { "kafka_log": { @@ -27,16 +27,55 @@ class CopycatStandaloneService(Service): "collect_default": True}, } - def __init__(self, context, kafka, files): - super(CopycatStandaloneService, self).__init__(context, 1) + def __init__(self, context, num_nodes, kafka, files): + super(CopycatServiceBase, self).__init__(context, num_nodes) self.kafka = kafka self.files = files + def pids(self, node): + """Return process ids for Copycat processes.""" + try: + return [pid for pid in node.account.ssh_capture("cat /mnt/copycat.pid", callback=int)] + except: + return [] + + def stop_node(self, node, clean_shutdown=True): + pids = self.pids(node) + sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL + + for pid in pids: + node.account.signal(pid, sig, allow_fail=False) + for pid in pids: + wait_until(lambda: not node.account.alive(pid), timeout_sec=10, err_msg="Copycat standalone process took too long to exit") + + node.account.ssh("rm -f /mnt/copycat.pid", allow_fail=False) + + def restart(self): + # We don't want to do any clean up here, just restart the process + for node in self.nodes: + self.stop_node(node) + self.start_node(node) + + def clean_node(self, node): + if len(self.pids(node)) > 0: + self.logger.warn("%s %s was still alive at cleanup time. Killing forcefully..." % + (self.__class__.__name__, node.account)) + for pid in self.pids(node): + node.account.signal(pid, signal.SIGKILL, allow_fail=False) + node.account.ssh("rm -rf /mnt/copycat.pid /mnt/copycat.log /mnt/copycat.properties /mnt/copycat-connector.properties " + " ".join(self.files), allow_fail=False) + + +class CopycatStandaloneService(CopycatServiceBase): + """Runs Copycat in standalone mode.""" + + def __init__(self, context, kafka, files): + super(CopycatStandaloneService, self).__init__(context, 1, kafka, files) + def set_configs(self, config_template, connector_config_template): """ - Set configurations for the standalone worker and the connector to run on - it. These are not provided in the constructor because at least the - standalone worker config generally needs access to ZK/Kafka services to + Set configurations for the worker and the connector to run on + it. These are not provided in the constructor because the worker + config generally needs access to ZK/Kafka services to create the configuration. """ self.config_template = config_template @@ -47,9 +86,6 @@ def set_configs(self, config_template, connector_config_template): def node(self): return self.nodes[0] - def start(self): - super(CopycatStandaloneService, self).start() - def start_node(self, node): node.account.create_file("/mnt/copycat.properties", self.config_template) node.account.create_file("/mnt/copycat-connector.properties", self.connector_config_template) @@ -60,36 +96,38 @@ def start_node(self, node): "1>> /mnt/copycat.log 2>> /mnt/copycat.log & echo $! > /mnt/copycat.pid") monitor.wait_until('Copycat started', timeout_sec=10, err_msg="Never saw message indicating Copycat finished startup") - if len(self.pids()) == 0: + if len(self.pids(node)) == 0: raise RuntimeError("No process ids recorded") - def pids(self): - """Return process ids for Copycat processes.""" - try: - return [pid for pid in self.node.account.ssh_capture("cat /mnt/copycat.pid", callback=int)] - except: - return [] - def stop_node(self, node, clean_shutdown=True): - pids = self.pids() - sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL - for pid in pids: - self.node.account.signal(pid, sig, allow_fail=False) - for pid in pids: - wait_until(lambda: not self.node.account.alive(pid), timeout_sec=10, err_msg="Copycat standalone process took too long to exit") +class CopycatDistributedService(CopycatServiceBase): + """Runs Copycat in distributed mode.""" - node.account.ssh("rm -f /mnt/copycat.pid", allow_fail=False) + def __init__(self, context, num_nodes, kafka, files, offsets_topic="copycat-offsets"): + super(CopycatDistributedService, self).__init__(context, num_nodes, kafka, files) + self.offsets_topic = offsets_topic - def restart(self): - # We don't want to do any clean up here, just restart the process - self.stop_node(self.node) - self.start_node(self.node) + def set_configs(self, config_template, connector_config_template): + """ + Set configurations for the worker and the connector to run on + it. These are not provided in the constructor because the worker + config generally needs access to ZK/Kafka services to + create the configuration. + """ + self.config_template = config_template + self.connector_config_template = connector_config_template + + def start_node(self, node): + node.account.create_file("/mnt/copycat.properties", self.config_template) + node.account.create_file("/mnt/copycat-connector.properties", self.connector_config_template) + + self.logger.info("Starting Copycat standalone process") + with node.account.monitor_log("/mnt/copycat.log") as monitor: + node.account.ssh("/opt/kafka/bin/copycat-distributed.sh /mnt/copycat.properties /mnt/copycat-connector.properties " + + "1>> /mnt/copycat.log 2>> /mnt/copycat.log & echo $! > /mnt/copycat.pid") + monitor.wait_until('Copycat started', timeout_sec=10, err_msg="Never saw message indicating Copycat finished startup") + + if len(self.pids(node)) == 0: + raise RuntimeError("No process ids recorded") - def clean_node(self, node): - if len(self.pids()) > 0: - self.logger.warn("%s %s was still alive at cleanup time. Killing forcefully..." % - (self.__class__.__name__, node.account)) - for pid in self.pids(): - self.node.account.signal(pid, signal.SIGKILL, allow_fail=False) - node.account.ssh("rm -rf /mnt/copycat.pid /mnt/copycat.log /mnt/copycat.properties /mnt/copycat-connector.properties " + " ".join(self.files), allow_fail=False) diff --git a/tests/kafkatest/tests/copycat_distributed_test.py b/tests/kafkatest/tests/copycat_distributed_test.py new file mode 100644 index 0000000000000..3afcd57d86d54 --- /dev/null +++ b/tests/kafkatest/tests/copycat_distributed_test.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from kafkatest.tests.kafka_test import KafkaTest +from kafkatest.services.copycat import CopycatDistributedService +from ducktape.utils.util import wait_until +import hashlib, subprocess, json, itertools + +class CopycatDistributedFileTest(KafkaTest): + """ + Simple test of Copycat in distributed mode, producing data from files on on Copycat cluster and consuming it on + another, validating the total output is identical to the input. + """ + + INPUT_FILE = "/mnt/copycat.input" + OUTPUT_FILE = "/mnt/copycat.output" + + TOPIC = "test" + OFFSETS_TOPIC = "copycat-offsets" + + FIRST_INPUT_LISTS = [["foo", "bar", "baz"], ["foo2", "bar2", "baz2"]] + FIRST_INPUTS = ["\n".join(input_list) + "\n" for input_list in FIRST_INPUT_LISTS] + SECOND_INPUT_LISTS = [["razz", "ma", "tazz"], ["razz2", "ma2", "tazz2"]] + SECOND_INPUTS = ["\n".join(input_list) + "\n" for input_list in SECOND_INPUT_LISTS] + + SCHEMA = { "type": "string", "optional": False } + + def __init__(self, test_context): + super(CopycatDistributedFileTest, self).__init__(test_context, num_zk=1, num_brokers=1, topics={ + 'test' : { 'partitions': 1, 'replication-factor': 1 } + }) + + self.source = CopycatDistributedService(test_context, 2, self.kafka, [self.INPUT_FILE]) + self.sink = CopycatDistributedService(test_context, 2, self.kafka, [self.OUTPUT_FILE]) + + def test_file_source_and_sink(self, converter="org.apache.kafka.copycat.json.JsonConverter", schemas=True): + assert converter != None, "converter type must be set" + # Template parameters + self.key_converter = converter + self.value_converter = converter + self.schemas = schemas + + # These need to be set + self.source.set_configs(self.render("copycat-distributed.properties"), self.render("copycat-file-source.properties")) + self.sink.set_configs(self.render("copycat-distributed.properties"), self.render("copycat-file-sink.properties")) + + self.source.start() + self.sink.start() + + # Generating data on the source node should generate new records and create new output on the sink node + for node, input in zip(self.source.nodes, self.FIRST_INPUTS): + node.account.ssh("echo -e -n " + repr(input) + " >> " + self.INPUT_FILE) + wait_until(lambda: self.validate_output(self.FIRST_INPUT_LISTS), timeout_sec=60, err_msg="Data added to input file was not seen in the output file in a reasonable amount of time.") + + # Restarting both should result in them picking up where they left off, + # only processing new data. + self.source.restart() + self.sink.restart() + + for node, input in zip(self.source.nodes, self.SECOND_INPUTS): + node.account.ssh("echo -e -n " + repr(input) + " >> " + self.INPUT_FILE) + wait_until(lambda: self.validate_output(self.FIRST_INPUT_LISTS + self.SECOND_INPUT_LISTS), timeout_sec=60, err_msg="Sink output file never converged to the same state as the input file") + + def validate_output(self, inputs): + try: + input_set = set(itertools.chain(*inputs)) + output_set = set(itertools.chain(*[ + [line.strip() for line in node.account.ssh_capture("cat " + self.OUTPUT_FILE)] for node in self.sink.nodes + ])) + return input_set == output_set + except subprocess.CalledProcessError: + return False diff --git a/tests/kafkatest/tests/copycat_test.py b/tests/kafkatest/tests/copycat_test.py index b4adf53face31..1bd8ccbae696c 100644 --- a/tests/kafkatest/tests/copycat_test.py +++ b/tests/kafkatest/tests/copycat_test.py @@ -53,7 +53,7 @@ def __init__(self, test_context): @parametrize(converter="org.apache.kafka.copycat.json.JsonConverter", schemas=True) @parametrize(converter="org.apache.kafka.copycat.json.JsonConverter", schemas=False) @parametrize(converter="org.apache.kafka.copycat.storage.StringConverter", schemas=None) - def test_file_source_and_sink(self, converter="org.apache.kafka.json.JsonConverter", schemas=True): + def test_file_source_and_sink(self, converter="org.apache.kafka.copycat.json.JsonConverter", schemas=True): assert converter != None, "converter type must be set" # Template parameters self.key_converter = converter diff --git a/tests/kafkatest/tests/templates/copycat-distributed.properties b/tests/kafkatest/tests/templates/copycat-distributed.properties new file mode 100644 index 0000000000000..da47423f56201 --- /dev/null +++ b/tests/kafkatest/tests/templates/copycat-distributed.properties @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bootstrap.servers={{ kafka.bootstrap_servers() }} + +key.converter={{ key_converter|default("org.apache.kafka.copycat.json.JsonConverter") }} +value.converter={{ value_converter|default("org.apache.kafka.copycat.json.JsonConverter") }} +{% if key_converter is not defined or key_converter.endswith("JsonConverter") %} +key.converter.schemas.enable={{ schemas|default(True)|string|lower }} +{% endif %} +{% if value_converter is not defined or value_converter.endswith("JsonConverter") %} +value.converter.schemas.enable={{ schemas|default(True)|string|lower }} +{% endif %} + +offset.key.converter=org.apache.kafka.copycat.json.JsonConverter +offset.value.converter=org.apache.kafka.copycat.json.JsonConverter +offset.key.converter.schemas.enable=false +offset.value.converter.schemas.enable=false + +offset.storage.topic={{ OFFSETS_TOPIC }} diff --git a/tests/setup.py b/tests/setup.py index 00e58dd21c252..d637eb8dd0bdf 100644 --- a/tests/setup.py +++ b/tests/setup.py @@ -17,7 +17,7 @@ from setuptools import find_packages, setup setup(name="kafkatest", - version="0.8.3.dev0", + version="0.9.0.dev0", description="Apache Kafka System Tests", author="Apache Kafka", platforms=["any"],