From 203b476a10fa584b2629e56307d907ca96c267f6 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Fri, 11 Aug 2017 17:25:06 -0700 Subject: [PATCH 01/40] Initial commit of client side changes with some tests --- .../kafka/clients/producer/KafkaProducer.java | 16 +- .../producer/internals/ProducerBatch.java | 37 ++++- .../producer/internals/RecordAccumulator.java | 69 +++++++-- .../clients/producer/internals/Sender.java | 21 ++- .../internals/TransactionManager.java | 25 ++++ .../common/record/MemoryRecordsBuilder.java | 7 +- .../producer/internals/SenderTest.java | 141 +++++++++++++++++- 7 files changed, 264 insertions(+), 52 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index 18248bb782d9f..86578fe648b48 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -365,7 +365,7 @@ private KafkaProducer(ProducerConfig config, Serializer keySerializer, Serial this.requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); this.transactionManager = configureTransactionState(config, logContext, log); int retries = configureRetries(config, transactionManager != null, log); - int maxInflightRequests = configureInflightRequests(config, transactionManager != null, log); + int maxInflightRequests = configureInflightRequests(config, transactionManager != null); short acks = configureAcks(config, transactionManager != null, log); this.apiVersions = new ApiVersions(); @@ -481,17 +481,9 @@ private static int configureRetries(ProducerConfig config, boolean idempotenceEn return config.getInt(ProducerConfig.RETRIES_CONFIG); } - private static int configureInflightRequests(ProducerConfig config, boolean idempotenceEnabled, Logger log) { - boolean userConfiguredInflights = false; - if (config.originals().containsKey(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION)) { - userConfiguredInflights = true; - } - if (idempotenceEnabled && !userConfiguredInflights) { - log.info("Overriding the default {} to 1 since idempontence is enabled.", ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION); - return 1; - } - if (idempotenceEnabled && config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION) != 1) { - throw new ConfigException("Must set " + ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION + " to 1 in order" + + private static int configureInflightRequests(ProducerConfig config, boolean idempotenceEnabled) { + if (idempotenceEnabled && 0 < config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION)) { + throw new ConfigException("Must set " + ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION + " to at most 5" + "to use the idempotent producer. Otherwise we cannot guarantee idempotence."); } return config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java index ee7d21a5df198..e7d3417d1af55 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java @@ -75,6 +75,7 @@ private enum FinalState { ABORTED, FAILED, SUCCEEDED } private long drainedMs; private String expiryErrorMessage; private boolean retry; + private boolean countedTowardSequence; public ProducerBatch(TopicPartition tp, MemoryRecordsBuilder recordsBuilder, long now) { this(tp, recordsBuilder, now, false); @@ -89,6 +90,7 @@ public ProducerBatch(TopicPartition tp, MemoryRecordsBuilder recordsBuilder, lon this.produceFuture = new ProduceRequestResult(topicPartition); this.retry = false; this.isSplitBatch = isSplitBatch; + this.countedTowardSequence = false; float compressionRatioEstimation = CompressionRatioEstimator.estimation(topicPartition.topic(), recordsBuilder.compressionType()); recordsBuilder.setEstimatedCompressionRatio(compressionRatioEstimation); @@ -233,12 +235,13 @@ public Deque split(int splitBatchSize) { assert thunkIter.hasNext(); Thunk thunk = thunkIter.next(); if (batch == null) - batch = createBatchOffAccumulatorForRecord(record, splitBatchSize); + batch = createBatchOffAccumulatorForRecord(record, splitBatchSize, baseSequence(), isTransactional()); // A newly created batch can always host the first message. if (!batch.tryAppendForSplit(record.timestamp(), record.key(), record.value(), record.headers(), thunk)) { + int nextSequence = batch.baseSequence() == RecordBatch.NO_SEQUENCE ? RecordBatch.NO_SEQUENCE : batch.baseSequence() + batch.recordCount; batches.add(batch); - batch = createBatchOffAccumulatorForRecord(record, splitBatchSize); + batch = createBatchOffAccumulatorForRecord(record, splitBatchSize, nextSequence, batch.isTransactional()); batch.tryAppendForSplit(record.timestamp(), record.key(), record.value(), record.headers(), thunk); } } @@ -252,7 +255,7 @@ public Deque split(int splitBatchSize) { return batches; } - private ProducerBatch createBatchOffAccumulatorForRecord(Record record, int batchSize) { + private ProducerBatch createBatchOffAccumulatorForRecord(Record record, int batchSize, int sequence, boolean isTransactional) { int initialSize = Math.max(AbstractRecords.estimateSizeInBytesUpperBound(magic(), recordsBuilder.compressionType(), record.key(), record.value(), record.headers()), batchSize); ByteBuffer buffer = ByteBuffer.allocate(initialSize); @@ -261,8 +264,11 @@ private ProducerBatch createBatchOffAccumulatorForRecord(Record record, int batc // for the newly created batch. This will be set when the batch is dequeued for sending (which is consistent // with how normal batches are handled). MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, magic(), recordsBuilder.compressionType(), - TimestampType.CREATE_TIME, 0L); - return new ProducerBatch(topicPartition, builder, this.createdMs, true); + TimestampType.CREATE_TIME, 0L, RecordBatch.NO_TIMESTAMP, RecordBatch.NO_PRODUCER_ID, + RecordBatch.NO_PRODUCER_EPOCH, sequence, isTransactional, RecordBatch.NO_PARTITION_LEADER_EPOCH); + ProducerBatch batch = new ProducerBatch(topicPartition, builder, this.createdMs, true); + batch.countSequenceNumber(); + return batch; } public boolean isCompressed() { @@ -374,9 +380,8 @@ public boolean isFull() { return recordsBuilder.isFull(); } - public void setProducerState(ProducerIdAndEpoch producerIdAndEpoch, int baseSequence, boolean isTransactional) { - recordsBuilder.setProducerState(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, - baseSequence, isTransactional); + public void setProducerState(ProducerIdAndEpoch producerIdAndEpoch) { + recordsBuilder.setProducerState(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch); } /** @@ -434,4 +439,20 @@ public long producerId() { public short producerEpoch() { return recordsBuilder.producerEpoch(); } + + public int baseSequence() { + return recordsBuilder.baseSequence(); + } + + private boolean isTransactional() { + return recordsBuilder.isTransactional(); + } + + void countSequenceNumber() { + countedTowardSequence = true; + } + + boolean hasBeenCountedTowardSequence() { + return countedTowardSequence; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index 38b5e517fcf84..e57b767505d8b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -48,6 +48,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; @@ -195,7 +196,7 @@ public RecordAppendResult append(TopicPartition tp, synchronized (dq) { if (closed) throw new IllegalStateException("Cannot send after the producer is closed."); - RecordAppendResult appendResult = tryAppend(timestamp, key, value, headers, callback, dq); + RecordAppendResult appendResult = tryAppend(tp, timestamp, key, value, headers, callback, dq); if (appendResult != null) return appendResult; } @@ -210,13 +211,13 @@ public RecordAppendResult append(TopicPartition tp, if (closed) throw new IllegalStateException("Cannot send after the producer is closed."); - RecordAppendResult appendResult = tryAppend(timestamp, key, value, headers, callback, dq); + RecordAppendResult appendResult = tryAppend(tp, timestamp, key, value, headers, callback, dq); if (appendResult != null) { // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... return appendResult; } - MemoryRecordsBuilder recordsBuilder = recordsBuilder(buffer, maxUsableMagic); + MemoryRecordsBuilder recordsBuilder = recordsBuilder(tp, buffer, maxUsableMagic); ProducerBatch batch = new ProducerBatch(tp, recordsBuilder, time.milliseconds()); FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, headers, callback, time.milliseconds())); @@ -235,11 +236,20 @@ public RecordAppendResult append(TopicPartition tp, } } - private MemoryRecordsBuilder recordsBuilder(ByteBuffer buffer, byte maxUsableMagic) { + private MemoryRecordsBuilder recordsBuilder(TopicPartition topicPartition, ByteBuffer buffer, byte maxUsableMagic) { if (transactionManager != null && maxUsableMagic < RecordBatch.MAGIC_VALUE_V2) { throw new UnsupportedVersionException("Attempting to use idempotence with a broker which does not " + "support the required message format (v2). The broker must be version 0.11 or later."); } + if (transactionManager != null) { + int sequenceNumber = transactionManager.sequenceNumber(topicPartition); + MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, maxUsableMagic, compression, TimestampType.CREATE_TIME, 0L, + RecordBatch.NO_TIMESTAMP, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, sequenceNumber, + transactionManager.isTransactional(), RecordBatch.NO_PARTITION_LEADER_EPOCH); + log.debug("ProducerId {}: Assigned sequence number to {} to newly created batch for partition {}", + transactionManager.producerIdAndEpoch().producerId, sequenceNumber, topicPartition); + return builder; + } return MemoryRecords.builder(buffer, maxUsableMagic, compression, TimestampType.CREATE_TIME, 0L); } @@ -251,15 +261,18 @@ private MemoryRecordsBuilder recordsBuilder(ByteBuffer buffer, byte maxUsableMag * and memory records built) in one of the following cases (whichever comes first): right before send, * if it is expired, or when the producer is closed. */ - private RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Header[] headers, Callback callback, Deque deque) { + private RecordAppendResult tryAppend(TopicPartition tp, long timestamp, byte[] key, byte[] value, Header[] headers, + Callback callback, Deque deque) { ProducerBatch last = deque.peekLast(); if (last != null) { FutureRecordMetadata future = last.tryAppend(timestamp, key, value, headers, callback, time.milliseconds()); - if (future == null) + if (future == null) { last.closeForRecordAppends(); - else + if (transactionManager != null) + transactionManager.maybeCountSequenceNumber(last); + } else { return new RecordAppendResult(future, deque.size() > 1 || last.isFull(), false); - + } } return null; } @@ -310,6 +323,21 @@ public void reenqueue(ProducerBatch batch, long now) { Deque deque = getOrCreateDeque(batch.topicPartition); synchronized (deque) { deque.addFirst(batch); + maybeReorderBySequenceNumber(deque); + } + } + + void maybeReorderBySequenceNumber(Deque deque) { + if (transactionManager != null) { + ArrayList copy = new ArrayList<>(deque); + Collections.sort(copy, new Comparator() { + @Override + public int compare(ProducerBatch o1, ProducerBatch o2) { + return o1.baseSequence() - o2.baseSequence(); + } + }); + deque.clear(); + deque.addAll(copy); } } @@ -334,6 +362,7 @@ public int splitAndReenqueue(ProducerBatch bigBatch) { partitionDequeue.addFirst(batch); } } + maybeReorderBySequenceNumber(partitionDequeue); return numSplitBatches; } @@ -458,7 +487,6 @@ public Map> drain(Cluster cluster, break; } else { ProducerIdAndEpoch producerIdAndEpoch = null; - boolean isTransactional = false; if (transactionManager != null) { if (!transactionManager.isSendToPartitionAllowed(tp)) break; @@ -467,8 +495,6 @@ public Map> drain(Cluster cluster, if (!producerIdAndEpoch.isValid()) // we cannot send the batch until we have refreshed the producer id break; - - isTransactional = transactionManager.isTransactional(); } ProducerBatch batch = deque.pollFirst(); @@ -478,11 +504,22 @@ public Map> drain(Cluster cluster, // the previous attempt may actually have been accepted, and if we change // the producer id and sequence here, this attempt will also be accepted, // causing a duplicate. - int sequenceNumber = transactionManager.sequenceNumber(batch.topicPartition); - log.debug("Assigning sequence number {} from producer {} to dequeued " + - "batch from partition {} bound for {}.", - sequenceNumber, producerIdAndEpoch, batch.topicPartition, node); - batch.setProducerState(producerIdAndEpoch, sequenceNumber, isTransactional); + batch.setProducerState(producerIdAndEpoch); + log.debug("Assigned producerId {} and producerEpoch {} to batch with sequence " + + "{} being sent to partition {}", producerIdAndEpoch.producerId, + producerIdAndEpoch.epoch, batch.baseSequence(), tp); + } + if (transactionManager != null && !batch.hasBeenCountedTowardSequence()) { + // We close the batch for writing once it is full. We would have incremented + // the sequence number at that point so that the next batch would get the + // correct next sequence. + // + // However, it is possible that the linger has expired and we are just sending + // an incompleted batch which was not yet closed for writing. + // + // In this case, we need to increment the sequence number here. + transactionManager.maybeCountSequenceNumber(batch); + log.debug("ProducerId {}, Partition {}, incremented sequence number to: {}", producerIdAndEpoch.producerId, tp, transactionManager.sequenceNumber(tp)); } batch.close(); size += batch.records().sizeInBytes(); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 8da411c751a07..a816c2eeea1c0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -422,6 +422,7 @@ private void maybeWaitForProducerId() { ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch( initProducerIdResponse.producerId(), initProducerIdResponse.epoch()); transactionManager.setProducerIdAndEpoch(producerIdAndEpoch); + return; } else if (error.exception() instanceof RetriableException) { log.debug("Retriable error from InitProducerId response", error.message()); } else { @@ -517,7 +518,7 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons // If idempotence is enabled only retry the request if the current producer id is the same as // the producer id of the batch. log.debug("Retrying batch to topic-partition {}. Sequence number : {}", batch.topicPartition, - transactionManager.sequenceNumber(batch.topicPartition)); + batch.baseSequence()); reenqueueBatch(batch, now); } else { failBatch(batch, response, new OutOfOrderSequenceException("Attempted to retry sending a " + @@ -560,9 +561,9 @@ private void reenqueueBatch(ProducerBatch batch, long currentTimeMs) { private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response) { if (transactionManager != null && transactionManager.hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch())) { - transactionManager.incrementSequenceNumber(batch.topicPartition, batch.recordCount); - log.debug("Incremented sequence number for topic-partition {} to {}", batch.topicPartition, - transactionManager.sequenceNumber(batch.topicPartition)); + transactionManager.setLastAckedSequence(batch.topicPartition, batch.baseSequence() + batch.recordCount - 1); + log.debug("ProducerId: {}; Set last ack'd sequence number for topic-partition {} to {}", batch.producerId(), batch.topicPartition, + transactionManager.lastAckedSequence(batch.topicPartition)); } batch.done(response.baseOffset, response.logAppendTime, null); @@ -594,16 +595,24 @@ private void failBatch(ProducerBatch batch, long baseOffset, long logAppendTime, } else if (transactionManager.isTransactional()) { transactionManager.transitionToAbortableError(exception); } + + if (transactionManager.hasProducerId()) { + + } } + batch.done(baseOffset, logAppendTime, exception); this.accumulator.deallocate(batch); } /** - * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed + * We can retry a send if the error is transient and the number of attempts taken is fewer than the maximum allowed. + * We can also retry OutOfOrderSequence exceptions for future batches, since if the first batch has failed, the future + * batches are certain to fail with an OutOfOrderSequence exception. */ private boolean canRetry(ProducerBatch batch, Errors error) { - return batch.attempts() < this.retries && error.exception() instanceof RetriableException; + return (batch.attempts() < this.retries && error.exception() instanceof RetriableException) || + (error.exception() instanceof OutOfOrderSequenceException && !transactionManager.isNextSequence(batch.topicPartition, batch.baseSequence())); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index 05d943c8fb0a3..df08ccc4e1a82 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -66,6 +66,7 @@ public class TransactionManager { private final int transactionTimeoutMs; private final Map sequenceNumbers; + private final Map lastAckedSequence; private final PriorityQueue pendingRequests; private final Set newPartitionsInTransaction; private final Set pendingPartitionsInTransaction; @@ -143,6 +144,7 @@ private enum Priority { public TransactionManager(LogContext logContext, String transactionalId, int transactionTimeoutMs, long retryBackoffMs) { this.producerIdAndEpoch = new ProducerIdAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); this.sequenceNumbers = new HashMap<>(); + this.lastAckedSequence = new HashMap<>(); this.transactionalId = transactionalId; this.log = logContext.logger(TransactionManager.class); this.transactionTimeoutMs = transactionTimeoutMs; @@ -377,6 +379,13 @@ synchronized Integer sequenceNumber(TopicPartition topicPartition) { return currentSequenceNumber; } + synchronized void maybeCountSequenceNumber(ProducerBatch batch) { + if (batch.hasBeenCountedTowardSequence()) + return; + incrementSequenceNumber(batch.topicPartition, batch.recordCount); + batch.countSequenceNumber(); + } + synchronized void incrementSequenceNumber(TopicPartition topicPartition, int increment) { Integer currentSequenceNumber = sequenceNumbers.get(topicPartition); if (currentSequenceNumber == null) @@ -386,6 +395,22 @@ synchronized void incrementSequenceNumber(TopicPartition topicPartition, int inc sequenceNumbers.put(topicPartition, currentSequenceNumber); } + synchronized void setLastAckedSequence(TopicPartition topicPartition, int sequence) { + lastAckedSequence.put(topicPartition, sequence); + } + + synchronized int lastAckedSequence(TopicPartition topicPartition) { + Integer currentLastAckedSequence = lastAckedSequence.get(topicPartition); + if (currentLastAckedSequence == null) + return -1; + return lastAckedSequence.get(topicPartition); + } + + synchronized boolean isNextSequence(TopicPartition topicPartition, int sequence) { + return (!lastAckedSequence.containsKey(topicPartition) && sequence == 0) || + (lastAckedSequence.containsKey(topicPartition) && (sequence - lastAckedSequence.get(topicPartition) == 1)); + } + synchronized TxnRequestHandler nextRequestHandler(boolean hasIncompleteBatches) { if (!newPartitionsInTransaction.isEmpty()) enqueueRequest(addPartitionsToTransactionHandler()); diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java index 19d25d7b52dc8..6f7337b28afdb 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java @@ -238,7 +238,7 @@ public RecordsInfo info() { } } - public void setProducerState(long producerId, short producerEpoch, int baseSequence, boolean isTransactional) { + public void setProducerState(long producerId, short producerEpoch) { if (isClosed()) { // Sequence numbers are assigned when the batch is closed while the accumulator is being drained. // If the resulting ProduceRequest to the partition leader failed for a retriable error, the batch will @@ -248,8 +248,6 @@ public void setProducerState(long producerId, short producerEpoch, int baseSeque } this.producerId = producerId; this.producerEpoch = producerEpoch; - this.baseSequence = baseSequence; - this.isTransactional = isTransactional; } public void overrideLastOffset(long lastOffset) { @@ -766,4 +764,7 @@ public short producerEpoch() { return this.producerEpoch; } + public int baseSequence() { + return this.baseSequence; + } } diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 6f98e522621e2..bd096d76845a8 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -467,6 +467,127 @@ public void testClusterAuthorizationExceptionInInitProducerIdRequest() throws Ex assertSendFailure(ClusterAuthorizationException.class); } + + @Test + public void testIdempotenceWithMultipleInflights() throws Exception { + final long producerId = 343434L; + TransactionManager transactionManager = new TransactionManager(); + setupWithTransactionState(transactionManager); + prepareAndReceiveInitProducerId(producerId, Errors.NONE); + assertTrue(transactionManager.hasProducerId()); + + assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); + + // Send first ProduceRequest + Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + String nodeId = client.requests().peek().destination(); + Node node = new Node(Integer.valueOf(nodeId), "localhost", 0); + assertEquals(1, client.inFlightRequestCount()); + assertEquals(1, transactionManager.sequenceNumber(tp0).longValue()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + + // Send second ProduceRequest + Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + assertEquals(2, client.inFlightRequestCount()); + assertEquals(2, transactionManager.sequenceNumber(tp0).longValue()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + assertFalse(request1.isDone()); + assertFalse(request2.isDone()); + assertTrue(client.isReady(node, time.milliseconds())); + + sendIdempotentProducerResponse(0, tp0, Errors.NONE, 0L); + + sender.run(time.milliseconds()); // receive response 0 + + assertEquals(1, client.inFlightRequestCount()); + assertEquals(0, transactionManager.lastAckedSequence(tp0)); + + sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L); + sender.run(time.milliseconds()); // receive response 1 + assertEquals(1, transactionManager.lastAckedSequence(tp0)); + assertFalse(client.hasInFlightRequests()); + } + + + @Test + public void testIdempotenceWithMultipleInflightsFirstFails() throws Exception { + final long producerId = 343434L; + TransactionManager transactionManager = new TransactionManager(); + setupWithTransactionState(transactionManager); + prepareAndReceiveInitProducerId(producerId, Errors.NONE); + assertTrue(transactionManager.hasProducerId()); + + assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); + + // Send first ProduceRequest + Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + String nodeId = client.requests().peek().destination(); + Node node = new Node(Integer.valueOf(nodeId), "localhost", 0); + assertEquals(1, client.inFlightRequestCount()); + assertEquals(1, transactionManager.sequenceNumber(tp0).longValue()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + + // Send second ProduceRequest + Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + assertEquals(2, client.inFlightRequestCount()); + assertEquals(2, transactionManager.sequenceNumber(tp0).longValue()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + assertFalse(request1.isDone()); + assertFalse(request2.isDone()); + assertTrue(client.isReady(node, time.milliseconds())); + + sendIdempotentProducerResponse(0, tp0, Errors.LEADER_NOT_AVAILABLE, -1L); + + sender.run(time.milliseconds()); // receive response 0 + + assertEquals(1, client.inFlightRequestCount()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + + sendIdempotentProducerResponse(1, tp0, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, -1L); + + sender.run(time.milliseconds()); // re send request 0, receive response 1 + + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + assertEquals(1, client.inFlightRequestCount()); + + sender.run(time.milliseconds()); // resend request 1 + + assertEquals(2, client.inFlightRequestCount()); + + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + + sendIdempotentProducerResponse(0, tp0, Errors.NONE, 0L); + sender.run(time.milliseconds()); // receive response 0 + + sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L); + sender.run(time.milliseconds()); // receive response 1 + + assertFalse(client.hasInFlightRequests()); + assertEquals(1, transactionManager.lastAckedSequence(tp0)); + } + + void sendIdempotentProducerResponse(final int expectedSequence, TopicPartition tp, Errors responseError, long responseOffset) { + client.respond(new MockClient.RequestMatcher() { + @Override + public boolean matches(AbstractRequest body) { + ProduceRequest produceRequest = (ProduceRequest) body; + assertTrue(produceRequest.isIdempotent()); + + MemoryRecords records = produceRequest.partitionRecordsOrFail().get(tp0); + Iterator batchIterator = records.batches().iterator(); + RecordBatch firstBatch = batchIterator.next(); + assertFalse(batchIterator.hasNext()); + assertEquals(expectedSequence, firstBatch.baseSequence()); + + return true; + } + }, produceResponse(tp, responseOffset, responseError, 0)); + } + @Test public void testClusterAuthorizationExceptionInProduceRequest() throws Exception { final long producerId = 343434L; @@ -604,7 +725,8 @@ public boolean matches(AbstractRequest body) { sender.run(time.milliseconds()); // receive response assertTrue(responseFuture.isDone()); - assertEquals((long) transactionManager.sequenceNumber(tp0), 1L); + assertEquals(0L, (long) transactionManager.lastAckedSequence(tp0)); + assertEquals(1L, (long) transactionManager.sequenceNumber(tp0)); } @Test @@ -632,6 +754,7 @@ public void testAbortRetryWhenProducerIdChanges() throws InterruptedException { assertEquals(0, client.inFlightRequestCount()); assertFalse("Client ready status should be false", client.isReady(node, 0L)); + transactionManager.resetProducerId(); transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId + 1, (short) 0)); sender.run(time.milliseconds()); // receive error sender.run(time.milliseconds()); // reconnect @@ -642,7 +765,7 @@ public void testAbortRetryWhenProducerIdChanges() throws InterruptedException { assertTrue("Expected non-zero value for record send errors", recordErrors.value() > 0); assertTrue(responseFuture.isDone()); - assertEquals((long) transactionManager.sequenceNumber(tp0), 0L); + assertEquals(0, (long) transactionManager.sequenceNumber(tp0)); } @Test @@ -720,7 +843,8 @@ private void testSplitBatchAndSend(TransactionManager txnManager, accumulator.append(tp, 0L, "key2".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT).future; sender.run(time.milliseconds()); // connect sender.run(time.milliseconds()); // send produce request - assertEquals("The sequence number should be 0", 0, txnManager.sequenceNumber(tp).longValue()); + + assertEquals("The next sequence should be 2", 2, txnManager.sequenceNumber(tp).longValue()); String id = client.requests().peek().destination(); assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey()); Node node = new Node(Integer.valueOf(id), "localhost", 0); @@ -735,7 +859,7 @@ private void testSplitBatchAndSend(TransactionManager txnManager, assertEquals(CompressionType.GZIP.rate - CompressionRatioEstimator.COMPRESSION_RATIO_IMPROVING_STEP, CompressionRatioEstimator.estimation(topic, CompressionType.GZIP), 0.01); sender.run(time.milliseconds()); // send produce request - assertEquals("The sequence number should be 0", 0, txnManager.sequenceNumber(tp).longValue()); + assertEquals("The next sequence number should be 2", 2, txnManager.sequenceNumber(tp).longValue()); assertFalse("The future shouldn't have been done.", f1.isDone()); assertFalse("The future shouldn't have been done.", f2.isDone()); id = client.requests().peek().destination(); @@ -750,7 +874,8 @@ private void testSplitBatchAndSend(TransactionManager txnManager, sender.run(time.milliseconds()); // receive assertTrue("The future should have been done.", f1.isDone()); - assertEquals("The sequence number should be 1", 1, txnManager.sequenceNumber(tp).longValue()); + assertEquals("The next sequence number should be 2", 2, txnManager.sequenceNumber(tp).longValue()); + assertEquals("The last ack'd sequence number should be 0", 0, txnManager.lastAckedSequence(tp)); assertFalse("The future shouldn't have been done.", f2.isDone()); assertEquals("Offset of the first message should be 0", 0L, f1.get().offset()); sender.run(time.milliseconds()); // send produce request @@ -766,7 +891,8 @@ private void testSplitBatchAndSend(TransactionManager txnManager, sender.run(time.milliseconds()); // receive assertTrue("The future should have been done.", f2.isDone()); - assertEquals("The sequence number should be 2", 2, txnManager.sequenceNumber(tp).longValue()); + assertEquals("The next sequence number should be 2", 2, txnManager.sequenceNumber(tp).longValue()); + assertEquals("The last ack'd sequence number should be 1", 1, txnManager.lastAckedSequence(tp)); assertEquals("Offset of the first message should be 1", 1L, f2.get().offset()); assertTrue("There should be no batch in the accumulator", accumulator.batches().get(tp).isEmpty()); @@ -828,9 +954,10 @@ private void setupWithTransactionState(TransactionManager transactionManager) { this.metrics = new Metrics(metricConfig, time); this.accumulator = new RecordAccumulator(logContext, batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time, apiVersions, transactionManager); + this.senderMetricsRegistry = new SenderMetricsRegistry(metricTags.keySet()); this.sender = new Sender(logContext, this.client, this.metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, - MAX_RETRIES, this.metrics, this.senderMetricsRegistry, this.time, REQUEST_TIMEOUT, 50, transactionManager, apiVersions); + Integer.MAX_VALUE, this.metrics, this.senderMetricsRegistry, this.time, REQUEST_TIMEOUT, 50, transactionManager, apiVersions); this.metadata.update(this.cluster, Collections.emptySet(), time.milliseconds()); } From 546a9a7102809def60055b335d1c19f2dc0a4ff5 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 17 Aug 2017 16:49:10 -0700 Subject: [PATCH 02/40] Implemented broker side changes to cache extra metadata. Todo: 1) Write more unit tests. 2) Handle deletion / retention / cleaning correctly. --- .../kafka/clients/producer/KafkaProducer.java | 2 +- .../producer/internals/RecordAccumulator.java | 3 +- .../internals/TransactionManager.java | 2 + .../producer/internals/SenderTest.java | 51 +++ clients/src/test/resources/log4j.properties | 2 + core/src/main/scala/kafka/log/Log.scala | 21 +- .../kafka/log/ProducerStateManager.scala | 320 ++++++++++++------ .../scala/kafka/tools/DumpLogSegments.scala | 8 +- core/src/test/resources/log4j.properties | 2 +- .../scala/unit/kafka/log/LogSegmentTest.scala | 3 +- .../kafka/log/ProducerStateManagerTest.scala | 34 +- 11 files changed, 314 insertions(+), 134 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index 86578fe648b48..3ef0a74d9988d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -482,7 +482,7 @@ private static int configureRetries(ProducerConfig config, boolean idempotenceEn } private static int configureInflightRequests(ProducerConfig config, boolean idempotenceEnabled) { - if (idempotenceEnabled && 0 < config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION)) { + if (idempotenceEnabled && 5 < config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION)) { throw new ConfigException("Must set " + ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION + " to at most 5" + "to use the idempotent producer. Otherwise we cannot guarantee idempotence."); } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index e57b767505d8b..618cc9b0a1ead 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -509,7 +509,7 @@ public Map> drain(Cluster cluster, "{} being sent to partition {}", producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, batch.baseSequence(), tp); } - if (transactionManager != null && !batch.hasBeenCountedTowardSequence()) { + if (transactionManager != null) { // We close the batch for writing once it is full. We would have incremented // the sequence number at that point so that the next batch would get the // correct next sequence. @@ -519,7 +519,6 @@ public Map> drain(Cluster cluster, // // In this case, we need to increment the sequence number here. transactionManager.maybeCountSequenceNumber(batch); - log.debug("ProducerId {}, Partition {}, incremented sequence number to: {}", producerIdAndEpoch.producerId, tp, transactionManager.sequenceNumber(tp)); } batch.close(); size += batch.records().sizeInBytes(); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index df08ccc4e1a82..bc83a86c62a8d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -383,6 +383,8 @@ synchronized void maybeCountSequenceNumber(ProducerBatch batch) { if (batch.hasBeenCountedTowardSequence()) return; incrementSequenceNumber(batch.topicPartition, batch.recordCount); + log.debug("ProducerId: {}; topic-partition: {}; incremented sequence number to {}", + producerIdAndEpoch().producerId, batch.topicPartition, sequenceNumber(batch.topicPartition)); batch.countSequenceNumber(); } diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index bd096d76845a8..7fb518844edc1 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -29,6 +29,7 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.ClusterAuthorizationException; +import org.apache.kafka.common.errors.OutOfOrderSequenceException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.UnsupportedForMessageFormatException; import org.apache.kafka.common.errors.UnsupportedVersionException; @@ -570,6 +571,56 @@ public void testIdempotenceWithMultipleInflightsFirstFails() throws Exception { assertEquals(1, transactionManager.lastAckedSequence(tp0)); } + @Test + public void testMustNotRetryOutOfOrderSequenceForNextBatch() throws Exception { + final long producerId = 343434L; + TransactionManager transactionManager = new TransactionManager(); + setupWithTransactionState(transactionManager); + prepareAndReceiveInitProducerId(producerId, Errors.NONE); + assertTrue(transactionManager.hasProducerId()); + + assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); + + // Send first ProduceRequest with multiple messages. + Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT); + sender.run(time.milliseconds()); + String nodeId = client.requests().peek().destination(); + Node node = new Node(Integer.valueOf(nodeId), "localhost", 0); + assertEquals(1, client.inFlightRequestCount()); + + // make sure the next sequence number accounts for multi-message batches. + assertEquals(2, transactionManager.sequenceNumber(tp0).longValue()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + sendIdempotentProducerResponse(0, tp0, Errors.NONE, 0); + + sender.run(time.milliseconds()); + + // Send second ProduceRequest + Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + assertEquals(1, client.inFlightRequestCount()); + assertEquals(3, transactionManager.sequenceNumber(tp0).longValue()); + assertEquals(1, transactionManager.lastAckedSequence(tp0)); + assertTrue(request1.isDone()); + assertFalse(request2.isDone()); + assertTrue(client.isReady(node, time.milliseconds())); + + // This OutOfOrderSequence is fatal since it is returned for the batch succeeding the last acknowledged batch. + sendIdempotentProducerResponse(2, tp0, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, -1L); + + sender.run(time.milliseconds()); + assertTrue(request2.isDone()); + + try { + request2.get(); + fail("Expected an OutOfOrderSequenceException"); + } catch (ExecutionException e) { + assert e.getCause() instanceof OutOfOrderSequenceException; + } + } + + void sendIdempotentProducerResponse(final int expectedSequence, TopicPartition tp, Errors responseError, long responseOffset) { client.respond(new MockClient.RequestMatcher() { @Override diff --git a/clients/src/test/resources/log4j.properties b/clients/src/test/resources/log4j.properties index b1d5b7f2b4091..30f06eacf74f5 100644 --- a/clients/src/test/resources/log4j.properties +++ b/clients/src/test/resources/log4j.properties @@ -19,3 +19,5 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n log4j.logger.org.apache.kafka=ERROR + +log4j.logger.org.apache.kafka.clients.producer.internals=TRACE \ No newline at end of file diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala index d3de24db643c4..11db803389eef 100644 --- a/core/src/main/scala/kafka/log/Log.scala +++ b/core/src/main/scala/kafka/log/Log.scala @@ -765,18 +765,25 @@ class Log(@volatile var dir: File, } private def analyzeAndValidateProducerState(records: MemoryRecords, isFromClient: Boolean): - (mutable.Map[Long, ProducerAppendInfo], List[CompletedTxn], Option[ProducerIdEntry]) = { + (mutable.Map[Long, ProducerAppendInfo], List[CompletedTxn], Option[BatchMetadata]) = { val updatedProducers = mutable.Map.empty[Long, ProducerAppendInfo] val completedTxns = ListBuffer.empty[CompletedTxn] for (batch <- records.batches.asScala if batch.hasProducerId) { val maybeLastEntry = producerStateManager.lastEntry(batch.producerId) - // if this is a client produce request, there will be only one batch. If that batch matches - // the last appended entry for that producer, then this request is a duplicate and we return - // the last appended entry to the client. - if (isFromClient && maybeLastEntry.exists(_.isDuplicate(batch))) - return (updatedProducers, completedTxns.toList, maybeLastEntry) - + // if this is a client produce request, there will be upto 5 batches which could have been duplicated. + // If we find a duplicate, we return the metadata of the appended batch to the client. + if (isFromClient) { + maybeLastEntry match { + case Some(lastEntry) => + lastEntry.duplicateOf(batch) match { + case Some(duplicateBatch) => + return (updatedProducers, completedTxns.toList, Option(duplicateBatch)) + case _ => + } + case _ => + } + } val maybeCompletedTxn = updateProducers(batch, updatedProducers, loadingFromLog = false) maybeCompletedTxn.foreach(completedTxns += _) } diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index fc2e34024c32c..6318a1bfd9bdf 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -48,33 +48,84 @@ private[log] case class TxnMetadata(producerId: Long, var firstOffset: LogOffset } private[log] object ProducerIdEntry { - val Empty = ProducerIdEntry(RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, - -1, 0, RecordBatch.NO_TIMESTAMP, -1, None) + def empty(producerId: Long) = new ProducerIdEntry(producerId, mutable.Queue[BatchMetadata](), RecordBatch.NO_PRODUCER_EPOCH, -1, None) } -private[log] case class ProducerIdEntry(producerId: Long, producerEpoch: Short, lastSeq: Int, lastOffset: Long, - offsetDelta: Int, timestamp: Long, coordinatorEpoch: Int, - currentTxnFirstOffset: Option[Long]) { - def firstSeq: Int = lastSeq - offsetDelta - def firstOffset: Long = lastOffset - offsetDelta +private[log] case class BatchMetadata(lastSeq: Int, lastOffset: Long, offsetDelta: Int, timestamp: Long) { + def firstSeq = lastSeq - offsetDelta + def firstOffset = lastOffset - offsetDelta - def isDuplicate(batch: RecordBatch): Boolean = { - batch.producerEpoch == producerEpoch && - batch.baseSequence == firstSeq && - batch.lastSequence == lastSeq + override def toString: String = { + "BatchMetadata(" + + s"firstSeq=$firstSeq, " + + s"lastSeq=$lastSeq, " + + s"firstOffset=$firstOffset, " + + s"lastOffset=$lastOffset, " + + s"timestamp=$timestamp)" + } +} + +private[log] case class ProducerIdEntry(producerId: Long, batchMetadata: mutable.Queue[BatchMetadata], + var producerEpoch: Short, var coordinatorEpoch: Int, + var currentTxnFirstOffset: Option[Long]) { + + private val NumBatchesToRetain = 5 + + def firstSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.front.firstSeq + def firstOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.front.firstOffset + + def lastSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.last.lastSeq + def lastOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.last.lastOffset + def lastTimestamp = if (batchMetadata.isEmpty) RecordBatch.NO_TIMESTAMP else batchMetadata.last.timestamp + + def addBatchMetadata(producerEpoch: Short, lastSeq: Int, lastOffset: Long, offsetDelta: Int, timestamp: Long) = { + maybeUpdateEpoch(producerEpoch) + + if (batchMetadata.size == NumBatchesToRetain) + batchMetadata.dequeue() + + batchMetadata.enqueue(BatchMetadata(lastSeq, lastOffset, offsetDelta, timestamp)) + } + + def maybeUpdateEpoch(producerEpoch: Short): Boolean = { + if (this.producerEpoch != producerEpoch) { + batchMetadata.clear() + this.producerEpoch = producerEpoch + return true + } + false + } + + def duplicateOf(batch: RecordBatch): Option[BatchMetadata] = { + if (batch.producerEpoch() != producerEpoch) + return None + + hasBatchWithSequenceRange(batch.baseSequence(), batch.lastSequence()) + } + + def hasBatchWithSequenceRange(firstSeq: Int, lastSeq:Int) : Option[BatchMetadata] = { + val duplicate = batchMetadata.filter{ case(metadata) => + firstSeq == metadata.firstSeq && lastSeq == metadata.lastSeq + } + + if (duplicate.size > 1) + throw new IllegalStateException(s"Found ${duplicate.size} duplicate batches in the cached producer metadata " + + s"for producerId: $producerId and producerEpoch: $producerEpoch. This means that it's likely there are " + + s"duplicates in the log as well.") + + if (duplicate.size == 1) + Some(duplicate.head) + else + None } override def toString: String = { "ProducerIdEntry(" + s"producerId=$producerId, " + s"producerEpoch=$producerEpoch, " + - s"firstSequence=$firstSeq, " + - s"lastSequence=$lastSeq, " + - s"firstOffset=$firstOffset, " + - s"lastOffset=$lastOffset, " + - s"timestamp=$timestamp, " + s"currentTxnFirstOffset=$currentTxnFirstOffset, " + - s"coordinatorEpoch=$coordinatorEpoch)" + s"coordinatorEpoch=$coordinatorEpoch, " + + s"batchMetadata=$batchMetadata" } } @@ -85,8 +136,10 @@ private[log] case class ProducerIdEntry(producerId: Long, producerEpoch: Short, * as the incoming records are validated. * * @param producerId The id of the producer appending to the log - * @param initialEntry The last entry associated with the producer id. Validation of the first append will be - * based off of this entry initially + * @param currentEntry The current entry associated with the producer id which contains metadata for a fixed number of + * the most recent appends made by the producer. Validation of the first incoming append will + * be made against the lastest append in the current entry. New appends will replace older appends + * in the current entry so that the space overhead is constant. * @param validateSequenceNumbers Whether or not sequence numbers should be validated. The only current use * of this is the consumer offsets topic which uses producer ids from incoming * TxnOffsetCommit, but has no sequence number to validate and does not depend @@ -98,48 +151,41 @@ private[log] case class ProducerIdEntry(producerId: Long, producerEpoch: Short, * retention enforcement. */ private[log] class ProducerAppendInfo(val producerId: Long, - initialEntry: ProducerIdEntry, + currentEntry: ProducerIdEntry, validateSequenceNumbers: Boolean, loadingFromLog: Boolean) { - private var producerEpoch = initialEntry.producerEpoch - private var firstSeq = initialEntry.firstSeq - private var lastSeq = initialEntry.lastSeq - private var lastOffset = initialEntry.lastOffset - private var maxTimestamp = initialEntry.timestamp - private var currentTxnFirstOffset = initialEntry.currentTxnFirstOffset - private var coordinatorEpoch = initialEntry.coordinatorEpoch + private val transactions = ListBuffer.empty[TxnMetadata] private def validateAppend(producerEpoch: Short, firstSeq: Int, lastSeq: Int) = { if (isFenced(producerEpoch)) { throw new ProducerFencedException(s"Producer's epoch is no longer valid. There is probably another producer " + - s"with a newer epoch. $producerEpoch (request epoch), ${this.producerEpoch} (server epoch)") + s"with a newer epoch. $producerEpoch (request epoch), ${currentEntry.producerEpoch} (server epoch)") } else if (validateSequenceNumbers) { - if (producerEpoch != this.producerEpoch) { + if (producerEpoch != currentEntry.producerEpoch) { if (firstSeq != 0) throw new OutOfOrderSequenceException(s"Invalid sequence number for new epoch: $producerEpoch " + s"(request epoch), $firstSeq (seq. number)") - } else if (this.firstSeq == RecordBatch.NO_SEQUENCE && firstSeq != 0) { + } else if (currentEntry.lastSeq == RecordBatch.NO_SEQUENCE && firstSeq != 0) { // the epoch was bumped by a control record, so we expect the sequence number to be reset throw new OutOfOrderSequenceException(s"Out of order sequence number for producerId $producerId: found $firstSeq " + s"(incoming seq. number), but expected 0") - } else if (firstSeq == this.firstSeq && lastSeq == this.lastSeq) { + } else if (currentEntry.hasBatchWithSequenceRange(firstSeq, lastSeq).isDefined) { throw new DuplicateSequenceNumberException(s"Duplicate sequence number for producerId $producerId: (incomingBatch.firstSeq, " + - s"incomingBatch.lastSeq): ($firstSeq, $lastSeq), (lastEntry.firstSeq, lastEntry.lastSeq): " + - s"(${this.firstSeq}, ${this.lastSeq}).") + s"incomingBatch.lastSeq): ($firstSeq, $lastSeq).") } else if (!inSequence(firstSeq, lastSeq)) { throw new OutOfOrderSequenceException(s"Out of order sequence number for producerId $producerId: $firstSeq " + - s"(incoming seq. number), ${this.lastSeq} (current end sequence number)") + s"(incoming seq. number), ${currentEntry.lastSeq} (current end sequence number)") } } } private def inSequence(firstSeq: Int, lastSeq: Int): Boolean = { - firstSeq == this.lastSeq + 1L || (firstSeq == 0 && this.lastSeq == Int.MaxValue) + firstSeq == currentEntry.lastSeq + 1L || (firstSeq == 0 && currentEntry.lastSeq == Int.MaxValue) } private def isFenced(producerEpoch: Short): Boolean = { - producerEpoch < this.producerEpoch + producerEpoch < currentEntry.producerEpoch } def append(batch: RecordBatch): Option[CompletedTxn] = { @@ -166,18 +212,14 @@ private[log] class ProducerAppendInfo(val producerId: Long, // will generally have removed the beginning entries from each producer id validateAppend(epoch, firstSeq, lastSeq) - this.producerEpoch = epoch - this.firstSeq = firstSeq - this.lastSeq = lastSeq - this.maxTimestamp = lastTimestamp - this.lastOffset = lastOffset + currentEntry.addBatchMetadata(epoch, lastSeq, lastOffset, lastSeq - firstSeq, lastTimestamp) - if (currentTxnFirstOffset.isDefined && !isTransactional) + if (currentEntry.currentTxnFirstOffset.isDefined && !isTransactional) throw new InvalidTxnStateException(s"Expected transactional write from producer $producerId") - if (isTransactional && currentTxnFirstOffset.isEmpty) { + if (isTransactional && currentEntry.currentTxnFirstOffset.isEmpty) { val firstOffset = lastOffset - (lastSeq - firstSeq) - currentTxnFirstOffset = Some(firstOffset) + currentEntry.currentTxnFirstOffset = Some(firstOffset) transactions += new TxnMetadata(producerId, firstOffset) } } @@ -187,44 +229,35 @@ private[log] class ProducerAppendInfo(val producerId: Long, offset: Long, timestamp: Long): CompletedTxn = { if (isFenced(producerEpoch)) - throw new ProducerFencedException(s"Invalid producer epoch: $producerEpoch (zombie): ${this.producerEpoch} (current)") + throw new ProducerFencedException(s"Invalid producer epoch: $producerEpoch (zombie): ${currentEntry.producerEpoch} (current)") - if (this.coordinatorEpoch > endTxnMarker.coordinatorEpoch) + if (currentEntry.coordinatorEpoch > endTxnMarker.coordinatorEpoch) throw new TransactionCoordinatorFencedException(s"Invalid coordinator epoch: ${endTxnMarker.coordinatorEpoch} " + - s"(zombie), $coordinatorEpoch (current)") + s"(zombie), ${currentEntry.coordinatorEpoch} (current)") - if (producerEpoch != this.producerEpoch) { - // it is possible that this control record is the first record seen from a new epoch (the producer - // may fail before sending to the partition or the request itself could fail for some reason). In this - // case, we bump the epoch and reset the sequence numbers - this.producerEpoch = producerEpoch - this.firstSeq = RecordBatch.NO_SEQUENCE - this.lastSeq = RecordBatch.NO_SEQUENCE - } else { - // the control record is the last append to the log, so the last offset will be updated to point to it. - // However, the sequence numbers still point to the previous batch, so the duplicate check would no longer - // be correct: it would return the wrong offset. To fix this, we treat the control record as a batch - // of size 1 which uses the last appended sequence number. - this.firstSeq = this.lastSeq - } + // TODO(reviewers): The semantics of the ProducerIdEntry have changed so that now explicitly caches the metadata + // of the last 5 RecordBatches appended to the partition by a producer. So we don't need to care about the offset + // of the control batches anymore, since they never need to de-duped. I think that the new code is simpler, but + // it would be worth discussing whether we want to preserve the old behavior of retaining the offset of the control + // batch in the review. + + // it is possible that this control record is the first record seen from a new epoch, for instance if the coordinator + // times out a transaction. In this case, we bump the epoch and reset the sequence numbers + currentEntry.maybeUpdateEpoch(producerEpoch) - val firstOffset = currentTxnFirstOffset match { + val firstOffset = currentEntry.currentTxnFirstOffset match { case Some(txnFirstOffset) => txnFirstOffset case None => transactions += new TxnMetadata(producerId, offset) offset } - this.lastOffset = offset - this.currentTxnFirstOffset = None - this.maxTimestamp = timestamp - this.coordinatorEpoch = endTxnMarker.coordinatorEpoch + currentEntry.currentTxnFirstOffset = None + currentEntry.coordinatorEpoch = endTxnMarker.coordinatorEpoch CompletedTxn(producerId, firstOffset, offset, endTxnMarker.controlType == ControlRecordType.ABORT) } - def lastEntry: ProducerIdEntry = - ProducerIdEntry(producerId, producerEpoch, lastSeq, lastOffset, lastSeq - firstSeq, maxTimestamp, - coordinatorEpoch, currentTxnFirstOffset) + def latestEntry: ProducerIdEntry = currentEntry def startedTransactions: List[TxnMetadata] = transactions.toList @@ -243,17 +276,17 @@ private[log] class ProducerAppendInfo(val producerId: Long, override def toString: String = { "ProducerAppendInfo(" + s"producerId=$producerId, " + - s"producerEpoch=$producerEpoch, " + - s"firstSequence=$firstSeq, " + - s"lastSequence=$lastSeq, " + - s"currentTxnFirstOffset=$currentTxnFirstOffset, " + - s"coordinatorEpoch=$coordinatorEpoch, " + + s"producerEpoch=${currentEntry.producerEpoch}, " + + s"firstSequence=${currentEntry.firstSeq}, " + + s"lastSequence=${currentEntry.lastSeq}, " + + s"currentTxnFirstOffset=${currentEntry.currentTxnFirstOffset}, " + + s"coordinatorEpoch=${currentEntry.coordinatorEpoch}, " + s"startedTransactions=$transactions)" } } object ProducerStateManager { - private val ProducerSnapshotVersion: Short = 1 + private val ProducerSnapshotVersion: Short = 2 private val VersionField = "version" private val CrcField = "crc" private val ProducerIdField = "producer_id" @@ -265,11 +298,34 @@ object ProducerStateManager { private val ProducerEntriesField = "producer_entries" private val CoordinatorEpochField = "coordinator_epoch" private val CurrentTxnFirstOffsetField = "current_txn_first_offset" + private val RecordBatchMetadataField = "record_batch_metadata" private val VersionOffset = 0 private val CrcOffset = VersionOffset + 2 private val ProducerEntriesOffset = CrcOffset + 4 + + val RecordBatchMetadataSchema = new Schema( + new Field(LastSequenceField, Type.INT32, "Last written sequence of the producer"), + new Field(LastOffsetField, Type.INT64, "Last written offset of the producer"), + new Field(OffsetDeltaField, Type.INT32, "The difference of the last sequence and first sequence in the last written batch"), + new Field(TimestampField, Type.INT64, "Max timestamp from the last written entry") + ) + + val ProducerSnapshotEntrySchemaV2 = new Schema( + new Field(ProducerIdField, Type.INT64, "The producer ID"), + new Field(ProducerEpochField, Type.INT16, "Current epoch of the producer"), + new Field(CoordinatorEpochField, Type.INT32, "The epoch of the last transaction coordinator to send an end transaction marker"), + new Field(CurrentTxnFirstOffsetField, Type.INT64, "The first offset of the on-going transaction (-1 if there is none)"), + new Field(RecordBatchMetadataField, new ArrayOf(RecordBatchMetadataSchema), "The record metadata for up to the last 5 batches appended by the producer") + ) + + val PidSnapshotMapSchemaV2 = new Schema( + new Field(VersionField, Type.INT16, "Version of the snapshot file."), + new Field(CrcField, Type.UNSIGNED_INT32, "CRC of the snapshot data"), + new Field(ProducerEntriesField, new ArrayOf(ProducerSnapshotEntrySchemaV2), "The entries in the producer table") + ) + val ProducerSnapshotEntrySchema = new Schema( new Field(ProducerIdField, Type.INT64, "The producer ID"), new Field(ProducerEpochField, Type.INT16, "Current epoch of the producer"), @@ -279,6 +335,7 @@ object ProducerStateManager { new Field(TimestampField, Type.INT64, "Max timestamp from the last written entry"), new Field(CoordinatorEpochField, Type.INT32, "The epoch of the last transaction coordinator to send an end transaction marker"), new Field(CurrentTxnFirstOffsetField, Type.INT64, "The first offset of the on-going transaction (-1 if there is none)")) + val PidSnapshotMapSchema = new Schema( new Field(VersionField, Type.INT16, "Version of the snapshot file"), new Field(CrcField, Type.UNSIGNED_INT32, "CRC of the snapshot data"), @@ -286,56 +343,109 @@ object ProducerStateManager { def readSnapshot(file: File): Iterable[ProducerIdEntry] = { try { - val buffer = Files.readAllBytes(file.toPath) - val struct = PidSnapshotMapSchema.read(ByteBuffer.wrap(buffer)) - - val version = struct.getShort(VersionField) - if (version != ProducerSnapshotVersion) - throw new CorruptSnapshotException(s"Snapshot contained an unknown file version $version") + val rawBuffer = Files.readAllBytes(file.toPath) + val buffer = ByteBuffer.wrap(rawBuffer) + val version = buffer.getShort() + buffer.rewind() + + val struct = version match { + case 1 => + PidSnapshotMapSchema.read(buffer) + case 2 => + PidSnapshotMapSchemaV2.read(buffer) + case _ => + throw new IllegalArgumentException(s"Snapshot contained unknown file version $version") + } val crc = struct.getUnsignedInt(CrcField) - val computedCrc = Crc32C.compute(buffer, ProducerEntriesOffset, buffer.length - ProducerEntriesOffset) + val computedCrc = Crc32C.compute(rawBuffer, ProducerEntriesOffset, rawBuffer.length - ProducerEntriesOffset) if (crc != computedCrc) throw new CorruptSnapshotException(s"Snapshot is corrupt (CRC is no longer valid). " + s"Stored crc: $crc. Computed crc: $computedCrc") - struct.getArray(ProducerEntriesField).map { producerEntryObj => - val producerEntryStruct = producerEntryObj.asInstanceOf[Struct] - val producerId: Long = producerEntryStruct.getLong(ProducerIdField) - val producerEpoch = producerEntryStruct.getShort(ProducerEpochField) - val seq = producerEntryStruct.getInt(LastSequenceField) - val offset = producerEntryStruct.getLong(LastOffsetField) - val timestamp = producerEntryStruct.getLong(TimestampField) - val offsetDelta = producerEntryStruct.getInt(OffsetDeltaField) - val coordinatorEpoch = producerEntryStruct.getInt(CoordinatorEpochField) - val currentTxnFirstOffset = producerEntryStruct.getLong(CurrentTxnFirstOffsetField) - val newEntry = ProducerIdEntry(producerId, producerEpoch, seq, offset, offsetDelta, timestamp, - coordinatorEpoch, if (currentTxnFirstOffset >= 0) Some(currentTxnFirstOffset) else None) - newEntry - } - } catch { + loadProducerEntries(struct, version) + } + catch { case e: SchemaException => throw new CorruptSnapshotException(s"Snapshot failed schema validation: ${e.getMessage}") } } + private def loadProducerEntries(struct: Struct, version: Short) : Iterable[ProducerIdEntry] = { + version match { + case 1 => + loadProducerEntriesV1(struct) + case 2 => + loadProducerEntriesV2(struct) + } + } + + private def loadProducerEntriesV1(struct: Struct) : Iterable[ProducerIdEntry] = { + struct.getArray(ProducerEntriesField).map { producerEntryObj => + val producerEntryStruct = producerEntryObj.asInstanceOf[Struct] + val producerId = producerEntryStruct.getLong(ProducerIdField) + val producerEpoch = producerEntryStruct.getShort(ProducerEpochField) + val seq = producerEntryStruct.getInt(LastSequenceField) + val offset = producerEntryStruct.getLong(LastOffsetField) + val timestamp = producerEntryStruct.getLong(TimestampField) + val offsetDelta = producerEntryStruct.getInt(OffsetDeltaField) + val coordinatorEpoch = producerEntryStruct.getInt(CoordinatorEpochField) + val currentTxnFirstOffset = producerEntryStruct.getLong(CurrentTxnFirstOffsetField) + val newEntry = ProducerIdEntry(producerId, + mutable.Queue[BatchMetadata](BatchMetadata(seq, offset, offsetDelta, timestamp)), producerEpoch, coordinatorEpoch, + if (currentTxnFirstOffset >= 0) Some(currentTxnFirstOffset) else None) + newEntry + } + } + + private def loadProducerEntriesV2(struct: Struct) : Iterable[ProducerIdEntry] = { + struct.getArray(ProducerEntriesField).map { producerEntryObj => + val producerEntryStruct = producerEntryObj.asInstanceOf[Struct] + val producerId = producerEntryStruct.getLong(ProducerIdField) + val producerEpoch= producerEntryStruct.getShort(ProducerEpochField) + val coordinatorEpoch = producerEntryStruct.getInt(CoordinatorEpochField) + val currentTxnFirstOffset = producerEntryStruct.getLong(CurrentTxnFirstOffsetField) + + val recordBatchMetadata = new mutable.Queue[BatchMetadata]() + producerEntryStruct.getArray(RecordBatchMetadataField).foreach { recordBatchMetadataObj => + val recordBatchMetadataStruct = recordBatchMetadataObj.asInstanceOf[Struct] + val batchMetadata = BatchMetadata(recordBatchMetadataStruct.getInt(LastSequenceField), + recordBatchMetadataStruct.getLong(LastOffsetField), recordBatchMetadataStruct.getInt(OffsetDeltaField), + recordBatchMetadataStruct.getLong(TimestampField)) + recordBatchMetadata.enqueue(batchMetadata) + } + + val newEntry = ProducerIdEntry(producerId, recordBatchMetadata, producerEpoch, coordinatorEpoch, + if (currentTxnFirstOffset >= 0) Some(currentTxnFirstOffset) else None) + newEntry + } + } + private def writeSnapshot(file: File, entries: mutable.Map[Long, ProducerIdEntry]) { - val struct = new Struct(PidSnapshotMapSchema) + val struct = new Struct(PidSnapshotMapSchemaV2) struct.set(VersionField, ProducerSnapshotVersion) struct.set(CrcField, 0L) // we'll fill this after writing the entries val entriesArray = entries.map { case (producerId, entry) => val producerEntryStruct = struct.instance(ProducerEntriesField) + + val batchMetadataArray = entry.batchMetadata.map { batchMetadata => + val batchMetadataStruct = new Struct(RecordBatchMetadataSchema) + batchMetadataStruct + .set(LastSequenceField, batchMetadata.lastSeq) + .set(LastOffsetField, batchMetadata.lastOffset) + .set(OffsetDeltaField, batchMetadata.offsetDelta) + .set(TimestampField, batchMetadata.timestamp) + batchMetadataStruct + }.toArray producerEntryStruct.set(ProducerIdField, producerId) .set(ProducerEpochField, entry.producerEpoch) - .set(LastSequenceField, entry.lastSeq) - .set(LastOffsetField, entry.lastOffset) - .set(OffsetDeltaField, entry.offsetDelta) - .set(TimestampField, entry.timestamp) .set(CoordinatorEpochField, entry.coordinatorEpoch) .set(CurrentTxnFirstOffsetField, entry.currentTxnFirstOffset.getOrElse(-1L)) + .set(RecordBatchMetadataField, batchMetadataArray) producerEntryStruct }.toArray + struct.set(ProducerEntriesField, entriesArray) val buffer = ByteBuffer.allocate(struct.sizeOf) @@ -472,7 +582,7 @@ class ProducerStateManager(val topicPartition: TopicPartition, } private def isProducerExpired(currentTimeMs: Long, producerIdEntry: ProducerIdEntry): Boolean = - producerIdEntry.currentTxnFirstOffset.isEmpty && currentTimeMs - producerIdEntry.timestamp >= maxProducerIdExpirationMs + producerIdEntry.currentTxnFirstOffset.isEmpty && currentTimeMs - producerIdEntry.lastTimestamp >= maxProducerIdExpirationMs /** * Expire any producer ids which have been idle longer than the configured maximum expiration timeout. @@ -508,7 +618,7 @@ class ProducerStateManager(val topicPartition: TopicPartition, } def prepareUpdate(producerId: Long, loadingFromLog: Boolean): ProducerAppendInfo = - new ProducerAppendInfo(producerId, lastEntry(producerId).getOrElse(ProducerIdEntry.Empty), validateSequenceNumbers, + new ProducerAppendInfo(producerId, lastEntry(producerId).getOrElse(ProducerIdEntry.empty(producerId)), validateSequenceNumbers, loadingFromLog) /** @@ -520,7 +630,7 @@ class ProducerStateManager(val topicPartition: TopicPartition, trace(s"Updated producer ${appendInfo.producerId} state to $appendInfo") - val entry = appendInfo.lastEntry + val entry = appendInfo.latestEntry producers.put(appendInfo.producerId, entry) appendInfo.startedTransactions.foreach { txn => ongoingTxns.put(txn.firstOffset.messageOffset, txn) diff --git a/core/src/main/scala/kafka/tools/DumpLogSegments.scala b/core/src/main/scala/kafka/tools/DumpLogSegments.scala index 025617fc6b3ea..86d19c7c9aae8 100755 --- a/core/src/main/scala/kafka/tools/DumpLogSegments.scala +++ b/core/src/main/scala/kafka/tools/DumpLogSegments.scala @@ -154,10 +154,10 @@ object DumpLogSegments { private def dumpPidSnapshot(file: File): Unit = { try { - ProducerStateManager.readSnapshot(file).foreach { entry=> - println(s"producerId: ${entry.producerId} producerEpoch: ${entry.producerEpoch} lastSequence: ${entry.lastSeq} " + - s"lastOffset: ${entry.lastOffset} offsetDelta: ${entry.offsetDelta} lastTimestamp: ${entry.timestamp} " + - s"coordinatorEpoch: ${entry.coordinatorEpoch} currentTxnFirstOffset: ${entry.currentTxnFirstOffset}") + ProducerStateManager.readSnapshot(file).foreach { entry => + println(s"producerId: ${entry.producerId} producerEpoch: ${entry.producerEpoch} " + + s"coordinatorEpoch: ${entry.coordinatorEpoch} currentTxnFirstOffset: ${entry.currentTxnFirstOffset} " + + s"cachedMetadata: ${entry.batchMetadata}") } } catch { case e: CorruptSnapshotException => diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties index d394a2abc9495..80966c3f94089 100644 --- a/core/src/test/resources/log4j.properties +++ b/core/src/test/resources/log4j.properties @@ -19,7 +19,7 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n log4j.logger.kafka=ERROR -log4j.logger.org.apache.kafka=ERROR +log4j.logger.org.apache.kafka.clients.producer.internals=TRACE # zkclient can be verbose, during debugging it is common to adjust it separately log4j.logger.org.I0Itec.zkclient.ZkClient=WARN diff --git a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala index 2a07532abaaa3..a0fa48110b672 100644 --- a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala @@ -317,7 +317,8 @@ class LogSegmentTest { // recover again, but this time assuming the transaction from pid2 began on a previous segment stateManager = new ProducerStateManager(topicPartition, logDir) - stateManager.loadProducerEntry(ProducerIdEntry(pid2, producerEpoch, 10, 90L, 5, RecordBatch.NO_TIMESTAMP, 0, Some(75L))) + stateManager.loadProducerEntry(ProducerIdEntry(pid2, mutable.Queue[BatchMetadata](BatchMetadata(10, 90L, 5, RecordBatch.NO_TIMESTAMP)), + producerEpoch, 0, Some(75L))) segment.recover(stateManager) assertEquals(108L, stateManager.mapEndOffset) diff --git a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala index bb41380160287..f2dee315fbab5 100644 --- a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala +++ b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala @@ -95,7 +95,14 @@ class ProducerStateManagerTest extends JUnitSuite { val lastEntry = maybeLastEntry.get assertEquals(epoch, lastEntry.producerEpoch) - assertEquals(0, lastEntry.firstSeq) + // TODO(reviewers): The semantics of the producer state manager have been changed so that we store the + // last N batches. As such, the ProducerIdEntry.firstSeq returns the first seq of the first batch in the cache. + // The ProducerIdEntry.lastSeq is the last seq of the last batch in the cache. As a result, during wraparound + // the firstSeq could be greater than the lastSeq. + // + // This seems reasonable, but it is worth at least discussing the new semantics of these methods when reviewing + // the new code. + assertEquals(Int.MaxValue, lastEntry.firstSeq) assertEquals(0, lastEntry.lastSeq) } @@ -158,7 +165,7 @@ class ProducerStateManagerTest extends JUnitSuite { val producerEpoch = 0.toShort val offset = 992342L val seq = 0 - val producerAppendInfo = new ProducerAppendInfo(producerId, ProducerIdEntry.Empty, validateSequenceNumbers = true, + val producerAppendInfo = new ProducerAppendInfo(producerId, ProducerIdEntry.empty(producerId), validateSequenceNumbers = true, loadingFromLog = false) producerAppendInfo.append(producerEpoch, seq, seq, time.milliseconds(), offset, isTransactional = true) @@ -175,7 +182,7 @@ class ProducerStateManagerTest extends JUnitSuite { val producerEpoch = 0.toShort val offset = 992342L val seq = 0 - val producerAppendInfo = new ProducerAppendInfo(producerId, ProducerIdEntry.Empty, validateSequenceNumbers = true, + val producerAppendInfo = new ProducerAppendInfo(producerId, ProducerIdEntry.empty(producerId), validateSequenceNumbers = true, loadingFromLog = false) producerAppendInfo.append(producerEpoch, seq, seq, time.milliseconds(), offset, isTransactional = true) @@ -198,21 +205,21 @@ class ProducerStateManagerTest extends JUnitSuite { val appendInfo = stateManager.prepareUpdate(producerId, loadingFromLog = false) appendInfo.append(producerEpoch, 1, 5, time.milliseconds(), 20L, isTransactional = true) - var lastEntry = appendInfo.lastEntry + var lastEntry = appendInfo.latestEntry assertEquals(producerEpoch, lastEntry.producerEpoch) - assertEquals(1, lastEntry.firstSeq) + assertEquals(0, lastEntry.firstSeq) assertEquals(5, lastEntry.lastSeq) - assertEquals(16L, lastEntry.firstOffset) + assertEquals(9L, lastEntry.firstOffset) assertEquals(20L, lastEntry.lastOffset) assertEquals(Some(16L), lastEntry.currentTxnFirstOffset) assertEquals(List(new TxnMetadata(producerId, 16L)), appendInfo.startedTransactions) appendInfo.append(producerEpoch, 6, 10, time.milliseconds(), 30L, isTransactional = true) - lastEntry = appendInfo.lastEntry + lastEntry = appendInfo.latestEntry assertEquals(producerEpoch, lastEntry.producerEpoch) - assertEquals(6, lastEntry.firstSeq) + assertEquals(0, lastEntry.firstSeq) assertEquals(10, lastEntry.lastSeq) - assertEquals(26L, lastEntry.firstOffset) + assertEquals(9L, lastEntry.firstOffset) assertEquals(30L, lastEntry.lastOffset) assertEquals(Some(16L), lastEntry.currentTxnFirstOffset) assertEquals(List(new TxnMetadata(producerId, 16L)), appendInfo.startedTransactions) @@ -224,12 +231,13 @@ class ProducerStateManagerTest extends JUnitSuite { assertEquals(40L, completedTxn.lastOffset) assertFalse(completedTxn.isAborted) - lastEntry = appendInfo.lastEntry + lastEntry = appendInfo.latestEntry assertEquals(producerEpoch, lastEntry.producerEpoch) - assertEquals(10, lastEntry.firstSeq) + // verify that appending the transaction marker doesn't affect the metadata of the cached record batches. + assertEquals(0, lastEntry.firstSeq) assertEquals(10, lastEntry.lastSeq) - assertEquals(40L, lastEntry.firstOffset) - assertEquals(40L, lastEntry.lastOffset) + assertEquals(9L, lastEntry.firstOffset) + assertEquals(30L, lastEntry.lastOffset) assertEquals(coordinatorEpoch, lastEntry.coordinatorEpoch) assertEquals(None, lastEntry.currentTxnFirstOffset) assertEquals(List(new TxnMetadata(producerId, 16L)), appendInfo.startedTransactions) From 0a5e9125188ff112e0363fba508bfda5546dc71e Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 22 Aug 2017 17:10:39 -0700 Subject: [PATCH 03/40] Change the client side code so that the sequence numbers are assigned and incremented during drain. If a batch is retried, it's sequence number is unset during the completion handler. If the first inflight batch returns an error, the next sequence to assign is reset to the last ack'd sequence + 1. --- .../producer/internals/ProducerBatch.java | 29 ++++------ .../producer/internals/RecordAccumulator.java | 55 +++++-------------- .../clients/producer/internals/Sender.java | 19 +++++-- .../internals/TransactionManager.java | 17 +++--- .../common/record/MemoryRecordsBuilder.java | 12 +++- .../producer/internals/SenderTest.java | 9 +-- .../internals/TransactionManagerTest.java | 35 +++++++----- 7 files changed, 85 insertions(+), 91 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java index e7d3417d1af55..8ca82846c4188 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java @@ -235,13 +235,12 @@ public Deque split(int splitBatchSize) { assert thunkIter.hasNext(); Thunk thunk = thunkIter.next(); if (batch == null) - batch = createBatchOffAccumulatorForRecord(record, splitBatchSize, baseSequence(), isTransactional()); + batch = createBatchOffAccumulatorForRecord(record, splitBatchSize); // A newly created batch can always host the first message. if (!batch.tryAppendForSplit(record.timestamp(), record.key(), record.value(), record.headers(), thunk)) { - int nextSequence = batch.baseSequence() == RecordBatch.NO_SEQUENCE ? RecordBatch.NO_SEQUENCE : batch.baseSequence() + batch.recordCount; batches.add(batch); - batch = createBatchOffAccumulatorForRecord(record, splitBatchSize, nextSequence, batch.isTransactional()); + batch = createBatchOffAccumulatorForRecord(record, splitBatchSize); batch.tryAppendForSplit(record.timestamp(), record.key(), record.value(), record.headers(), thunk); } } @@ -255,7 +254,7 @@ public Deque split(int splitBatchSize) { return batches; } - private ProducerBatch createBatchOffAccumulatorForRecord(Record record, int batchSize, int sequence, boolean isTransactional) { + private ProducerBatch createBatchOffAccumulatorForRecord(Record record, int batchSize) { int initialSize = Math.max(AbstractRecords.estimateSizeInBytesUpperBound(magic(), recordsBuilder.compressionType(), record.key(), record.value(), record.headers()), batchSize); ByteBuffer buffer = ByteBuffer.allocate(initialSize); @@ -264,11 +263,8 @@ private ProducerBatch createBatchOffAccumulatorForRecord(Record record, int batc // for the newly created batch. This will be set when the batch is dequeued for sending (which is consistent // with how normal batches are handled). MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, magic(), recordsBuilder.compressionType(), - TimestampType.CREATE_TIME, 0L, RecordBatch.NO_TIMESTAMP, RecordBatch.NO_PRODUCER_ID, - RecordBatch.NO_PRODUCER_EPOCH, sequence, isTransactional, RecordBatch.NO_PARTITION_LEADER_EPOCH); - ProducerBatch batch = new ProducerBatch(topicPartition, builder, this.createdMs, true); - batch.countSequenceNumber(); - return batch; + TimestampType.CREATE_TIME, 0L); + return new ProducerBatch(topicPartition, builder, this.createdMs, true); } public boolean isCompressed() { @@ -380,8 +376,12 @@ public boolean isFull() { return recordsBuilder.isFull(); } - public void setProducerState(ProducerIdAndEpoch producerIdAndEpoch) { - recordsBuilder.setProducerState(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch); + public void setProducerState(ProducerIdAndEpoch producerIdAndEpoch, int baseSequence, boolean isTransactional) { + recordsBuilder.setProducerState(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, baseSequence, isTransactional); + } + + public void unsetProducerState() { + recordsBuilder.unsetProducerState(); } /** @@ -448,11 +448,4 @@ private boolean isTransactional() { return recordsBuilder.isTransactional(); } - void countSequenceNumber() { - countedTowardSequence = true; - } - - boolean hasBeenCountedTowardSequence() { - return countedTowardSequence; - } } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index 618cc9b0a1ead..fc8638239dffa 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -48,7 +48,6 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; @@ -241,15 +240,6 @@ private MemoryRecordsBuilder recordsBuilder(TopicPartition topicPartition, ByteB throw new UnsupportedVersionException("Attempting to use idempotence with a broker which does not " + "support the required message format (v2). The broker must be version 0.11 or later."); } - if (transactionManager != null) { - int sequenceNumber = transactionManager.sequenceNumber(topicPartition); - MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, maxUsableMagic, compression, TimestampType.CREATE_TIME, 0L, - RecordBatch.NO_TIMESTAMP, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, sequenceNumber, - transactionManager.isTransactional(), RecordBatch.NO_PARTITION_LEADER_EPOCH); - log.debug("ProducerId {}: Assigned sequence number to {} to newly created batch for partition {}", - transactionManager.producerIdAndEpoch().producerId, sequenceNumber, topicPartition); - return builder; - } return MemoryRecords.builder(buffer, maxUsableMagic, compression, TimestampType.CREATE_TIME, 0L); } @@ -268,8 +258,6 @@ private RecordAppendResult tryAppend(TopicPartition tp, long timestamp, byte[] k FutureRecordMetadata future = last.tryAppend(timestamp, key, value, headers, callback, time.milliseconds()); if (future == null) { last.closeForRecordAppends(); - if (transactionManager != null) - transactionManager.maybeCountSequenceNumber(last); } else { return new RecordAppendResult(future, deque.size() > 1 || last.isFull(), false); } @@ -323,21 +311,6 @@ public void reenqueue(ProducerBatch batch, long now) { Deque deque = getOrCreateDeque(batch.topicPartition); synchronized (deque) { deque.addFirst(batch); - maybeReorderBySequenceNumber(deque); - } - } - - void maybeReorderBySequenceNumber(Deque deque) { - if (transactionManager != null) { - ArrayList copy = new ArrayList<>(deque); - Collections.sort(copy, new Comparator() { - @Override - public int compare(ProducerBatch o1, ProducerBatch o2) { - return o1.baseSequence() - o2.baseSequence(); - } - }); - deque.clear(); - deque.addAll(copy); } } @@ -362,7 +335,6 @@ public int splitAndReenqueue(ProducerBatch bigBatch) { partitionDequeue.addFirst(batch); } } - maybeReorderBySequenceNumber(partitionDequeue); return numSplitBatches; } @@ -487,6 +459,7 @@ public Map> drain(Cluster cluster, break; } else { ProducerIdAndEpoch producerIdAndEpoch = null; + boolean isTransactional = false; if (transactionManager != null) { if (!transactionManager.isSendToPartitionAllowed(tp)) break; @@ -495,31 +468,31 @@ public Map> drain(Cluster cluster, if (!producerIdAndEpoch.isValid()) // we cannot send the batch until we have refreshed the producer id break; + + isTransactional = transactionManager.isTransactional(); } ProducerBatch batch = deque.pollFirst(); - if (producerIdAndEpoch != null && !batch.inRetry()) { + if (producerIdAndEpoch != null) { // If the batch is in retry, then we should not change the producer id and // sequence number, since this may introduce duplicates. In particular, // the previous attempt may actually have been accepted, and if we change // the producer id and sequence here, this attempt will also be accepted, // causing a duplicate. - batch.setProducerState(producerIdAndEpoch); + + // set the sequence to be the next sequence here, increment the next + // sequence by the record count of the batch. + // + // If a batch errors out fatally, then the next sequence will be set to the sequence + // of that batch in the error handling step. Future batches will fail with an + // out of sequence error. Those batches will be 'reset' (retry bit cleared, etc.) and reenqueued. + // When they come here they will inherit the sequence of the lost batch and proceed. + batch.setProducerState(producerIdAndEpoch, transactionManager.sequenceNumber(batch.topicPartition), isTransactional); + transactionManager.incrementSequenceNumber(batch.topicPartition, batch.recordCount); log.debug("Assigned producerId {} and producerEpoch {} to batch with sequence " + "{} being sent to partition {}", producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, batch.baseSequence(), tp); } - if (transactionManager != null) { - // We close the batch for writing once it is full. We would have incremented - // the sequence number at that point so that the next batch would get the - // correct next sequence. - // - // However, it is possible that the linger has expired and we are just sending - // an incompleted batch which was not yet closed for writing. - // - // In this case, we need to increment the sequence number here. - transactionManager.maybeCountSequenceNumber(batch); - } batch.close(); size += batch.records().sizeInBytes(); ready.add(batch); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index a816c2eeea1c0..42d329a3275ca 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -493,6 +493,9 @@ private void handleProduceResponse(ClientResponse response, Map 1 && (batch.magic() >= RecordBatch.MAGIC_VALUE_V2 || batch.isCompressed())) { // If the batch is too large, we split the batch and send the split batches again. We do not decrement @@ -554,7 +557,19 @@ else if (error == Errors.CLUSTER_AUTHORIZATION_FAILED) this.accumulator.unmutePartition(batch.topicPartition); } + private void maybeResetNextSequenceForPartition(TopicPartition topicPartition, int nextSequence) { + if (transactionManager != null) + if (transactionManager.isNextSequence(topicPartition, nextSequence)) + // If we are retrying the first in flight request, then reset the next sequence number to be the + // the last ack'd sequence + 1. Subsequent batches will get the succeeding sequence numbers. + transactionManager.setNextSequence(topicPartition, nextSequence); + + } + private void reenqueueBatch(ProducerBatch batch, long currentTimeMs) { + if (transactionManager != null) + // Reset the sequence number for the retried batch. It will be set again on the next drain. + batch.unsetProducerState(); this.accumulator.reenqueue(batch, currentTimeMs); this.sensors.recordRetries(batch.topicPartition.topic(), batch.recordCount); } @@ -595,10 +610,6 @@ private void failBatch(ProducerBatch batch, long baseOffset, long logAppendTime, } else if (transactionManager.isTransactional()) { transactionManager.transitionToAbortableError(exception); } - - if (transactionManager.hasProducerId()) { - - } } batch.done(baseOffset, logAppendTime, exception); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index bc83a86c62a8d..d77576a695a44 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -365,6 +365,7 @@ synchronized void resetProducerId() { "You must either abort the ongoing transaction or reinitialize the transactional producer instead"); setProducerIdAndEpoch(ProducerIdAndEpoch.NONE); this.sequenceNumbers.clear(); + this.lastAckedSequence.clear(); } /** @@ -379,15 +380,6 @@ synchronized Integer sequenceNumber(TopicPartition topicPartition) { return currentSequenceNumber; } - synchronized void maybeCountSequenceNumber(ProducerBatch batch) { - if (batch.hasBeenCountedTowardSequence()) - return; - incrementSequenceNumber(batch.topicPartition, batch.recordCount); - log.debug("ProducerId: {}; topic-partition: {}; incremented sequence number to {}", - producerIdAndEpoch().producerId, batch.topicPartition, sequenceNumber(batch.topicPartition)); - batch.countSequenceNumber(); - } - synchronized void incrementSequenceNumber(TopicPartition topicPartition, int increment) { Integer currentSequenceNumber = sequenceNumbers.get(topicPartition); if (currentSequenceNumber == null) @@ -413,6 +405,13 @@ synchronized boolean isNextSequence(TopicPartition topicPartition, int sequence) (lastAckedSequence.containsKey(topicPartition) && (sequence - lastAckedSequence.get(topicPartition) == 1)); } + synchronized void setNextSequence(TopicPartition topicPartition, int sequence) { + if (!sequenceNumbers.containsKey(topicPartition) && sequence != 0) + throw new IllegalStateException("Trying to set the sequence number for " + topicPartition + " to " + sequence + + ", but the sequence number was never set for this partition."); + sequenceNumbers.put(topicPartition, sequence); + } + synchronized TxnRequestHandler nextRequestHandler(boolean hasIncompleteBatches) { if (!newPartitionsInTransaction.isEmpty()) enqueueRequest(addPartitionsToTransactionHandler()); diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java index 6f7337b28afdb..9247440c82562 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java @@ -238,7 +238,7 @@ public RecordsInfo info() { } } - public void setProducerState(long producerId, short producerEpoch) { + public void setProducerState(long producerId, short producerEpoch, int baseSequence, boolean isTransactional) { if (isClosed()) { // Sequence numbers are assigned when the batch is closed while the accumulator is being drained. // If the resulting ProduceRequest to the partition leader failed for a retriable error, the batch will @@ -248,6 +248,8 @@ public void setProducerState(long producerId, short producerEpoch) { } this.producerId = producerId; this.producerEpoch = producerEpoch; + this.baseSequence = baseSequence; + this.isTransactional = isTransactional; } public void overrideLastOffset(long lastOffset) { @@ -277,6 +279,14 @@ public void abort() { aborted = true; } + public void unsetProducerState() { + this.producerId = RecordBatch.NO_PRODUCER_ID; + this.producerEpoch = RecordBatch.NO_PRODUCER_EPOCH; + this.baseSequence = RecordBatch.NO_SEQUENCE; + this.isTransactional = false; + builtRecords = null; + } + public void close() { if (aborted) throw new IllegalStateException("Cannot close MemoryRecordsBuilder as it has already been aborted"); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 7fb518844edc1..9474606f25cbb 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -906,11 +906,12 @@ private void testSplitBatchAndSend(TransactionManager txnManager, responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.MESSAGE_TOO_LARGE)); client.respond(new ProduceResponse(responseMap)); sender.run(time.milliseconds()); // split and reenqueue + assertEquals("The next sequence should have been reset to 0", 0, txnManager.sequenceNumber(tp).longValue()); // The compression ratio should have been improved once. assertEquals(CompressionType.GZIP.rate - CompressionRatioEstimator.COMPRESSION_RATIO_IMPROVING_STEP, CompressionRatioEstimator.estimation(topic, CompressionType.GZIP), 0.01); - sender.run(time.milliseconds()); // send produce request - assertEquals("The next sequence number should be 2", 2, txnManager.sequenceNumber(tp).longValue()); + sender.run(time.milliseconds()); // send the first produce request + assertEquals("The next sequence number should be 1", 1, txnManager.sequenceNumber(tp).longValue()); assertFalse("The future shouldn't have been done.", f1.isDone()); assertFalse("The future shouldn't have been done.", f2.isDone()); id = client.requests().peek().destination(); @@ -925,11 +926,11 @@ private void testSplitBatchAndSend(TransactionManager txnManager, sender.run(time.milliseconds()); // receive assertTrue("The future should have been done.", f1.isDone()); - assertEquals("The next sequence number should be 2", 2, txnManager.sequenceNumber(tp).longValue()); + assertEquals("The next sequence number should still be 1", 1, txnManager.sequenceNumber(tp).longValue()); assertEquals("The last ack'd sequence number should be 0", 0, txnManager.lastAckedSequence(tp)); assertFalse("The future shouldn't have been done.", f2.isDone()); assertEquals("Offset of the first message should be 0", 0L, f1.get().offset()); - sender.run(time.milliseconds()); // send produce request + sender.run(time.milliseconds()); // send the seconcd produce request id = client.requests().peek().destination(); assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey()); node = new Node(Integer.valueOf(id), "localhost", 0); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java index 53bba1c13fdad..fb0c4047fd52e 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java @@ -67,6 +67,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -1166,11 +1167,11 @@ public void testRecoveryFromAbortableErrorProduceRequestInRetry() throws Excepti assertFalse(transactionManager.isPartitionAdded(unauthorizedPartition)); assertFalse(authorizedTopicProduceFuture.isDone()); - prepareProduceResponse(Errors.NONE, pid, epoch); - sender.run(time.milliseconds()); - assertFutureFailed(unauthorizedTopicProduceFuture); + sender.run(time.milliseconds()); // will abort produce requests to both the authorized and unauthorized partitions since the transaction has been aborted. assertTrue(authorizedTopicProduceFuture.isDone()); - assertNotNull(authorizedTopicProduceFuture.get()); + assertTrue(unauthorizedTopicProduceFuture.isDone()); + assertFutureFailed(unauthorizedTopicProduceFuture); + assertFutureFailed(authorizedTopicProduceFuture); prepareEndTxnResponse(Errors.NONE, TransactionResult.ABORT, pid, epoch); transactionManager.beginAbort(); @@ -1682,8 +1683,7 @@ public void testAbortResendsProduceRequestIfRetried() throws Exception { TransactionalRequestResult abortResult = transactionManager.beginAbort(); - // we should resend the ProduceRequest before aborting - prepareProduceResponse(Errors.NONE, producerId, producerEpoch); + // the produce response will be aborted once the transaction begins abort. prepareEndTxnResponse(Errors.NONE, TransactionResult.ABORT, producerId, producerEpoch); sender.run(time.milliseconds()); // Resend ProduceRequest @@ -1693,8 +1693,9 @@ public void testAbortResendsProduceRequestIfRetried() throws Exception { assertTrue(abortResult.isSuccessful()); assertTrue(transactionManager.isReady()); // make sure we are ready for a transaction now. - RecordMetadata recordMetadata = responseFuture.get(); - assertEquals(tp0.topic(), recordMetadata.topic()); + // ensure that the future failed + assertTrue(responseFuture.isDone()); + assertFutureFailed(responseFuture); } @Test @@ -1946,7 +1947,7 @@ public void testRaiseErrorWhenNoPartitionsPendingOnDrain() throws InterruptedExc assertTrue(drainedBatches.get(node1.id()).isEmpty()); } - @Test + @Test(expected = ExecutionException.class) public void resendFailedProduceRequestAfterAbortableError() throws Exception { final long pid = 13131L; final short epoch = 1; @@ -1960,17 +1961,23 @@ public void resendFailedProduceRequestAfterAbortableError() throws Exception { prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); prepareProduceResponse(Errors.NOT_LEADER_FOR_PARTITION, pid, epoch); - sender.run(time.milliseconds()); // AddPartitions - sender.run(time.milliseconds()); // Produce + sender.run(time.milliseconds()); // Send AddPartitions and receive response + sender.run(time.milliseconds()); // Send produce request and receive NOT_LEADER_FOR_PARTITION response. assertFalse(responseFuture.isDone()); transactionManager.transitionToAbortableError(new KafkaException()); - prepareProduceResponse(Errors.NONE, pid, epoch); - sender.run(time.milliseconds()); + Deque unsentBatches = accumulator.batches().get(tp0); + + // The one unsent batch should have a reset sequence and should still be open. + assertEquals(1, unsentBatches.size()); + assertEquals(RecordBatch.NO_SEQUENCE, unsentBatches.getFirst().baseSequence()); + assertFalse(unsentBatches.getFirst().isClosed()); + + sender.run(time.milliseconds()); // abort the unsent batch. assertTrue(responseFuture.isDone()); - assertNotNull(responseFuture.get()); + responseFuture.get(); // should throw the exception which caused the transaction to be aborted. } @Test From f4d39a8361cc8bde17b2c9c7006ca425164a9068 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 23 Aug 2017 18:33:54 -0700 Subject: [PATCH 04/40] WIP --- .../clients/producer/internals/ProducerBatch.java | 6 +----- .../kafka/clients/producer/internals/Sender.java | 2 +- core/src/main/scala/kafka/log/LogCleaner.scala | 4 ++-- .../main/scala/kafka/log/ProducerStateManager.scala | 10 ++++++++++ .../integration/kafka/api/ProducerBounceTest.scala | 2 +- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java index 8ca82846c4188..e00a253462f47 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java @@ -380,7 +380,7 @@ public void setProducerState(ProducerIdAndEpoch producerIdAndEpoch, int baseSequ recordsBuilder.setProducerState(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, baseSequence, isTransactional); } - public void unsetProducerState() { + public void reopenBatchAndResetProducerState() { recordsBuilder.unsetProducerState(); } @@ -444,8 +444,4 @@ public int baseSequence() { return recordsBuilder.baseSequence(); } - private boolean isTransactional() { - return recordsBuilder.isTransactional(); - } - } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 42d329a3275ca..656e6587c374a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -569,7 +569,7 @@ private void maybeResetNextSequenceForPartition(TopicPartition topicPartition, i private void reenqueueBatch(ProducerBatch batch, long currentTimeMs) { if (transactionManager != null) // Reset the sequence number for the retried batch. It will be set again on the next drain. - batch.unsetProducerState(); + batch.reopenBatchAndResetProducerState(); this.accumulator.reenqueue(batch, currentTimeMs); this.sensors.recordRetries(batch.topicPartition.topic(), batch.recordCount); } diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala index 4f53b41df4fc0..09b80f03a35f7 100644 --- a/core/src/main/scala/kafka/log/LogCleaner.scala +++ b/core/src/main/scala/kafka/log/LogCleaner.scala @@ -513,9 +513,9 @@ private[log] class Cleaner(val id: Int, // note that we will never delete a marker until all the records from that transaction are removed. discardBatchRecords = shouldDiscardBatch(batch, transactionMetadata, retainTxnMarkers = retainDeletes) - // check if the batch contains the last sequence number for the producer. if so, we cannot + // check if the batch metadata is currently being cached. if so, we cannot // remove the batch just yet or the producer may see an out of sequence error. - if (batch.hasProducerId && activeProducers.get(batch.producerId).exists(_.lastSeq == batch.lastSequence)) + if (batch.hasProducerId && activeProducers.get(batch.producerId).exists(_.hasBatchWithSequenceRange(batch.baseSequence, batch.lastSequence).nonEmpty)) BatchRetention.RETAIN_EMPTY else if (discardBatchRecords) BatchRetention.DELETE diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index 6318a1bfd9bdf..f768c35d66a02 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -96,6 +96,14 @@ private[log] case class ProducerIdEntry(producerId: Long, batchMetadata: mutable false } + def removeBatchesOlderThan(offset: Long) = { + val retainedMetadata = batchMetadata.filter(_.lastOffset >= offset) + if (retainedMetadata.size != batchMetadata.size) { + batchMetadata.clear() + retainedMetadata.foreach(batchMetadata.enqueue(_)) + } + } + def duplicateOf(batch: RecordBatch): Option[BatchMetadata] = { if (batch.producerEpoch() != producerEpoch) return None @@ -103,6 +111,7 @@ private[log] case class ProducerIdEntry(producerId: Long, batchMetadata: mutable hasBatchWithSequenceRange(batch.baseSequence(), batch.lastSequence()) } + // Return the batch metadata of the cached batch having the exact sequence range, if any. def hasBatchWithSequenceRange(firstSeq: Int, lastSeq:Int) : Option[BatchMetadata] = { val duplicate = batchMetadata.filter{ case(metadata) => firstSeq == metadata.firstSeq && lastSeq == metadata.lastSeq @@ -672,6 +681,7 @@ class ProducerStateManager(val topicPartition: TopicPartition, def oldestSnapshotOffset: Option[Long] = oldestSnapshotFile.map(file => offsetFromFilename(file.getName)) private def isProducerRetained(producerIdEntry: ProducerIdEntry, logStartOffset: Long): Boolean = { + producerIdEntry.removeBatchesOlderThan(logStartOffset) producerIdEntry.lastOffset >= logStartOffset } diff --git a/core/src/test/scala/integration/kafka/api/ProducerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ProducerBounceTest.scala index a11972eec3dd3..1bde7b1211093 100644 --- a/core/src/test/scala/integration/kafka/api/ProducerBounceTest.scala +++ b/core/src/test/scala/integration/kafka/api/ProducerBounceTest.scala @@ -122,7 +122,7 @@ class ProducerBounceTest extends KafkaServerTestHarness { val producerConfig = new Properties() producerConfig.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true") - producerConfig.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1") + producerConfig.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "5") val producerConfigWithCompression = new Properties() producerConfigWithCompression ++= producerConfig producerConfigWithCompression.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "lz4") From a4ef1c4f9539fdae509bcf73ffa5701c9502e463 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 24 Aug 2017 23:23:50 -0700 Subject: [PATCH 05/40] Implemented log cleaning functionality with tests --- .../kafka/clients/producer/KafkaProducer.java | 2 +- .../kafka/log/ProducerStateManager.scala | 5 +- core/src/test/resources/log4j.properties | 1 - .../kafka/api/TransactionsBounceTest.scala | 22 ++++- .../scala/unit/kafka/log/LogCleanerTest.scala | 80 +++++++++++-------- .../scala/unit/kafka/utils/TestUtils.scala | 5 +- .../tools/TransactionalMessageCopier.java | 6 +- 7 files changed, 77 insertions(+), 44 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index 3ef0a74d9988d..1130a352defe8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -484,7 +484,7 @@ private static int configureRetries(ProducerConfig config, boolean idempotenceEn private static int configureInflightRequests(ProducerConfig config, boolean idempotenceEnabled) { if (idempotenceEnabled && 5 < config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION)) { throw new ConfigException("Must set " + ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION + " to at most 5" + - "to use the idempotent producer. Otherwise we cannot guarantee idempotence."); + " to use the idempotent producer. Otherwise we cannot guarantee idempotence."); } return config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION); } diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index f768c35d66a02..78a12130f2547 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -48,6 +48,7 @@ private[log] case class TxnMetadata(producerId: Long, var firstOffset: LogOffset } private[log] object ProducerIdEntry { + private[log] val NumBatchesToRetain = 5 def empty(producerId: Long) = new ProducerIdEntry(producerId, mutable.Queue[BatchMetadata](), RecordBatch.NO_PRODUCER_EPOCH, -1, None) } @@ -69,7 +70,7 @@ private[log] case class ProducerIdEntry(producerId: Long, batchMetadata: mutable var producerEpoch: Short, var coordinatorEpoch: Int, var currentTxnFirstOffset: Option[Long]) { - private val NumBatchesToRetain = 5 + def firstSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.front.firstSeq def firstOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.front.firstOffset @@ -81,7 +82,7 @@ private[log] case class ProducerIdEntry(producerId: Long, batchMetadata: mutable def addBatchMetadata(producerEpoch: Short, lastSeq: Int, lastOffset: Long, offsetDelta: Int, timestamp: Long) = { maybeUpdateEpoch(producerEpoch) - if (batchMetadata.size == NumBatchesToRetain) + if (batchMetadata.size == ProducerIdEntry.NumBatchesToRetain) batchMetadata.dequeue() batchMetadata.enqueue(BatchMetadata(lastSeq, lastOffset, offsetDelta, timestamp)) diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties index 80966c3f94089..9ec020eeb0194 100644 --- a/core/src/test/resources/log4j.properties +++ b/core/src/test/resources/log4j.properties @@ -19,7 +19,6 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n log4j.logger.kafka=ERROR -log4j.logger.org.apache.kafka.clients.producer.internals=TRACE # zkclient can be verbose, during debugging it is common to adjust it separately log4j.logger.org.I0Itec.zkclient.ZkClient=WARN diff --git a/core/src/test/scala/integration/kafka/api/TransactionsBounceTest.scala b/core/src/test/scala/integration/kafka/api/TransactionsBounceTest.scala index 810f48192f8f5..1a85d340c634d 100644 --- a/core/src/test/scala/integration/kafka/api/TransactionsBounceTest.scala +++ b/core/src/test/scala/integration/kafka/api/TransactionsBounceTest.scala @@ -24,12 +24,15 @@ import kafka.server.KafkaConfig import kafka.utils.{ShutdownableThread, TestUtils} import org.apache.kafka.clients.consumer.ConsumerConfig import org.apache.kafka.clients.producer.internals.ErrorLoggingCallback +import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.protocol.SecurityProtocol import org.junit.Test import scala.collection.JavaConverters._ import org.junit.Assert._ +import scala.collection.mutable + class TransactionsBounceTest extends KafkaServerTestHarness { private val producerBufferSize = 65536 @@ -76,12 +79,12 @@ class TransactionsBounceTest extends KafkaServerTestHarness { // basic idea is to seed a topic with 10000 records, and copy it transactionally while bouncing brokers // constantly through the period. val consumerGroup = "myGroup" - val numInputRecords = 5000 + val numInputRecords = 10000 createTopics() TestUtils.seedTopicWithNumberedRecords(inputTopic, numInputRecords, servers) val consumer = createConsumerAndSubscribeToTopics(consumerGroup, List(inputTopic)) - val producer = TestUtils.createTransactionalProducer("test-txn", servers) + val producer = TestUtils.createTransactionalProducer("test-txn", servers, 512) producer.initTransactions() @@ -125,9 +128,20 @@ class TransactionsBounceTest extends KafkaServerTestHarness { scheduler.shutdown() val verifyingConsumer = createConsumerAndSubscribeToTopics("randomGroup", List(outputTopic), readCommitted = true) - val outputRecords = TestUtils.pollUntilAtLeastNumRecords(verifyingConsumer, numInputRecords).map { record => - TestUtils.assertCommittedAndGetValue(record).toInt + val recordsByPartition = new mutable.HashMap[TopicPartition, mutable.ListBuffer[Int]]() + TestUtils.pollUntilAtLeastNumRecords(verifyingConsumer, numInputRecords).foreach { record => + val value = TestUtils.assertCommittedAndGetValue(record).toInt + val topicPartition = new TopicPartition(record.topic(), record.partition()) + recordsByPartition.getOrElseUpdate(topicPartition, new mutable.ListBuffer[Int]) + .append(value) + } + + val outputRecords = new mutable.ListBuffer[Int]() + recordsByPartition.values.foreach { case (partitionValues) => + assertEquals("Out of order messages detected", partitionValues, partitionValues.sorted) + outputRecords.appendAll(partitionValues) } + val recordSet = outputRecords.toSet assertEquals(numInputRecords, recordSet.size) diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala index 517e8760a870f..c4d1c0d220a38 100755 --- a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala @@ -281,23 +281,26 @@ class LogCleanerTest extends JUnitSuite { assertEquals(List(0, 2, 3, 4, 5), offsetsInLog(log)) appendProducer(Seq(1, 3)) + appendProducer(Seq(4)) + appendProducer(Seq(5)) + appendProducer(Seq(6)) log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false) log.roll() // the first cleaning preserves the commit marker (at offset 3) since there were still records for the transaction dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1 - assertEquals(List(2, 1, 3), keysInLog(log)) - assertEquals(List(3, 4, 5, 6, 7, 8), offsetsInLog(log)) + assertEquals(List(2, 1, 3, 4, 5, 6), keysInLog(log)) + assertEquals(List(3, 4, 5, 6, 7, 8, 9, 10, 11), offsetsInLog(log)) // delete horizon forced to 0 to verify marker is not removed early dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = 0L)._1 - assertEquals(List(2, 1, 3), keysInLog(log)) - assertEquals(List(3, 4, 5, 6, 7, 8), offsetsInLog(log)) + assertEquals(List(2, 1, 3, 4, 5, 6), keysInLog(log)) + assertEquals(List(3, 4, 5, 6, 7, 8, 9, 10, 11), offsetsInLog(log)) // clean again with large delete horizon and verify the marker is removed dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1 - assertEquals(List(2, 1, 3), keysInLog(log)) - assertEquals(List(4, 5, 6, 7, 8), offsetsInLog(log)) + assertEquals(List(2, 1, 3, 4, 5, 6), keysInLog(log)) + assertEquals(List(4, 5, 6, 7, 8, 9, 10, 11), offsetsInLog(log)) } @Test @@ -332,19 +335,22 @@ class LogCleanerTest extends JUnitSuite { assertEquals(List(2, 3, 4), offsetsInLog(log)) // commit marker is still retained assertEquals(List(1, 2, 3, 4), lastOffsetsPerBatchInLog(log)) // empty batch is retained - // append a new record from the producer to allow cleaning of the empty batch - appendProducer(Seq(1)) + // append a new records from the producer to allow cleaning of the empty batch + for (i <- Range(1, ProducerIdEntry.NumBatchesToRetain + 1)) { + appendProducer(Seq(i)) + } + log.roll() dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1 - assertEquals(List(2, 3, 1), keysInLog(log)) - assertEquals(List(2, 3, 4, 5), offsetsInLog(log)) // commit marker is still retained - assertEquals(List(2, 3, 4, 5), lastOffsetsPerBatchInLog(log)) // empty batch should be gone + assertEquals(List(1, 2, 3, 4, 5), keysInLog(log)) + assertEquals(List(2, 5, 6, 7, 8, 9), offsetsInLog(log)) // commit marker is still retained + assertEquals(List(2, 5, 6, 7, 8, 9), lastOffsetsPerBatchInLog(log)) // empty batch should be gone dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1 - assertEquals(List(2, 3, 1), keysInLog(log)) - assertEquals(List(3, 4, 5), offsetsInLog(log)) // commit marker is gone - assertEquals(List(3, 4, 5), lastOffsetsPerBatchInLog(log)) // empty batch is gone + assertEquals(List(1, 2, 3, 4, 5), keysInLog(log)) + assertEquals(List(5, 6, 7, 8, 9), offsetsInLog(log)) // commit marker is gone + assertEquals(List(5, 6, 7, 8, 9), lastOffsetsPerBatchInLog(log)) // empty batch is gone } @Test @@ -362,19 +368,21 @@ class LogCleanerTest extends JUnitSuite { appendProducer(Seq(1)) appendProducer(Seq(2, 3)) log.appendAsLeader(abortMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false) - appendProducer(Seq(3)) + for (i <- Range(3, ProducerIdEntry.NumBatchesToRetain + 3)) { + appendProducer(Seq(i)) + } log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false) log.roll() // delete horizon set to 0 to verify marker is not removed early val dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = 0L)._1 - assertEquals(List(3), keysInLog(log)) - assertEquals(List(3, 4, 5), offsetsInLog(log)) + assertEquals(List(3, 4, 5, 6, 7), keysInLog(log)) + assertEquals(List(3, 4, 5, 6, 7, 8, 9), offsetsInLog(log)) - // clean again with large delete horizon and verify the marker is removed + // clean again with large delete horizon and verify the abort marker is removed cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue) - assertEquals(List(3), keysInLog(log)) - assertEquals(List(4, 5), offsetsInLog(log)) + assertEquals(List(3, 4, 5, 6, 7), keysInLog(log)) + assertEquals(List(4, 5, 6, 7, 8, 9), offsetsInLog(log)) } @Test @@ -417,20 +425,22 @@ class LogCleanerTest extends JUnitSuite { assertEquals(List(2), offsetsInLog(log)) // abort marker is still retained assertEquals(List(1, 2), lastOffsetsPerBatchInLog(log)) // empty batch is retained - // now update the last sequence so that the empty batch can be removed - appendProducer(Seq(1)) + // now create enough batches so that the empty batch can be removed + for (i <- Range(0, ProducerIdEntry.NumBatchesToRetain)) { + appendProducer(Seq(1)) + } log.roll() dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1 assertAbortedTransactionIndexed() assertEquals(List(1), keysInLog(log)) - assertEquals(List(2, 3), offsetsInLog(log)) // abort marker is not yet gone because we read the empty batch - assertEquals(List(2, 3), lastOffsetsPerBatchInLog(log)) // but we do not preserve the empty batch + assertEquals(List(2, 7), offsetsInLog(log)) // abort marker is not yet gone because we read the empty batch + assertEquals(List(2, 3, 4, 5, 6, 7), lastOffsetsPerBatchInLog(log)) // but we do not preserve the original empty batch dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1 assertEquals(List(1), keysInLog(log)) - assertEquals(List(3), offsetsInLog(log)) // abort marker is gone - assertEquals(List(3), lastOffsetsPerBatchInLog(log)) + assertEquals(List(7), offsetsInLog(log)) // abort marker is gone + assertEquals(List(3, 4, 5, 6, 7), lastOffsetsPerBatchInLog(log)) // we do not bother retaining the aborted transaction in the index assertEquals(0, log.collectAbortedTransactions(0L, 100L).size) @@ -539,7 +549,7 @@ class LogCleanerTest extends JUnitSuite { log.roll() cleaner.clean(LogToClean(new TopicPartition("test", 0), log, 0L, log.activeSegment.baseOffset)) - assertEquals(List(1, 3, 4), lastOffsetsPerBatchInLog(log)) + assertEquals(List(1, 2, 3, 4), lastOffsetsPerBatchInLog(log)) assertEquals(Map(1L -> 0, 2L -> 1, 3L -> 0), lastSequencesInLog(log)) assertEquals(List(0, 1), keysInLog(log)) assertEquals(List(3, 4), offsetsInLog(log)) @@ -562,20 +572,24 @@ class LogCleanerTest extends JUnitSuite { log.roll() cleaner.clean(LogToClean(new TopicPartition("test", 0), log, 0L, log.activeSegment.baseOffset)) - assertEquals(List(2, 3), lastOffsetsPerBatchInLog(log)) + assertEquals(List(0, 2, 3), lastOffsetsPerBatchInLog(log)) assertEquals(Map(producerId -> 2), lastSequencesInLog(log)) assertEquals(List(), keysInLog(log)) assertEquals(List(3), offsetsInLog(log)) - // Append a new entry from the producer and verify that the empty batch is cleaned up - appendProducer(Seq(1, 5)) + // Append new entries from the producer and verify that the empty batches are cleaned up + for (i <- Range(0, ProducerIdEntry.NumBatchesToRetain)) { + appendProducer(Seq(1, 5)) + } + log.roll() cleaner.clean(LogToClean(new TopicPartition("test", 0), log, 0L, log.activeSegment.baseOffset)) - assertEquals(List(3, 5), lastOffsetsPerBatchInLog(log)) - assertEquals(Map(producerId -> 4), lastSequencesInLog(log)) + // We now retain the last 5 batches. + assertEquals(List(3, 5, 7, 9, 11, 13), lastOffsetsPerBatchInLog(log)) + assertEquals(Map(producerId -> 12), lastSequencesInLog(log)) assertEquals(List(1, 5), keysInLog(log)) - assertEquals(List(3, 4, 5), offsetsInLog(log)) + assertEquals(List(3, 12, 13), offsetsInLog(log)) } @Test diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index a52c83c11fed8..902d1c3baccdd 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -1379,11 +1379,12 @@ object TestUtils extends Logging { records } - def createTransactionalProducer(transactionalId: String, servers: Seq[KafkaServer]) = { + def createTransactionalProducer(transactionalId: String, servers: Seq[KafkaServer], batchSize: Int = 16384) = { val props = new Properties() props.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, transactionalId) - props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1") + props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "5") props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true") + props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize.toString) TestUtils.createNewProducer(TestUtils.getBrokerListStrFromServers(servers), retries = Integer.MAX_VALUE, acks = -1, props = Some(props)) } diff --git a/tools/src/main/java/org/apache/kafka/tools/TransactionalMessageCopier.java b/tools/src/main/java/org/apache/kafka/tools/TransactionalMessageCopier.java index 3903a3a8fae00..0d74645379ebe 100644 --- a/tools/src/main/java/org/apache/kafka/tools/TransactionalMessageCopier.java +++ b/tools/src/main/java/org/apache/kafka/tools/TransactionalMessageCopier.java @@ -149,6 +149,11 @@ private static KafkaProducer createProducer(Namespace parsedArgs "org.apache.kafka.common.serialization.StringSerializer"); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + // We set a small batch size to ensure that we have multiple inflight requests per transaction. + // If it is left at the default, each transaction will have only one batch per partition, hence not testing + // the case with multiple inflights. + props.put(ProducerConfig.BATCH_SIZE_CONFIG, "512"); + props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "5"); return new KafkaProducer<>(props); } @@ -252,7 +257,6 @@ public static void main(String[] args) throws IOException { maxMessages = Math.min(messagesRemaining(consumer, inputPartition), maxMessages); final boolean enableRandomAborts = parsedArgs.getBoolean("enableRandomAborts"); - producer.initTransactions(); final AtomicBoolean isShuttingDown = new AtomicBoolean(false); From 7ae93661792b322642a2b9f933d3324b5997cadc Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Fri, 25 Aug 2017 12:27:03 -0700 Subject: [PATCH 06/40] Fix merge issues aftre rebasing onto trunk --- core/src/main/scala/kafka/log/ProducerStateManager.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index 78a12130f2547..978932bb32728 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -17,7 +17,7 @@ package kafka.log import java.io._ -import java.nio.ByteBuffer +import java.nio.{BufferUnderflowException, ByteBuffer} import java.nio.file.Files import kafka.common.KafkaException @@ -378,6 +378,8 @@ object ProducerStateManager { catch { case e: SchemaException => throw new CorruptSnapshotException(s"Snapshot failed schema validation: ${e.getMessage}") + case e: BufferUnderflowException => + throw new CorruptSnapshotException(s"Could not detect the version of the snapshot schema: ${e.getMessage}") } } From f4ecc43b368335cedb5308a981a5f14f21ae6de5 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Fri, 25 Aug 2017 13:33:34 -0700 Subject: [PATCH 07/40] Remove some unneeded variables and function parameters --- .../kafka/clients/producer/internals/ProducerBatch.java | 2 -- .../kafka/clients/producer/internals/RecordAccumulator.java | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java index e00a253462f47..722252385806c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java @@ -75,7 +75,6 @@ private enum FinalState { ABORTED, FAILED, SUCCEEDED } private long drainedMs; private String expiryErrorMessage; private boolean retry; - private boolean countedTowardSequence; public ProducerBatch(TopicPartition tp, MemoryRecordsBuilder recordsBuilder, long now) { this(tp, recordsBuilder, now, false); @@ -90,7 +89,6 @@ public ProducerBatch(TopicPartition tp, MemoryRecordsBuilder recordsBuilder, lon this.produceFuture = new ProduceRequestResult(topicPartition); this.retry = false; this.isSplitBatch = isSplitBatch; - this.countedTowardSequence = false; float compressionRatioEstimation = CompressionRatioEstimator.estimation(topicPartition.topic(), recordsBuilder.compressionType()); recordsBuilder.setEstimatedCompressionRatio(compressionRatioEstimation); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index fc8638239dffa..7fdf310e291ed 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -195,7 +195,7 @@ public RecordAppendResult append(TopicPartition tp, synchronized (dq) { if (closed) throw new IllegalStateException("Cannot send after the producer is closed."); - RecordAppendResult appendResult = tryAppend(tp, timestamp, key, value, headers, callback, dq); + RecordAppendResult appendResult = tryAppend(timestamp, key, value, headers, callback, dq); if (appendResult != null) return appendResult; } @@ -210,7 +210,7 @@ public RecordAppendResult append(TopicPartition tp, if (closed) throw new IllegalStateException("Cannot send after the producer is closed."); - RecordAppendResult appendResult = tryAppend(tp, timestamp, key, value, headers, callback, dq); + RecordAppendResult appendResult = tryAppend(timestamp, key, value, headers, callback, dq); if (appendResult != null) { // Somebody else found us a batch, return the one we waited for! Hopefully this doesn't happen often... return appendResult; @@ -251,7 +251,7 @@ private MemoryRecordsBuilder recordsBuilder(TopicPartition topicPartition, ByteB * and memory records built) in one of the following cases (whichever comes first): right before send, * if it is expired, or when the producer is closed. */ - private RecordAppendResult tryAppend(TopicPartition tp, long timestamp, byte[] key, byte[] value, Header[] headers, + private RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Header[] headers, Callback callback, Deque deque) { ProducerBatch last = deque.peekLast(); if (last != null) { From 427cd71a127304f52a2e47dc4e3ac2f325bf999d Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Fri, 25 Aug 2017 13:39:28 -0700 Subject: [PATCH 08/40] More minor cleanups --- .../kafka/clients/producer/internals/RecordAccumulator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index 7fdf310e291ed..d0018d78ef9b8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -216,7 +216,7 @@ public RecordAppendResult append(TopicPartition tp, return appendResult; } - MemoryRecordsBuilder recordsBuilder = recordsBuilder(tp, buffer, maxUsableMagic); + MemoryRecordsBuilder recordsBuilder = recordsBuilder(buffer, maxUsableMagic); ProducerBatch batch = new ProducerBatch(tp, recordsBuilder, time.milliseconds()); FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, headers, callback, time.milliseconds())); @@ -235,7 +235,7 @@ public RecordAppendResult append(TopicPartition tp, } } - private MemoryRecordsBuilder recordsBuilder(TopicPartition topicPartition, ByteBuffer buffer, byte maxUsableMagic) { + private MemoryRecordsBuilder recordsBuilder(ByteBuffer buffer, byte maxUsableMagic) { if (transactionManager != null && maxUsableMagic < RecordBatch.MAGIC_VALUE_V2) { throw new UnsupportedVersionException("Attempting to use idempotence with a broker which does not " + "support the required message format (v2). The broker must be version 0.11 or later."); From 833b39cf0e3e92e9ddbc8ab2d5946a8e293a6cb8 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 31 Aug 2017 14:54:09 -0700 Subject: [PATCH 09/40] WIP --- .../org/apache/kafka/clients/producer/internals/Sender.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 656e6587c374a..10449ad4e6fd2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -273,11 +273,13 @@ private long sendProducerData(long now) { for (ProducerBatch expiredBatch : expiredBatches) { failBatch(expiredBatch, -1, NO_TIMESTAMP, expiredBatch.timeoutException()); if (transactionManager != null && expiredBatch.inRetry()) { + // transition to unknown state. mute partition. needsTransactionStateReset = true; } this.sensors.recordErrors(expiredBatch.topicPartition.topic(), expiredBatch.recordCount); } + metadata.fetch().leaderFor(new TopicPartition("fo", 1)).idString(); if (needsTransactionStateReset) { transactionManager.resetProducerId(); return 0; From 3bd9dec440e6e891689d81cbb0a089a99200a488 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 6 Sep 2017 14:25:50 -0700 Subject: [PATCH 10/40] Handled remaining corner cases: 1. Batch expiry on the producer. In this case, we need mute the partition and make sure all the inflight requests are fully resolved. If they are successful, then all is well. If they are also expired or fail due to OutOfOrderSequence, we need to reset the producer id. 2. We now deal correctly with out of order responses to in flight requests which may happen due to leadership changes. We keep track of all inflight batches, ordered by sequence. Whenever there is an error, we reduce to one inflight request by ensuring only the next batch by sequence can be drained. When we requeue a batch, we check to make sure that the queue remains in sequence order. When a batch completes, it is removed from the in flight list, and the next batch by sequence changes. Some other changes: 1. We never change sequence numbers unless we are certain that a batch failed fatally on the broker (got a fatal error code), _and_ future batches retured OutOfOrderSequence. Before the latest commit, we would always reset sequence numbers on retry, and have other mechanisms to ensure that a given batch got the same sequence as before on the next drain. This is now much tighter, and needs to be in order to correctly handle the out of order response problem. Todo: Add unit tests for out of order responses. Add unit tests new cases around batch expiry (all batches expired, all batches succeeded, future batches failed with OutOfSequence). --- .../producer/internals/ProducerBatch.java | 17 ++++ .../producer/internals/RecordAccumulator.java | 68 ++++++++++++- .../clients/producer/internals/Sender.java | 65 +++++++------ .../internals/TransactionManager.java | 97 ++++++++++++++++++- .../common/record/MemoryRecordsBuilder.java | 4 - .../producer/internals/SenderTest.java | 15 +-- .../internals/TransactionManagerTest.java | 36 +++---- 7 files changed, 233 insertions(+), 69 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java index 722252385806c..a3e120a996b6d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java @@ -249,6 +249,16 @@ public Deque split(int splitBatchSize) { produceFuture.set(ProduceResponse.INVALID_OFFSET, NO_TIMESTAMP, new RecordBatchTooLargeException()); produceFuture.done(); + + if (hasSequence()) { + int sequence = baseSequence(); + ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId(), producerEpoch()); + for (ProducerBatch newBatch : batches) { + newBatch.setProducerState(producerIdAndEpoch, sequence, isTransactional()); + newBatch.retry = true; + sequence += newBatch.recordCount; + } + } return batches; } @@ -442,4 +452,11 @@ public int baseSequence() { return recordsBuilder.baseSequence(); } + public boolean hasSequence() { + return baseSequence() != RecordBatch.NO_SEQUENCE; + } + + public boolean isTransactional() { + return recordsBuilder.isTransactional(); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index d0018d78ef9b8..3d0a4fa7a78e9 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -256,11 +256,10 @@ private RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, H ProducerBatch last = deque.peekLast(); if (last != null) { FutureRecordMetadata future = last.tryAppend(timestamp, key, value, headers, callback, time.milliseconds()); - if (future == null) { + if (future == null) last.closeForRecordAppends(); - } else { + else return new RecordAppendResult(future, deque.size() > 1 || last.isFull(), false); - } } return null; } @@ -311,6 +310,7 @@ public void reenqueue(ProducerBatch batch, long now) { Deque deque = getOrCreateDeque(batch.topicPartition); synchronized (deque) { deque.addFirst(batch); + maybeEnsureQueueIsOrdered(deque, batch); } } @@ -333,11 +333,53 @@ public int splitAndReenqueue(ProducerBatch bigBatch) { // We treat the newly split batches as if they are not even tried. synchronized (partitionDequeue) { partitionDequeue.addFirst(batch); + if (transactionManager != null) { + // We should track the newly created batches since they already have assigned sequences. + transactionManager.startTrackingBatch(batch); + maybeEnsureQueueIsOrdered(partitionDequeue, batch); + } } } return numSplitBatches; } + private void maybeEnsureQueueIsOrdered(Deque deque, ProducerBatch batch) { + if (transactionManager != null) { + // When we are requeing and have enabled idempotence, the reenqueued batch must always have a sequence. + if (batch.baseSequence() == RecordBatch.NO_SEQUENCE) + throw new IllegalStateException("Trying to reenqueue a batch which doesn't have a sequence even " + + "though idempotence is enabled."); + + if (batch.baseSequence() != transactionManager.nextBatchBySequence(batch.topicPartition).baseSequence()) { + // The head of the queues have diverged. This means that the incoming batch should be placed somewhere further back. + // We need to find the right place for the incoming batch and insert it there. + // We will only enter this branch if we have multiple inflights sent to different brokers, perhaps + // because a leadership change occurred in between the drains. In this scenario, responses can come + // back out of order, requiring us to re order the batches ourselves rather than relying on the + // implicit ordering guarantees of the network client which are only on a per connection basis. + + ProducerBatch incomingBatch = deque.pollFirst(); + List orderedBatches = new ArrayList<>(); + while (deque.peekFirst() != null && deque.peekFirst().hasSequence() && deque.peekFirst().baseSequence() < incomingBatch.baseSequence()) + orderedBatches.add(deque.pollFirst()); + + log.debug("Reordered incoming batch with sequence {} for partition {}. It was placed in the queue at " + + "position {}" + incomingBatch.baseSequence(), incomingBatch.topicPartition, orderedBatches.size()); + // Either we have reached a point where there are batches without a sequence (ie. never been drained + // and are hence in order by default), or the batch at the front of the queue has a sequence greater + // than the incoming batch. This is the right place to add the incoming batch. + deque.addFirst(incomingBatch); + + // Now we have to re insert the previously queued batches in the right order. + for (int i = orderedBatches.size() - 1; i >= 0; --i) { + deque.addFirst(orderedBatches.get(i)); + } + + // At this point, the incoming batch has been queued in the correct place according to its sequence. + } + } + } + /** * Get a list of nodes whose partitions are ready to be sent, and the earliest time at which any non-sendable * partition will be ready; Also return the flag for whether there are any unknown leaders for the accumulated @@ -470,10 +512,24 @@ public Map> drain(Cluster cluster, break; isTransactional = transactionManager.isTransactional(); + + if (!first.hasSequence() && transactionManager.partitionHasUnresolvedSequence(first.topicPartition)) + // Don't drain any new batches while the state of previous sequence numbers + // is unknown. The previous batches would be unknown if they were aborted + // on the client after being sent to the broker at least once. + break; + + if (first.hasSequence() && !first.equals(transactionManager.nextBatchBySequence(first.topicPartition))) + // If the queued batch already has an assigned sequence, then it is being + // retried. In this case, we wait until the next immediate batch is ready + // and drain that. We only move on when the next in line batch is complete (either successfully + // or due to a fatal broker error). This effectively reduces our + // in flight request count to 1. + break; } ProducerBatch batch = deque.pollFirst(); - if (producerIdAndEpoch != null) { + if (producerIdAndEpoch != null && !batch.hasSequence()) { // If the batch is in retry, then we should not change the producer id and // sequence number, since this may introduce duplicates. In particular, // the previous attempt may actually have been accepted, and if we change @@ -492,6 +548,8 @@ public Map> drain(Cluster cluster, log.debug("Assigned producerId {} and producerEpoch {} to batch with sequence " + "{} being sent to partition {}", producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, batch.baseSequence(), tp); + + transactionManager.startTrackingBatch(batch); } batch.close(); size += batch.records().sizeInBytes(); @@ -643,7 +701,7 @@ void abortUndrainedBatches(RuntimeException reason) { Deque dq = getDeque(batch.topicPartition); boolean aborted = false; synchronized (dq) { - if (!batch.isClosed()) { + if ((transactionManager != null && !batch.hasSequence()) || (transactionManager == null && !batch.isClosed())) { aborted = true; batch.abortRecordAppends(); dq.remove(batch); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 10449ad4e6fd2..1270b9c8e3df5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -228,6 +228,12 @@ void run(long now) { private long sendProducerData(long now) { Cluster cluster = metadata.fetch(); + + + if (transactionManager != null && transactionManager.maybeResolveSequences()) { + transactionManager.resetProducerId(); + return 0; + } // get the list of partitions with data ready to send RecordAccumulator.ReadyCheckResult result = this.accumulator.ready(cluster, now); @@ -264,27 +270,20 @@ private long sendProducerData(long now) { } List expiredBatches = this.accumulator.expiredBatches(this.requestTimeout, now); - boolean needsTransactionStateReset = false; // Reset the producer id if an expired batch has previously been sent to the broker. Also update the metrics // for expired batches. see the documentation of @TransactionState.resetProducerId to understand why // we need to reset the producer id here. if (!expiredBatches.isEmpty()) log.trace("Expired {} batches in accumulator", expiredBatches.size()); for (ProducerBatch expiredBatch : expiredBatches) { - failBatch(expiredBatch, -1, NO_TIMESTAMP, expiredBatch.timeoutException()); + failBatch(expiredBatch, -1, NO_TIMESTAMP, expiredBatch.timeoutException(), false); if (transactionManager != null && expiredBatch.inRetry()) { - // transition to unknown state. mute partition. - needsTransactionStateReset = true; + // This ensures that no new batches are drained until the current in flight batches are fully resolved. + transactionManager.markSequenceUnresolved(expiredBatch.topicPartition); } this.sensors.recordErrors(expiredBatch.topicPartition.topic(), expiredBatch.recordCount); } - metadata.fetch().leaderFor(new TopicPartition("fo", 1)).idString(); - if (needsTransactionStateReset) { - transactionManager.resetProducerId(); - return 0; - } - sensors.updateProduceRequestMetrics(batches); // If we have any nodes that are ready to send + have sendable data, poll with 0 timeout so this can immediately @@ -495,8 +494,6 @@ private void handleProduceResponse(ClientResponse response, Map 1 && (batch.magic() >= RecordBatch.MAGIC_VALUE_V2 || batch.isCompressed())) { @@ -507,6 +504,8 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons batch.topicPartition, this.retries - batch.attempts(), error); + if (transactionManager != null) + transactionManager.stopTrackingBatch(batch); this.accumulator.splitAndReenqueue(batch); this.accumulator.deallocate(batch); this.sensors.recordBatchSplit(); @@ -528,7 +527,7 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons } else { failBatch(batch, response, new OutOfOrderSequenceException("Attempted to retry sending a " + "batch but the producer id changed from " + batch.producerId() + " to " + - transactionManager.producerIdAndEpoch().producerId + " in the mean time. This batch will be dropped.")); + transactionManager.producerIdAndEpoch().producerId + " in the mean time. This batch will be dropped."), false); this.sensors.recordErrors(batch.topicPartition.topic(), batch.recordCount); } } else { @@ -540,7 +539,7 @@ else if (error == Errors.CLUSTER_AUTHORIZATION_FAILED) else exception = error.exception(); // tell the user the result of their request - failBatch(batch, response, exception); + failBatch(batch, response, exception, true); this.sensors.recordErrors(batch.topicPartition.topic(), batch.recordCount); } if (error.exception() instanceof InvalidMetadataException) { @@ -559,15 +558,6 @@ else if (error == Errors.CLUSTER_AUTHORIZATION_FAILED) this.accumulator.unmutePartition(batch.topicPartition); } - private void maybeResetNextSequenceForPartition(TopicPartition topicPartition, int nextSequence) { - if (transactionManager != null) - if (transactionManager.isNextSequence(topicPartition, nextSequence)) - // If we are retrying the first in flight request, then reset the next sequence number to be the - // the last ack'd sequence + 1. Subsequent batches will get the succeeding sequence numbers. - transactionManager.setNextSequence(topicPartition, nextSequence); - - } - private void reenqueueBatch(ProducerBatch batch, long currentTimeMs) { if (transactionManager != null) // Reset the sequence number for the retried batch. It will be set again on the next drain. @@ -577,21 +567,25 @@ private void reenqueueBatch(ProducerBatch batch, long currentTimeMs) { } private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response) { - if (transactionManager != null && transactionManager.hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch())) { - transactionManager.setLastAckedSequence(batch.topicPartition, batch.baseSequence() + batch.recordCount - 1); - log.debug("ProducerId: {}; Set last ack'd sequence number for topic-partition {} to {}", batch.producerId(), batch.topicPartition, - transactionManager.lastAckedSequence(batch.topicPartition)); + if (transactionManager != null) { + if (transactionManager.hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch())) { + transactionManager.setLastAckedSequence(batch.topicPartition, batch.baseSequence() + batch.recordCount - 1); + log.debug("ProducerId: {}; Set last ack'd sequence number for topic-partition {} to {}", batch.producerId(), batch.topicPartition, + transactionManager.lastAckedSequence(batch.topicPartition)); + } + transactionManager.stopTrackingBatch(batch); } batch.done(response.baseOffset, response.logAppendTime, null); + this.accumulator.deallocate(batch); } - private void failBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response, RuntimeException exception) { - failBatch(batch, response.baseOffset, response.logAppendTime, exception); + private void failBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response, RuntimeException exception, boolean adjustSequenceNumbers) { + failBatch(batch, response.baseOffset, response.logAppendTime, exception, adjustSequenceNumbers); } - private void failBatch(ProducerBatch batch, long baseOffset, long logAppendTime, RuntimeException exception) { + private void failBatch(ProducerBatch batch, long baseOffset, long logAppendTime, RuntimeException exception, boolean adjustSequenceNumbers) { if (transactionManager != null) { if (exception instanceof OutOfOrderSequenceException && !transactionManager.isTransactional() @@ -612,6 +606,9 @@ private void failBatch(ProducerBatch batch, long baseOffset, long logAppendTime, } else if (transactionManager.isTransactional()) { transactionManager.transitionToAbortableError(exception); } + transactionManager.stopTrackingBatch(batch); + if (adjustSequenceNumbers) + transactionManager.adjustSequencesDueToFailedBatch(batch); } batch.done(baseOffset, logAppendTime, exception); @@ -624,8 +621,12 @@ private void failBatch(ProducerBatch batch, long baseOffset, long logAppendTime, * batches are certain to fail with an OutOfOrderSequence exception. */ private boolean canRetry(ProducerBatch batch, Errors error) { - return (batch.attempts() < this.retries && error.exception() instanceof RetriableException) || - (error.exception() instanceof OutOfOrderSequenceException && !transactionManager.isNextSequence(batch.topicPartition, batch.baseSequence())); + return batch.attempts() < this.retries && + ((error.exception() instanceof RetriableException) || + (error.exception() instanceof OutOfOrderSequenceException && + transactionManager.hasProducerId(batch.producerId()) && + !transactionManager.partitionHasUnresolvedSequence(batch.topicPartition) && + !transactionManager.isNextSequence(batch.topicPartition, batch.baseSequence()))); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index d77576a695a44..bce8d703e171d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -67,6 +67,8 @@ public class TransactionManager { private final Map sequenceNumbers; private final Map lastAckedSequence; + private final Set partitionsWithUnresolvedSequenceNumbers; + private final Map> inflightBatchesBySequence; private final PriorityQueue pendingRequests; private final Set newPartitionsInTransaction; private final Set pendingPartitionsInTransaction; @@ -161,6 +163,9 @@ public int compare(TxnRequestHandler o1, TxnRequestHandler o2) { } }); + this.partitionsWithUnresolvedSequenceNumbers = new HashSet<>(); + this.inflightBatchesBySequence = new HashMap<>(); + this.retryBackoffMs = retryBackoffMs; } @@ -366,6 +371,8 @@ synchronized void resetProducerId() { setProducerIdAndEpoch(ProducerIdAndEpoch.NONE); this.sequenceNumbers.clear(); this.lastAckedSequence.clear(); + this.inflightBatchesBySequence.clear(); + this.partitionsWithUnresolvedSequenceNumbers.clear(); } /** @@ -389,8 +396,33 @@ synchronized void incrementSequenceNumber(TopicPartition topicPartition, int inc sequenceNumbers.put(topicPartition, currentSequenceNumber); } + synchronized void startTrackingBatch(ProducerBatch batch) { + if (!batch.hasSequence()) + throw new IllegalStateException("Can't track batch for partition " + batch.topicPartition + " when sequence is not set."); + if (!inflightBatchesBySequence.containsKey(batch.topicPartition)) { + inflightBatchesBySequence.put(batch.topicPartition, new PriorityQueue<>(5, new Comparator() { + @Override + public int compare(ProducerBatch o1, ProducerBatch o2) { + return o1.baseSequence() - o2.baseSequence(); + } + })); + } + inflightBatchesBySequence.get(batch.topicPartition).offer(batch); + } + + + synchronized ProducerBatch nextBatchBySequence(TopicPartition topicPartition) { + return inflightBatchesBySequence.get(topicPartition).peek(); + } + + synchronized void stopTrackingBatch(ProducerBatch batch) { + if (inflightBatchesBySequence.containsKey(batch.topicPartition)) + inflightBatchesBySequence.get(batch.topicPartition).remove(batch); + } + synchronized void setLastAckedSequence(TopicPartition topicPartition, int sequence) { - lastAckedSequence.put(topicPartition, sequence); + if (sequence > lastAckedSequence(topicPartition)) + lastAckedSequence.put(topicPartition, sequence); } synchronized int lastAckedSequence(TopicPartition topicPartition) { @@ -400,6 +432,69 @@ synchronized int lastAckedSequence(TopicPartition topicPartition) { return lastAckedSequence.get(topicPartition); } + // If a batch is failed fatally, the sequence numbers for future batches bound for the partition must be adjusted + // so that they don't fail with the OutOfOrderSequenceException. + // + // This method must only be called when we know that the batch is question has been unequivocally failed by the broker, + // ie. it has received a confirmed fatal status code like 'Message Too Large' or something similar. + synchronized void adjustSequencesDueToFailedBatch(ProducerBatch batch) { + if (!this.sequenceNumbers.containsKey(batch.topicPartition)) + // Sequence numbers are not being tracked for this partition. This could happen if the producer id was just + // reset due to a previous OutOfOrderSequenceException. + return; + int currentSequence = sequenceNumber(batch.topicPartition); + currentSequence -= batch.recordCount; + if (currentSequence < 0) + throw new IllegalStateException("Sequence number for partition " + batch.topicPartition + " is going to become negative : " + currentSequence); + + setNextSequence(batch.topicPartition, currentSequence); + + for (ProducerBatch inFlightBatch : inflightBatchesBySequence.get(batch.topicPartition)) { + int newSequence = inFlightBatch.baseSequence() - batch.recordCount; + if (newSequence < 0) + throw new IllegalStateException("Sequence number for batch with sequence " + inFlightBatch.baseSequence() + + " for partition " + batch.topicPartition + " is going to become negative :" + newSequence); + + log.debug("Setting sequence number of batch with current sequence {} for partition {} to {}", batch.baseSequence(), batch.topicPartition, newSequence); + inFlightBatch.setProducerState(new ProducerIdAndEpoch(inFlightBatch.producerId(), inFlightBatch.producerEpoch()), newSequence, inFlightBatch.isTransactional()); + } + } + + synchronized boolean hasInflightBatches(TopicPartition topicPartition) { + return inflightBatchesBySequence.containsKey(topicPartition) && !inflightBatchesBySequence.get(topicPartition).isEmpty(); + } + + synchronized boolean partitionHasUnresolvedSequence(TopicPartition topicPartition) { + return partitionsWithUnresolvedSequenceNumbers.contains(topicPartition); + } + + synchronized void markSequenceUnresolved(TopicPartition topicPartition) { + partitionsWithUnresolvedSequenceNumbers.add(topicPartition); + } + + // Checks if there are any partitions with unresolved partitions which may now be resolved. Returns true if + // the producer id needs a reset, false otherwise. + + synchronized boolean maybeResolveSequences() { + for (TopicPartition topicPartition : partitionsWithUnresolvedSequenceNumbers) { + if (!hasInflightBatches(topicPartition)) { + // The partition has been fully drained. At this point, the last ack'd sequence should be once less than + // next sequence destined for the partition. If so, the partition is fully resolved. If not, we should + // reset the sequence number if necessary. + if (lastAckedSequence(topicPartition) == sequenceNumber(topicPartition) - 1) { + // This would happen when a batch was expired, but subsequent batches succeeded. + partitionsWithUnresolvedSequenceNumbers.remove(topicPartition); + } else { + // We would enter this branch if all in flight batches were ultimately expired in the producer. + log.info("No inflight batches remaining for {}, last ack'd sequence for partition is {}, next sequence is {}. " + + "Going to reset producer state.", topicPartition, lastAckedSequence(topicPartition), sequenceNumber(topicPartition)); + return true; + } + } + } + return false; + } + synchronized boolean isNextSequence(TopicPartition topicPartition, int sequence) { return (!lastAckedSequence.containsKey(topicPartition) && sequence == 0) || (lastAckedSequence.containsKey(topicPartition) && (sequence - lastAckedSequence.get(topicPartition) == 1)); diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java index 9247440c82562..72d3e1ac1caca 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java @@ -280,10 +280,6 @@ public void abort() { } public void unsetProducerState() { - this.producerId = RecordBatch.NO_PRODUCER_ID; - this.producerEpoch = RecordBatch.NO_PRODUCER_EPOCH; - this.baseSequence = RecordBatch.NO_SEQUENCE; - this.isTransactional = false; builtRecords = null; } diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 9474606f25cbb..07fd6f8a27489 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -555,15 +555,18 @@ public void testIdempotenceWithMultipleInflightsFirstFails() throws Exception { assertEquals(-1, transactionManager.lastAckedSequence(tp0)); assertEquals(1, client.inFlightRequestCount()); - sender.run(time.milliseconds()); // resend request 1 + sender.run(time.milliseconds()); // Do nothing, we are reduced to one in flight request during retries. - assertEquals(2, client.inFlightRequestCount()); + assertEquals(1, client.inFlightRequestCount()); assertEquals(-1, transactionManager.lastAckedSequence(tp0)); sendIdempotentProducerResponse(0, tp0, Errors.NONE, 0L); sender.run(time.milliseconds()); // receive response 0 - + assertEquals(0, transactionManager.lastAckedSequence(tp0)); + assertEquals(0, client.inFlightRequestCount()); + sender.run(time.milliseconds()); // send request 1 + assertEquals(1, client.inFlightRequestCount()); sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L); sender.run(time.milliseconds()); // receive response 1 @@ -906,12 +909,12 @@ private void testSplitBatchAndSend(TransactionManager txnManager, responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.MESSAGE_TOO_LARGE)); client.respond(new ProduceResponse(responseMap)); sender.run(time.milliseconds()); // split and reenqueue - assertEquals("The next sequence should have been reset to 0", 0, txnManager.sequenceNumber(tp).longValue()); + assertEquals("The next sequence should be 2", 2, txnManager.sequenceNumber(tp).longValue()); // The compression ratio should have been improved once. assertEquals(CompressionType.GZIP.rate - CompressionRatioEstimator.COMPRESSION_RATIO_IMPROVING_STEP, CompressionRatioEstimator.estimation(topic, CompressionType.GZIP), 0.01); sender.run(time.milliseconds()); // send the first produce request - assertEquals("The next sequence number should be 1", 1, txnManager.sequenceNumber(tp).longValue()); + assertEquals("The next sequence number should be 2", 2, txnManager.sequenceNumber(tp).longValue()); assertFalse("The future shouldn't have been done.", f1.isDone()); assertFalse("The future shouldn't have been done.", f2.isDone()); id = client.requests().peek().destination(); @@ -926,7 +929,7 @@ private void testSplitBatchAndSend(TransactionManager txnManager, sender.run(time.milliseconds()); // receive assertTrue("The future should have been done.", f1.isDone()); - assertEquals("The next sequence number should still be 1", 1, txnManager.sequenceNumber(tp).longValue()); + assertEquals("The next sequence number should still be 2", 2, txnManager.sequenceNumber(tp).longValue()); assertEquals("The last ack'd sequence number should be 0", 0, txnManager.lastAckedSequence(tp)); assertFalse("The future shouldn't have been done.", f2.isDone()); assertEquals("Offset of the first message should be 0", 0L, f1.get().offset()); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java index fb0c4047fd52e..d90a18611d8cc 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java @@ -67,7 +67,6 @@ import java.util.Arrays; import java.util.Collections; -import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -1167,11 +1166,12 @@ public void testRecoveryFromAbortableErrorProduceRequestInRetry() throws Excepti assertFalse(transactionManager.isPartitionAdded(unauthorizedPartition)); assertFalse(authorizedTopicProduceFuture.isDone()); - sender.run(time.milliseconds()); // will abort produce requests to both the authorized and unauthorized partitions since the transaction has been aborted. - assertTrue(authorizedTopicProduceFuture.isDone()); - assertTrue(unauthorizedTopicProduceFuture.isDone()); + prepareProduceResponse(Errors.NONE, pid, epoch); + sender.run(time.milliseconds()); assertFutureFailed(unauthorizedTopicProduceFuture); - assertFutureFailed(authorizedTopicProduceFuture); + assertTrue(authorizedTopicProduceFuture.isDone()); + assertNotNull(authorizedTopicProduceFuture.get()); + assertTrue(authorizedTopicProduceFuture.isDone()); prepareEndTxnResponse(Errors.NONE, TransactionResult.ABORT, pid, epoch); transactionManager.beginAbort(); @@ -1683,7 +1683,8 @@ public void testAbortResendsProduceRequestIfRetried() throws Exception { TransactionalRequestResult abortResult = transactionManager.beginAbort(); - // the produce response will be aborted once the transaction begins abort. + // we should resend the ProduceRequest before aborting + prepareProduceResponse(Errors.NONE, producerId, producerEpoch); prepareEndTxnResponse(Errors.NONE, TransactionResult.ABORT, producerId, producerEpoch); sender.run(time.milliseconds()); // Resend ProduceRequest @@ -1693,9 +1694,8 @@ public void testAbortResendsProduceRequestIfRetried() throws Exception { assertTrue(abortResult.isSuccessful()); assertTrue(transactionManager.isReady()); // make sure we are ready for a transaction now. - // ensure that the future failed - assertTrue(responseFuture.isDone()); - assertFutureFailed(responseFuture); + RecordMetadata recordMetadata = responseFuture.get(); + assertEquals(tp0.topic(), recordMetadata.topic()); } @Test @@ -1947,7 +1947,7 @@ public void testRaiseErrorWhenNoPartitionsPendingOnDrain() throws InterruptedExc assertTrue(drainedBatches.get(node1.id()).isEmpty()); } - @Test(expected = ExecutionException.class) + @Test public void resendFailedProduceRequestAfterAbortableError() throws Exception { final long pid = 13131L; final short epoch = 1; @@ -1961,23 +1961,17 @@ public void resendFailedProduceRequestAfterAbortableError() throws Exception { prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); prepareProduceResponse(Errors.NOT_LEADER_FOR_PARTITION, pid, epoch); - sender.run(time.milliseconds()); // Send AddPartitions and receive response - sender.run(time.milliseconds()); // Send produce request and receive NOT_LEADER_FOR_PARTITION response. + sender.run(time.milliseconds()); // Add partitions + sender.run(time.milliseconds()); // Produce assertFalse(responseFuture.isDone()); transactionManager.transitionToAbortableError(new KafkaException()); + prepareProduceResponse(Errors.NONE, pid, epoch); + sender.run(time.milliseconds()); - Deque unsentBatches = accumulator.batches().get(tp0); - - // The one unsent batch should have a reset sequence and should still be open. - assertEquals(1, unsentBatches.size()); - assertEquals(RecordBatch.NO_SEQUENCE, unsentBatches.getFirst().baseSequence()); - assertFalse(unsentBatches.getFirst().isClosed()); - - sender.run(time.milliseconds()); // abort the unsent batch. assertTrue(responseFuture.isDone()); - responseFuture.get(); // should throw the exception which caused the transaction to be aborted. + assertNotNull(responseFuture.get()); // should throw the exception which caused the transaction to be aborted. } @Test From c32515338d1fa9f8fd191c986defc4b1304a7a5a Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 6 Sep 2017 14:59:20 -0700 Subject: [PATCH 11/40] Rebased onto trunk and fixed some merge conflicts --- .../apache/kafka/clients/producer/internals/SenderTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 07fd6f8a27489..2ef4e1089333f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -1011,7 +1011,8 @@ private void setupWithTransactionState(TransactionManager transactionManager) { apiVersions, transactionManager); this.senderMetricsRegistry = new SenderMetricsRegistry(metricTags.keySet()); - this.sender = new Sender(logContext, this.client, this.metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, + + this.sender = new Sender(logContext, this.client, this.metadata, this.accumulator, false, MAX_REQUEST_SIZE, ACKS_ALL, Integer.MAX_VALUE, this.metrics, this.senderMetricsRegistry, this.time, REQUEST_TIMEOUT, 50, transactionManager, apiVersions); this.metadata.update(this.cluster, Collections.emptySet(), time.milliseconds()); } From 76ffdfbebd80d82d24817fec713419c87aa8f44d Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 6 Sep 2017 22:42:35 -0700 Subject: [PATCH 12/40] Added unit tests for out of order responses and expiring batches --- .../org/apache/kafka/clients/MockClient.java | 10 + .../producer/internals/SenderTest.java | 276 ++++++++++++++++++ 2 files changed, 286 insertions(+) diff --git a/clients/src/test/java/org/apache/kafka/clients/MockClient.java b/clients/src/test/java/org/apache/kafka/clients/MockClient.java index 9960ccef953d1..71e32ff759a38 100644 --- a/clients/src/test/java/org/apache/kafka/clients/MockClient.java +++ b/clients/src/test/java/org/apache/kafka/clients/MockClient.java @@ -224,6 +224,16 @@ public void respond(RequestMatcher matcher, AbstractResponse response) { respond(response); } + // Utility method to enable out of order responses + public void respondToRequest(ClientRequest clientRequest, AbstractResponse response) { + AbstractRequest request = clientRequest.requestBuilder().build(); + requests.remove(clientRequest); + short version = clientRequest.requestBuilder().desiredOrLatestVersion(); + responses.add(new ClientResponse(clientRequest.makeHeader(version), clientRequest.callback(), clientRequest.destination(), + clientRequest.createdTimeMs(), time.milliseconds(), false, null, response)); + } + + public void respond(AbstractResponse response, boolean disconnected) { ClientRequest request = requests.remove(); short version = request.requestBuilder().desiredOrLatestVersion(); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 2ef4e1089333f..1fac1238cf929 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -67,6 +67,7 @@ import java.nio.ByteBuffer; import java.util.Collections; +import java.util.Deque; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; @@ -79,6 +80,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -623,6 +625,280 @@ public void testMustNotRetryOutOfOrderSequenceForNextBatch() throws Exception { } } + @Test + public void testCorrectHandlingOfOutOfOrderResponses() throws Exception { + final long producerId = 343434L; + TransactionManager transactionManager = new TransactionManager(); + setupWithTransactionState(transactionManager); + prepareAndReceiveInitProducerId(producerId, Errors.NONE); + assertTrue(transactionManager.hasProducerId()); + + assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); + + // Send first ProduceRequest + Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + String nodeId = client.requests().peek().destination(); + Node node = new Node(Integer.valueOf(nodeId), "localhost", 0); + assertEquals(1, client.inFlightRequestCount()); + assertEquals(1, transactionManager.sequenceNumber(tp0).longValue()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + + // Send second ProduceRequest + Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + assertEquals(2, client.inFlightRequestCount()); + assertEquals(2, transactionManager.sequenceNumber(tp0).longValue()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + assertFalse(request1.isDone()); + assertFalse(request2.isDone()); + assertTrue(client.isReady(node, time.milliseconds())); + + ClientRequest firstClientRequest = client.requests().peek(); + ClientRequest secondClientRequest = (ClientRequest) client.requests().toArray()[1]; + + client.respondToRequest(secondClientRequest, produceResponse(tp0, -1, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, -1)); + + sender.run(time.milliseconds()); // receive response 1 + Deque queuedBatches = accumulator.batches().get(tp0); + + // Make sure that we are queueing the second batch first. + assertEquals(1, queuedBatches.size()); + assertEquals(1, queuedBatches.peekFirst().baseSequence()); + assertEquals(1, client.inFlightRequestCount()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + + client.respondToRequest(firstClientRequest, produceResponse(tp0, -1, Errors.NOT_LEADER_FOR_PARTITION, -1)); + + sender.run(time.milliseconds()); // receive response 0 + + // Make sure we requeued both batches in the correct order. + assertEquals(2, queuedBatches.size()); + assertEquals(0, queuedBatches.peekFirst().baseSequence()); + assertEquals(1, queuedBatches.peekLast().baseSequence()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + assertEquals(0, client.inFlightRequestCount()); + + sender.run(time.milliseconds()); // send request 0 + assertEquals(1, client.inFlightRequestCount()); + sender.run(time.milliseconds()); // don't do anything, only one inflight allowed once we are retrying. + + assertEquals(1, client.inFlightRequestCount()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + + // Make sure that the requests are sent in order, even though the previous responses were not in order. + sendIdempotentProducerResponse(0, tp0, Errors.NONE, 0L); + sender.run(time.milliseconds()); // receive response 0 + assertEquals(0, transactionManager.lastAckedSequence(tp0)); + assertEquals(0, client.inFlightRequestCount()); + sender.run(time.milliseconds()); // send request 1 + assertEquals(1, client.inFlightRequestCount()); + sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L); + sender.run(time.milliseconds()); // receive response 1 + + assertFalse(client.hasInFlightRequests()); + assertEquals(1, transactionManager.lastAckedSequence(tp0)); + } + + @Test + public void testCorrectHandlingOfOutOfOrderResponsesWhenSecondSucceeds() throws Exception { + final long producerId = 343434L; + TransactionManager transactionManager = new TransactionManager(); + setupWithTransactionState(transactionManager); + prepareAndReceiveInitProducerId(producerId, Errors.NONE); + assertTrue(transactionManager.hasProducerId()); + + assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); + + // Send first ProduceRequest + Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + String nodeId = client.requests().peek().destination(); + Node node = new Node(Integer.valueOf(nodeId), "localhost", 0); + assertEquals(1, client.inFlightRequestCount()); + + // Send second ProduceRequest + Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + assertEquals(2, client.inFlightRequestCount()); + assertFalse(request1.isDone()); + assertFalse(request2.isDone()); + assertTrue(client.isReady(node, time.milliseconds())); + + ClientRequest firstClientRequest = client.requests().peek(); + ClientRequest secondClientRequest = (ClientRequest) client.requests().toArray()[1]; + + client.respondToRequest(secondClientRequest, produceResponse(tp0, 1, Errors.NONE, -1)); + + sender.run(time.milliseconds()); // receive response 1 + assertNotNull(request2.get()); + assertFalse(request1.isDone()); + Deque queuedBatches = accumulator.batches().get(tp0); + + assertEquals(0, queuedBatches.size()); + assertEquals(1, client.inFlightRequestCount()); + assertEquals(1, transactionManager.lastAckedSequence(tp0)); + + client.respondToRequest(firstClientRequest, produceResponse(tp0, -1, Errors.REQUEST_TIMED_OUT, -1)); + + sender.run(time.milliseconds()); // receive response 0 + + // Make sure we requeued both batches in the correct order. + assertEquals(1, queuedBatches.size()); + assertEquals(0, queuedBatches.peekFirst().baseSequence()); + assertEquals(1, transactionManager.lastAckedSequence(tp0)); + assertEquals(0, client.inFlightRequestCount()); + + sender.run(time.milliseconds()); // resend request 0 + assertEquals(1, client.inFlightRequestCount()); + + assertEquals(1, client.inFlightRequestCount()); + assertEquals(1, transactionManager.lastAckedSequence(tp0)); + + // Make sure we handle the out of order successful responses correctly. + sendIdempotentProducerResponse(0, tp0, Errors.NONE, 0L); + sender.run(time.milliseconds()); // receive response 0 + assertEquals(0, queuedBatches.size()); + assertEquals(1, transactionManager.lastAckedSequence(tp0)); + assertEquals(0, client.inFlightRequestCount()); + + assertFalse(client.hasInFlightRequests()); + assertNotNull(request1.get()); + } + + @Test + public void testExpiryOfUnsentBatchesShouldNotCauseUnresolvedSequences() throws Exception { + final long producerId = 343434L; + TransactionManager transactionManager = new TransactionManager(); + setupWithTransactionState(transactionManager); + prepareAndReceiveInitProducerId(producerId, Errors.NONE); + assertTrue(transactionManager.hasProducerId()); + + assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); + + // Send first ProduceRequest + Future request1 = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + Node node = this.cluster.nodes().get(0); + time.sleep(10000L); + client.disconnect(node.idString()); + client.blackout(node, 10); + + sender.run(time.milliseconds()); + + assertTrue(request1.isDone()); + try { + request1.get(); + fail("Should have raised timeout exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof TimeoutException); + } + assertFalse(transactionManager.partitionHasUnresolvedSequence(tp0)); + } + + @Test + public void testExpiryOfFirstBatchShouldNotCauseUnresolvedSequencesIfFutureBatchesSucceed() throws Exception { + final long producerId = 343434L; + TransactionManager transactionManager = new TransactionManager(); + setupWithTransactionState(transactionManager); + prepareAndReceiveInitProducerId(producerId, Errors.NONE); + assertTrue(transactionManager.hasProducerId()); + + assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); + + // Send first ProduceRequest + Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // send request + + Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // send request + + assertEquals(2, client.inFlightRequestCount()); + + sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_FOR_PARTITION, -1); + sender.run(time.milliseconds()); // receive first response + + Node node = this.cluster.nodes().get(0); + time.sleep(10000L); + client.disconnect(node.idString()); + client.blackout(node, 10); + + sender.run(time.milliseconds()); // now expire the first batch. + assertTrue(request1.isDone()); + try { + request1.get(); + fail("Should have raised timeout exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof TimeoutException); + } + assertTrue(transactionManager.partitionHasUnresolvedSequence(tp0)); + // let's enqueue another batch, which should not be dequeued until the unresolved state is clear. + Future request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + + time.sleep(20); + + assertFalse(request2.isDone()); + + sender.run(time.milliseconds()); // send second request + sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1); + sender.run(time.milliseconds()); // receive second response. + assertNotNull(request2.get()); + Deque batches = accumulator.batches().get(tp0); + + assertEquals(1, batches.size()); + assertFalse(batches.peekFirst().hasSequence()); + assertFalse(client.hasInFlightRequests()); + assertEquals(2L, transactionManager.sequenceNumber(tp0).longValue()); + assertTrue(transactionManager.partitionHasUnresolvedSequence(tp0)); + + sender.run(time.milliseconds()); // clear the unresolved state, send the pending request. + assertFalse(transactionManager.partitionHasUnresolvedSequence(tp0)); + assertTrue(transactionManager.hasProducerId()); + assertEquals(0, batches.size()); + assertEquals(1, client.inFlightRequestCount()); + assertFalse(request3.isDone()); + } + + @Test + public void testExpiryOfAllSentBatchesShouldCauseUnresolvedSequences() throws Exception { + final long producerId = 343434L; + TransactionManager transactionManager = new TransactionManager(); + setupWithTransactionState(transactionManager); + prepareAndReceiveInitProducerId(producerId, Errors.NONE); + assertTrue(transactionManager.hasProducerId()); + + assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); + + // Send first ProduceRequest + Future request1 = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // send request + sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_FOR_PARTITION, -1); + sender.run(time.milliseconds()); // receive response + + assertEquals(1L, transactionManager.sequenceNumber(tp0).longValue()); + + Node node = this.cluster.nodes().get(0); + time.sleep(10000L); + client.disconnect(node.idString()); + client.blackout(node, 10); + + sender.run(time.milliseconds()); // now expire the batch. + + assertTrue(request1.isDone()); + try { + request1.get(); + fail("Should have raised timeout exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof TimeoutException); + } + assertTrue(transactionManager.partitionHasUnresolvedSequence(tp0)); + assertFalse(client.hasInFlightRequests()); + Deque batches = accumulator.batches().get(tp0); + assertEquals(0, batches.size()); + assertTrue(transactionManager.hasProducerId()); + + sender.run(time.milliseconds()); // we should reset the producer state. + assertFalse(transactionManager.hasProducerId()); + } void sendIdempotentProducerResponse(final int expectedSequence, TopicPartition tp, Errors responseError, long responseOffset) { client.respond(new MockClient.RequestMatcher() { From addbc8faa1da4b431d0675def014fa705d74a037 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 7 Sep 2017 15:05:17 -0700 Subject: [PATCH 13/40] Address PR comments on client side code --- .../kafka/clients/producer/KafkaProducer.java | 2 +- .../producer/internals/RecordAccumulator.java | 15 ++--- .../clients/producer/internals/Sender.java | 6 +- .../internals/TransactionManager.java | 53 ++++++++++------ .../producer/internals/SenderTest.java | 61 ++++++++++++++++++- 5 files changed, 104 insertions(+), 33 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index 1130a352defe8..b630d611dc235 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -484,7 +484,7 @@ private static int configureRetries(ProducerConfig config, boolean idempotenceEn private static int configureInflightRequests(ProducerConfig config, boolean idempotenceEnabled) { if (idempotenceEnabled && 5 < config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION)) { throw new ConfigException("Must set " + ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION + " to at most 5" + - " to use the idempotent producer. Otherwise we cannot guarantee idempotence."); + " to use the idempotent producer."); } return config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION); } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index 3d0a4fa7a78e9..e866093e6214e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -530,22 +530,19 @@ public Map> drain(Cluster cluster, ProducerBatch batch = deque.pollFirst(); if (producerIdAndEpoch != null && !batch.hasSequence()) { - // If the batch is in retry, then we should not change the producer id and + // If the batch already has an assigned sequence, then we should not change the producer id and // sequence number, since this may introduce duplicates. In particular, // the previous attempt may actually have been accepted, and if we change // the producer id and sequence here, this attempt will also be accepted, // causing a duplicate. - - // set the sequence to be the next sequence here, increment the next - // sequence by the record count of the batch. // - // If a batch errors out fatally, then the next sequence will be set to the sequence - // of that batch in the error handling step. Future batches will fail with an - // out of sequence error. Those batches will be 'reset' (retry bit cleared, etc.) and reenqueued. - // When they come here they will inherit the sequence of the lost batch and proceed. + // Additionally, we update the next sequence number bound for the partition, + // and also have the transaction manager track the batch so as to ensure + // that sequence ordering is maintained even if we receive out of order + // responses. batch.setProducerState(producerIdAndEpoch, transactionManager.sequenceNumber(batch.topicPartition), isTransactional); transactionManager.incrementSequenceNumber(batch.topicPartition, batch.recordCount); - log.debug("Assigned producerId {} and producerEpoch {} to batch with sequence " + + log.debug("Assigned producerId {} and producerEpoch {} to batch with base sequence " + "{} being sent to partition {}", producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, batch.baseSequence(), tp); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 1270b9c8e3df5..7f18c7047b74a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -281,7 +281,6 @@ private long sendProducerData(long now) { // This ensures that no new batches are drained until the current in flight batches are fully resolved. transactionManager.markSequenceUnresolved(expiredBatch.topicPartition); } - this.sensors.recordErrors(expiredBatch.topicPartition.topic(), expiredBatch.recordCount); } sensors.updateProduceRequestMetrics(batches); @@ -528,7 +527,6 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons failBatch(batch, response, new OutOfOrderSequenceException("Attempted to retry sending a " + "batch but the producer id changed from " + batch.producerId() + " to " + transactionManager.producerIdAndEpoch().producerId + " in the mean time. This batch will be dropped."), false); - this.sensors.recordErrors(batch.topicPartition.topic(), batch.recordCount); } } else { final RuntimeException exception; @@ -540,7 +538,6 @@ else if (error == Errors.CLUSTER_AUTHORIZATION_FAILED) exception = error.exception(); // tell the user the result of their request failBatch(batch, response, exception, true); - this.sensors.recordErrors(batch.topicPartition.topic(), batch.recordCount); } if (error.exception() instanceof InvalidMetadataException) { if (error.exception() instanceof UnknownTopicOrPartitionException) @@ -577,7 +574,7 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons } batch.done(response.baseOffset, response.logAppendTime, null); - + this.sensors.recordErrors(batch.topicPartition.topic(), batch.recordCount); this.accumulator.deallocate(batch); } @@ -611,6 +608,7 @@ private void failBatch(ProducerBatch batch, long baseOffset, long logAppendTime, transactionManager.adjustSequencesDueToFailedBatch(batch); } + this.sensors.recordErrors(batch.topicPartition.topic(), batch.recordCount); batch.done(baseOffset, logAppendTime, exception); this.accumulator.deallocate(batch); } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index bce8d703e171d..e9489c0b94428 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -65,10 +65,27 @@ public class TransactionManager { private final String transactionalId; private final int transactionTimeoutMs; - private final Map sequenceNumbers; + // The base sequence of the next batch bound for a given partition. + private final Map nextSequence; + + // The sequence of the last record of the last ack'd batch from the given partition. When there are no + // in flight requests for a partition, the lastAckedSequence(topicPartition) == nextSequence(topicPartition) - 1. private final Map lastAckedSequence; - private final Set partitionsWithUnresolvedSequenceNumbers; + + // If a batch bound for a partition expired locally after being sent at least once, the partition has is considered + // to have an unresolved state. We keep track fo such partitions here, and cannot assign any more sequence numbers + // for this partition until the unresolved state gets cleared. This may happen if other inflight batches returned + // successfully (indicating that the expired batch actually made it to the broker). If we don't get any successful + // responses for the partition once the inflight request count falls to zero, we reset the producer id and + // consequently clear this data structure as well. + private final Set partitionsWithUnresolvedSequences; + + // Keep track of the in flight batches bound for a partition, ordered by sequence. This helps us to ensure that + // we continue to order batches by the sequence numbers even when the responses come back out of order during + // leader failover. We add a batch to the queue when it is drained, and remove it when the batch completes + // (either successfully or through a fatal failure). private final Map> inflightBatchesBySequence; + private final PriorityQueue pendingRequests; private final Set newPartitionsInTransaction; private final Set pendingPartitionsInTransaction; @@ -145,7 +162,7 @@ private enum Priority { public TransactionManager(LogContext logContext, String transactionalId, int transactionTimeoutMs, long retryBackoffMs) { this.producerIdAndEpoch = new ProducerIdAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); - this.sequenceNumbers = new HashMap<>(); + this.nextSequence = new HashMap<>(); this.lastAckedSequence = new HashMap<>(); this.transactionalId = transactionalId; this.log = logContext.logger(TransactionManager.class); @@ -163,7 +180,7 @@ public int compare(TxnRequestHandler o1, TxnRequestHandler o2) { } }); - this.partitionsWithUnresolvedSequenceNumbers = new HashSet<>(); + this.partitionsWithUnresolvedSequences = new HashSet<>(); this.inflightBatchesBySequence = new HashMap<>(); this.retryBackoffMs = retryBackoffMs; @@ -177,7 +194,7 @@ public synchronized TransactionalRequestResult initializeTransactions() { ensureTransactional(); transitionTo(State.INITIALIZING); setProducerIdAndEpoch(ProducerIdAndEpoch.NONE); - this.sequenceNumbers.clear(); + this.nextSequence.clear(); InitProducerIdRequest.Builder builder = new InitProducerIdRequest.Builder(transactionalId, transactionTimeoutMs); InitProducerIdHandler handler = new InitProducerIdHandler(builder); enqueueRequest(handler); @@ -369,31 +386,31 @@ synchronized void resetProducerId() { throw new IllegalStateException("Cannot reset producer state for a transactional producer. " + "You must either abort the ongoing transaction or reinitialize the transactional producer instead"); setProducerIdAndEpoch(ProducerIdAndEpoch.NONE); - this.sequenceNumbers.clear(); + this.nextSequence.clear(); this.lastAckedSequence.clear(); this.inflightBatchesBySequence.clear(); - this.partitionsWithUnresolvedSequenceNumbers.clear(); + this.partitionsWithUnresolvedSequences.clear(); } /** * Returns the next sequence number to be written to the given TopicPartition. */ synchronized Integer sequenceNumber(TopicPartition topicPartition) { - Integer currentSequenceNumber = sequenceNumbers.get(topicPartition); + Integer currentSequenceNumber = nextSequence.get(topicPartition); if (currentSequenceNumber == null) { currentSequenceNumber = 0; - sequenceNumbers.put(topicPartition, currentSequenceNumber); + nextSequence.put(topicPartition, currentSequenceNumber); } return currentSequenceNumber; } synchronized void incrementSequenceNumber(TopicPartition topicPartition, int increment) { - Integer currentSequenceNumber = sequenceNumbers.get(topicPartition); + Integer currentSequenceNumber = nextSequence.get(topicPartition); if (currentSequenceNumber == null) throw new IllegalStateException("Attempt to increment sequence number for a partition with no current sequence."); currentSequenceNumber += increment; - sequenceNumbers.put(topicPartition, currentSequenceNumber); + nextSequence.put(topicPartition, currentSequenceNumber); } synchronized void startTrackingBatch(ProducerBatch batch) { @@ -438,7 +455,7 @@ synchronized int lastAckedSequence(TopicPartition topicPartition) { // This method must only be called when we know that the batch is question has been unequivocally failed by the broker, // ie. it has received a confirmed fatal status code like 'Message Too Large' or something similar. synchronized void adjustSequencesDueToFailedBatch(ProducerBatch batch) { - if (!this.sequenceNumbers.containsKey(batch.topicPartition)) + if (!this.nextSequence.containsKey(batch.topicPartition)) // Sequence numbers are not being tracked for this partition. This could happen if the producer id was just // reset due to a previous OutOfOrderSequenceException. return; @@ -465,25 +482,25 @@ synchronized boolean hasInflightBatches(TopicPartition topicPartition) { } synchronized boolean partitionHasUnresolvedSequence(TopicPartition topicPartition) { - return partitionsWithUnresolvedSequenceNumbers.contains(topicPartition); + return partitionsWithUnresolvedSequences.contains(topicPartition); } synchronized void markSequenceUnresolved(TopicPartition topicPartition) { - partitionsWithUnresolvedSequenceNumbers.add(topicPartition); + partitionsWithUnresolvedSequences.add(topicPartition); } // Checks if there are any partitions with unresolved partitions which may now be resolved. Returns true if // the producer id needs a reset, false otherwise. synchronized boolean maybeResolveSequences() { - for (TopicPartition topicPartition : partitionsWithUnresolvedSequenceNumbers) { + for (TopicPartition topicPartition : partitionsWithUnresolvedSequences) { if (!hasInflightBatches(topicPartition)) { // The partition has been fully drained. At this point, the last ack'd sequence should be once less than // next sequence destined for the partition. If so, the partition is fully resolved. If not, we should // reset the sequence number if necessary. if (lastAckedSequence(topicPartition) == sequenceNumber(topicPartition) - 1) { // This would happen when a batch was expired, but subsequent batches succeeded. - partitionsWithUnresolvedSequenceNumbers.remove(topicPartition); + partitionsWithUnresolvedSequences.remove(topicPartition); } else { // We would enter this branch if all in flight batches were ultimately expired in the producer. log.info("No inflight batches remaining for {}, last ack'd sequence for partition is {}, next sequence is {}. " + @@ -501,10 +518,10 @@ synchronized boolean isNextSequence(TopicPartition topicPartition, int sequence) } synchronized void setNextSequence(TopicPartition topicPartition, int sequence) { - if (!sequenceNumbers.containsKey(topicPartition) && sequence != 0) + if (!nextSequence.containsKey(topicPartition) && sequence != 0) throw new IllegalStateException("Trying to set the sequence number for " + topicPartition + " to " + sequence + ", but the sequence number was never set for this partition."); - sequenceNumbers.put(topicPartition, sequence); + nextSequence.put(topicPartition, sequence); } synchronized TxnRequestHandler nextRequestHandler(boolean hasIncompleteBatches) { diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 1fac1238cf929..0d4067d34a6be 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -840,7 +840,7 @@ public void testExpiryOfFirstBatchShouldNotCauseUnresolvedSequencesIfFutureBatch sender.run(time.milliseconds()); // send second request sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1); - sender.run(time.milliseconds()); // receive second response. + sender.run(time.milliseconds()); // receive second response, the third request shouldn't be sent since we are in an unresolved state. assertNotNull(request2.get()); Deque batches = accumulator.batches().get(tp0); @@ -858,6 +858,65 @@ public void testExpiryOfFirstBatchShouldNotCauseUnresolvedSequencesIfFutureBatch assertFalse(request3.isDone()); } + @Test + public void testExpiryOfFirstBatchShouldCauseResetIfFutureBatchesFail() throws Exception { + final long producerId = 343434L; + TransactionManager transactionManager = new TransactionManager(); + setupWithTransactionState(transactionManager); + prepareAndReceiveInitProducerId(producerId, Errors.NONE); + assertTrue(transactionManager.hasProducerId()); + + assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); + + // Send first ProduceRequest + Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // send request + + Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); // send request + + assertEquals(2, client.inFlightRequestCount()); + + sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_FOR_PARTITION, -1); + sender.run(time.milliseconds()); // receive first response + + Node node = this.cluster.nodes().get(0); + time.sleep(10000L); + client.disconnect(node.idString()); + client.blackout(node, 10); + + sender.run(time.milliseconds()); // now expire the first batch. + assertTrue(request1.isDone()); + try { + request1.get(); + fail("Should have raised timeout exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof TimeoutException); + } + assertTrue(transactionManager.partitionHasUnresolvedSequence(tp0)); + // let's enqueue another batch, which should not be dequeued until the unresolved state is clear. + Future request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + + time.sleep(20); + + assertFalse(request2.isDone()); + + sender.run(time.milliseconds()); // send second request + sendIdempotentProducerResponse(1, tp0, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, 1); + sender.run(time.milliseconds()); // receive second response, the third request shouldn't be sent since we are in an unresolved state. + assertTrue(request2.isDone()); + Deque batches = accumulator.batches().get(tp0); + + // The second request should not be requeued. + assertEquals(1, batches.size()); + assertFalse(batches.peekFirst().hasSequence()); + assertFalse(client.hasInFlightRequests()); + + // The producer state should be reset. + assertFalse(transactionManager.hasProducerId()); + assertFalse(transactionManager.partitionHasUnresolvedSequence(tp0)); + } + @Test public void testExpiryOfAllSentBatchesShouldCauseUnresolvedSequences() throws Exception { final long producerId = 343434L; From c733e52c90fff1c07e9fadd981185b822e442ce6 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 7 Sep 2017 21:29:39 -0700 Subject: [PATCH 14/40] WIP --- .../src/main/scala/kafka/log/LogCleaner.scala | 4 +- .../kafka/log/ProducerStateManager.scala | 143 ++++-------------- 2 files changed, 33 insertions(+), 114 deletions(-) diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala index 09b80f03a35f7..4f53b41df4fc0 100644 --- a/core/src/main/scala/kafka/log/LogCleaner.scala +++ b/core/src/main/scala/kafka/log/LogCleaner.scala @@ -513,9 +513,9 @@ private[log] class Cleaner(val id: Int, // note that we will never delete a marker until all the records from that transaction are removed. discardBatchRecords = shouldDiscardBatch(batch, transactionMetadata, retainTxnMarkers = retainDeletes) - // check if the batch metadata is currently being cached. if so, we cannot + // check if the batch contains the last sequence number for the producer. if so, we cannot // remove the batch just yet or the producer may see an out of sequence error. - if (batch.hasProducerId && activeProducers.get(batch.producerId).exists(_.hasBatchWithSequenceRange(batch.baseSequence, batch.lastSequence).nonEmpty)) + if (batch.hasProducerId && activeProducers.get(batch.producerId).exists(_.lastSeq == batch.lastSequence)) BatchRetention.RETAIN_EMPTY else if (discardBatchRecords) BatchRetention.DELETE diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index 978932bb32728..cb2db671ab9ea 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -70,8 +70,6 @@ private[log] case class ProducerIdEntry(producerId: Long, batchMetadata: mutable var producerEpoch: Short, var coordinatorEpoch: Int, var currentTxnFirstOffset: Option[Long]) { - - def firstSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.front.firstSeq def firstOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.front.firstOffset @@ -103,18 +101,18 @@ private[log] case class ProducerIdEntry(producerId: Long, batchMetadata: mutable batchMetadata.clear() retainedMetadata.foreach(batchMetadata.enqueue(_)) } - } + } def duplicateOf(batch: RecordBatch): Option[BatchMetadata] = { if (batch.producerEpoch() != producerEpoch) return None hasBatchWithSequenceRange(batch.baseSequence(), batch.lastSequence()) - } + } // Return the batch metadata of the cached batch having the exact sequence range, if any. def hasBatchWithSequenceRange(firstSeq: Int, lastSeq:Int) : Option[BatchMetadata] = { - val duplicate = batchMetadata.filter{ case(metadata) => + val duplicate = batchMetadata.filter { case(metadata) => firstSeq == metadata.firstSeq && lastSeq == metadata.lastSeq } @@ -296,7 +294,7 @@ private[log] class ProducerAppendInfo(val producerId: Long, } object ProducerStateManager { - private val ProducerSnapshotVersion: Short = 2 + private val ProducerSnapshotVersion: Short = 1 private val VersionField = "version" private val CrcField = "crc" private val ProducerIdField = "producer_id" @@ -308,34 +306,11 @@ object ProducerStateManager { private val ProducerEntriesField = "producer_entries" private val CoordinatorEpochField = "coordinator_epoch" private val CurrentTxnFirstOffsetField = "current_txn_first_offset" - private val RecordBatchMetadataField = "record_batch_metadata" private val VersionOffset = 0 private val CrcOffset = VersionOffset + 2 private val ProducerEntriesOffset = CrcOffset + 4 - - val RecordBatchMetadataSchema = new Schema( - new Field(LastSequenceField, Type.INT32, "Last written sequence of the producer"), - new Field(LastOffsetField, Type.INT64, "Last written offset of the producer"), - new Field(OffsetDeltaField, Type.INT32, "The difference of the last sequence and first sequence in the last written batch"), - new Field(TimestampField, Type.INT64, "Max timestamp from the last written entry") - ) - - val ProducerSnapshotEntrySchemaV2 = new Schema( - new Field(ProducerIdField, Type.INT64, "The producer ID"), - new Field(ProducerEpochField, Type.INT16, "Current epoch of the producer"), - new Field(CoordinatorEpochField, Type.INT32, "The epoch of the last transaction coordinator to send an end transaction marker"), - new Field(CurrentTxnFirstOffsetField, Type.INT64, "The first offset of the on-going transaction (-1 if there is none)"), - new Field(RecordBatchMetadataField, new ArrayOf(RecordBatchMetadataSchema), "The record metadata for up to the last 5 batches appended by the producer") - ) - - val PidSnapshotMapSchemaV2 = new Schema( - new Field(VersionField, Type.INT16, "Version of the snapshot file."), - new Field(CrcField, Type.UNSIGNED_INT32, "CRC of the snapshot data"), - new Field(ProducerEntriesField, new ArrayOf(ProducerSnapshotEntrySchemaV2), "The entries in the producer table") - ) - val ProducerSnapshotEntrySchema = new Schema( new Field(ProducerIdField, Type.INT64, "The producer ID"), new Field(ProducerEpochField, Type.INT16, "Current epoch of the producer"), @@ -345,7 +320,6 @@ object ProducerStateManager { new Field(TimestampField, Type.INT64, "Max timestamp from the last written entry"), new Field(CoordinatorEpochField, Type.INT32, "The epoch of the last transaction coordinator to send an end transaction marker"), new Field(CurrentTxnFirstOffsetField, Type.INT64, "The first offset of the on-going transaction (-1 if there is none)")) - val PidSnapshotMapSchema = new Schema( new Field(VersionField, Type.INT16, "Version of the snapshot file"), new Field(CrcField, Type.UNSIGNED_INT32, "CRC of the snapshot data"), @@ -353,111 +327,56 @@ object ProducerStateManager { def readSnapshot(file: File): Iterable[ProducerIdEntry] = { try { - val rawBuffer = Files.readAllBytes(file.toPath) - val buffer = ByteBuffer.wrap(rawBuffer) - val version = buffer.getShort() - buffer.rewind() - - val struct = version match { - case 1 => - PidSnapshotMapSchema.read(buffer) - case 2 => - PidSnapshotMapSchemaV2.read(buffer) - case _ => - throw new IllegalArgumentException(s"Snapshot contained unknown file version $version") - } + val buffer = Files.readAllBytes(file.toPath) + val struct = PidSnapshotMapSchema.read(ByteBuffer.wrap(buffer)) + + val version = struct.getShort(VersionField) + if (version != ProducerSnapshotVersion) + throw new CorruptSnapshotException(s"Snapshot contained an unknown file version $version") val crc = struct.getUnsignedInt(CrcField) - val computedCrc = Crc32C.compute(rawBuffer, ProducerEntriesOffset, rawBuffer.length - ProducerEntriesOffset) + val computedCrc = Crc32C.compute(buffer, ProducerEntriesOffset, buffer.length - ProducerEntriesOffset) if (crc != computedCrc) throw new CorruptSnapshotException(s"Snapshot is corrupt (CRC is no longer valid). " + s"Stored crc: $crc. Computed crc: $computedCrc") - loadProducerEntries(struct, version) - } - catch { + struct.getArray(ProducerEntriesField).map { producerEntryObj => + val producerEntryStruct = producerEntryObj.asInstanceOf[Struct] + val producerId: Long = producerEntryStruct.getLong(ProducerIdField) + val producerEpoch = producerEntryStruct.getShort(ProducerEpochField) + val seq = producerEntryStruct.getInt(LastSequenceField) + val offset = producerEntryStruct.getLong(LastOffsetField) + val timestamp = producerEntryStruct.getLong(TimestampField) + val offsetDelta = producerEntryStruct.getInt(OffsetDeltaField) + val coordinatorEpoch = producerEntryStruct.getInt(CoordinatorEpochField) + val currentTxnFirstOffset = producerEntryStruct.getLong(CurrentTxnFirstOffsetField) + val newEntry = ProducerIdEntry(producerId, mutable.Queue[BatchMetadata](BatchMetadata(seq, offset, offsetDelta, timestamp)), producerEpoch, + coordinatorEpoch, if (currentTxnFirstOffset >= 0) Some(currentTxnFirstOffset) else None) + newEntry + } + } catch { case e: SchemaException => throw new CorruptSnapshotException(s"Snapshot failed schema validation: ${e.getMessage}") - case e: BufferUnderflowException => - throw new CorruptSnapshotException(s"Could not detect the version of the snapshot schema: ${e.getMessage}") - } - } - - private def loadProducerEntries(struct: Struct, version: Short) : Iterable[ProducerIdEntry] = { - version match { - case 1 => - loadProducerEntriesV1(struct) - case 2 => - loadProducerEntriesV2(struct) - } - } - - private def loadProducerEntriesV1(struct: Struct) : Iterable[ProducerIdEntry] = { - struct.getArray(ProducerEntriesField).map { producerEntryObj => - val producerEntryStruct = producerEntryObj.asInstanceOf[Struct] - val producerId = producerEntryStruct.getLong(ProducerIdField) - val producerEpoch = producerEntryStruct.getShort(ProducerEpochField) - val seq = producerEntryStruct.getInt(LastSequenceField) - val offset = producerEntryStruct.getLong(LastOffsetField) - val timestamp = producerEntryStruct.getLong(TimestampField) - val offsetDelta = producerEntryStruct.getInt(OffsetDeltaField) - val coordinatorEpoch = producerEntryStruct.getInt(CoordinatorEpochField) - val currentTxnFirstOffset = producerEntryStruct.getLong(CurrentTxnFirstOffsetField) - val newEntry = ProducerIdEntry(producerId, - mutable.Queue[BatchMetadata](BatchMetadata(seq, offset, offsetDelta, timestamp)), producerEpoch, coordinatorEpoch, - if (currentTxnFirstOffset >= 0) Some(currentTxnFirstOffset) else None) - newEntry - } - } - - private def loadProducerEntriesV2(struct: Struct) : Iterable[ProducerIdEntry] = { - struct.getArray(ProducerEntriesField).map { producerEntryObj => - val producerEntryStruct = producerEntryObj.asInstanceOf[Struct] - val producerId = producerEntryStruct.getLong(ProducerIdField) - val producerEpoch= producerEntryStruct.getShort(ProducerEpochField) - val coordinatorEpoch = producerEntryStruct.getInt(CoordinatorEpochField) - val currentTxnFirstOffset = producerEntryStruct.getLong(CurrentTxnFirstOffsetField) - - val recordBatchMetadata = new mutable.Queue[BatchMetadata]() - producerEntryStruct.getArray(RecordBatchMetadataField).foreach { recordBatchMetadataObj => - val recordBatchMetadataStruct = recordBatchMetadataObj.asInstanceOf[Struct] - val batchMetadata = BatchMetadata(recordBatchMetadataStruct.getInt(LastSequenceField), - recordBatchMetadataStruct.getLong(LastOffsetField), recordBatchMetadataStruct.getInt(OffsetDeltaField), - recordBatchMetadataStruct.getLong(TimestampField)) - recordBatchMetadata.enqueue(batchMetadata) - } - - val newEntry = ProducerIdEntry(producerId, recordBatchMetadata, producerEpoch, coordinatorEpoch, - if (currentTxnFirstOffset >= 0) Some(currentTxnFirstOffset) else None) - newEntry } } private def writeSnapshot(file: File, entries: mutable.Map[Long, ProducerIdEntry]) { - val struct = new Struct(PidSnapshotMapSchemaV2) + val struct = new Struct(PidSnapshotMapSchema) struct.set(VersionField, ProducerSnapshotVersion) struct.set(CrcField, 0L) // we'll fill this after writing the entries val entriesArray = entries.map { case (producerId, entry) => val producerEntryStruct = struct.instance(ProducerEntriesField) - - val batchMetadataArray = entry.batchMetadata.map { batchMetadata => - val batchMetadataStruct = new Struct(RecordBatchMetadataSchema) - batchMetadataStruct - .set(LastSequenceField, batchMetadata.lastSeq) - .set(LastOffsetField, batchMetadata.lastOffset) - .set(OffsetDeltaField, batchMetadata.offsetDelta) - .set(TimestampField, batchMetadata.timestamp) - batchMetadataStruct - }.toArray producerEntryStruct.set(ProducerIdField, producerId) .set(ProducerEpochField, entry.producerEpoch) + .set(LastSequenceField, entry.batchMetadata.last.lastSeq) + .set(LastOffsetField, entry.batchMetadata.last.lastOffset) + .set(OffsetDeltaField, entry.batchMetadata.last.offsetDelta) + .set(TimestampField, entry.batchMetadata.last.timestamp) .set(CoordinatorEpochField, entry.coordinatorEpoch) .set(CurrentTxnFirstOffsetField, entry.currentTxnFirstOffset.getOrElse(-1L)) - .set(RecordBatchMetadataField, batchMetadataArray) producerEntryStruct }.toArray - struct.set(ProducerEntriesField, entriesArray) val buffer = ByteBuffer.allocate(struct.sizeOf) From 3c03f67be9214341dcb16d099d2c7a187660aa6d Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 7 Sep 2017 22:28:58 -0700 Subject: [PATCH 15/40] Revert changes where we stored the metadata for the last 5 batches. We don't need this information except in extremely rare cases, and it doesn't seem worth the cost of supporting two on disk versions of the snapshot, etc. --- .../kafka/log/ProducerStateManager.scala | 6 +- .../scala/unit/kafka/log/LogCleanerTest.scala | 80 ++++++++----------- 2 files changed, 36 insertions(+), 50 deletions(-) diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index cb2db671ab9ea..72b0637a2fb14 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -107,11 +107,11 @@ private[log] case class ProducerIdEntry(producerId: Long, batchMetadata: mutable if (batch.producerEpoch() != producerEpoch) return None - hasBatchWithSequenceRange(batch.baseSequence(), batch.lastSequence()) + batchWithSequenceRange(batch.baseSequence(), batch.lastSequence()) } // Return the batch metadata of the cached batch having the exact sequence range, if any. - def hasBatchWithSequenceRange(firstSeq: Int, lastSeq:Int) : Option[BatchMetadata] = { + def batchWithSequenceRange(firstSeq: Int, lastSeq:Int) : Option[BatchMetadata] = { val duplicate = batchMetadata.filter { case(metadata) => firstSeq == metadata.firstSeq && lastSeq == metadata.lastSeq } @@ -178,7 +178,7 @@ private[log] class ProducerAppendInfo(val producerId: Long, // the epoch was bumped by a control record, so we expect the sequence number to be reset throw new OutOfOrderSequenceException(s"Out of order sequence number for producerId $producerId: found $firstSeq " + s"(incoming seq. number), but expected 0") - } else if (currentEntry.hasBatchWithSequenceRange(firstSeq, lastSeq).isDefined) { + } else if (currentEntry.batchWithSequenceRange(firstSeq, lastSeq).isDefined) { throw new DuplicateSequenceNumberException(s"Duplicate sequence number for producerId $producerId: (incomingBatch.firstSeq, " + s"incomingBatch.lastSeq): ($firstSeq, $lastSeq).") } else if (!inSequence(firstSeq, lastSeq)) { diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala index c4d1c0d220a38..517e8760a870f 100755 --- a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala @@ -281,26 +281,23 @@ class LogCleanerTest extends JUnitSuite { assertEquals(List(0, 2, 3, 4, 5), offsetsInLog(log)) appendProducer(Seq(1, 3)) - appendProducer(Seq(4)) - appendProducer(Seq(5)) - appendProducer(Seq(6)) log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false) log.roll() // the first cleaning preserves the commit marker (at offset 3) since there were still records for the transaction dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1 - assertEquals(List(2, 1, 3, 4, 5, 6), keysInLog(log)) - assertEquals(List(3, 4, 5, 6, 7, 8, 9, 10, 11), offsetsInLog(log)) + assertEquals(List(2, 1, 3), keysInLog(log)) + assertEquals(List(3, 4, 5, 6, 7, 8), offsetsInLog(log)) // delete horizon forced to 0 to verify marker is not removed early dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = 0L)._1 - assertEquals(List(2, 1, 3, 4, 5, 6), keysInLog(log)) - assertEquals(List(3, 4, 5, 6, 7, 8, 9, 10, 11), offsetsInLog(log)) + assertEquals(List(2, 1, 3), keysInLog(log)) + assertEquals(List(3, 4, 5, 6, 7, 8), offsetsInLog(log)) // clean again with large delete horizon and verify the marker is removed dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1 - assertEquals(List(2, 1, 3, 4, 5, 6), keysInLog(log)) - assertEquals(List(4, 5, 6, 7, 8, 9, 10, 11), offsetsInLog(log)) + assertEquals(List(2, 1, 3), keysInLog(log)) + assertEquals(List(4, 5, 6, 7, 8), offsetsInLog(log)) } @Test @@ -335,22 +332,19 @@ class LogCleanerTest extends JUnitSuite { assertEquals(List(2, 3, 4), offsetsInLog(log)) // commit marker is still retained assertEquals(List(1, 2, 3, 4), lastOffsetsPerBatchInLog(log)) // empty batch is retained - // append a new records from the producer to allow cleaning of the empty batch - for (i <- Range(1, ProducerIdEntry.NumBatchesToRetain + 1)) { - appendProducer(Seq(i)) - } - + // append a new record from the producer to allow cleaning of the empty batch + appendProducer(Seq(1)) log.roll() dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1 - assertEquals(List(1, 2, 3, 4, 5), keysInLog(log)) - assertEquals(List(2, 5, 6, 7, 8, 9), offsetsInLog(log)) // commit marker is still retained - assertEquals(List(2, 5, 6, 7, 8, 9), lastOffsetsPerBatchInLog(log)) // empty batch should be gone + assertEquals(List(2, 3, 1), keysInLog(log)) + assertEquals(List(2, 3, 4, 5), offsetsInLog(log)) // commit marker is still retained + assertEquals(List(2, 3, 4, 5), lastOffsetsPerBatchInLog(log)) // empty batch should be gone dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1 - assertEquals(List(1, 2, 3, 4, 5), keysInLog(log)) - assertEquals(List(5, 6, 7, 8, 9), offsetsInLog(log)) // commit marker is gone - assertEquals(List(5, 6, 7, 8, 9), lastOffsetsPerBatchInLog(log)) // empty batch is gone + assertEquals(List(2, 3, 1), keysInLog(log)) + assertEquals(List(3, 4, 5), offsetsInLog(log)) // commit marker is gone + assertEquals(List(3, 4, 5), lastOffsetsPerBatchInLog(log)) // empty batch is gone } @Test @@ -368,21 +362,19 @@ class LogCleanerTest extends JUnitSuite { appendProducer(Seq(1)) appendProducer(Seq(2, 3)) log.appendAsLeader(abortMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false) - for (i <- Range(3, ProducerIdEntry.NumBatchesToRetain + 3)) { - appendProducer(Seq(i)) - } + appendProducer(Seq(3)) log.appendAsLeader(commitMarker(producerId, producerEpoch), leaderEpoch = 0, isFromClient = false) log.roll() // delete horizon set to 0 to verify marker is not removed early val dirtyOffset = cleaner.doClean(LogToClean(tp, log, 0L, 100L), deleteHorizonMs = 0L)._1 - assertEquals(List(3, 4, 5, 6, 7), keysInLog(log)) - assertEquals(List(3, 4, 5, 6, 7, 8, 9), offsetsInLog(log)) + assertEquals(List(3), keysInLog(log)) + assertEquals(List(3, 4, 5), offsetsInLog(log)) - // clean again with large delete horizon and verify the abort marker is removed + // clean again with large delete horizon and verify the marker is removed cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue) - assertEquals(List(3, 4, 5, 6, 7), keysInLog(log)) - assertEquals(List(4, 5, 6, 7, 8, 9), offsetsInLog(log)) + assertEquals(List(3), keysInLog(log)) + assertEquals(List(4, 5), offsetsInLog(log)) } @Test @@ -425,22 +417,20 @@ class LogCleanerTest extends JUnitSuite { assertEquals(List(2), offsetsInLog(log)) // abort marker is still retained assertEquals(List(1, 2), lastOffsetsPerBatchInLog(log)) // empty batch is retained - // now create enough batches so that the empty batch can be removed - for (i <- Range(0, ProducerIdEntry.NumBatchesToRetain)) { - appendProducer(Seq(1)) - } + // now update the last sequence so that the empty batch can be removed + appendProducer(Seq(1)) log.roll() dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1 assertAbortedTransactionIndexed() assertEquals(List(1), keysInLog(log)) - assertEquals(List(2, 7), offsetsInLog(log)) // abort marker is not yet gone because we read the empty batch - assertEquals(List(2, 3, 4, 5, 6, 7), lastOffsetsPerBatchInLog(log)) // but we do not preserve the original empty batch + assertEquals(List(2, 3), offsetsInLog(log)) // abort marker is not yet gone because we read the empty batch + assertEquals(List(2, 3), lastOffsetsPerBatchInLog(log)) // but we do not preserve the empty batch dirtyOffset = cleaner.doClean(LogToClean(tp, log, dirtyOffset, 100L), deleteHorizonMs = Long.MaxValue)._1 assertEquals(List(1), keysInLog(log)) - assertEquals(List(7), offsetsInLog(log)) // abort marker is gone - assertEquals(List(3, 4, 5, 6, 7), lastOffsetsPerBatchInLog(log)) + assertEquals(List(3), offsetsInLog(log)) // abort marker is gone + assertEquals(List(3), lastOffsetsPerBatchInLog(log)) // we do not bother retaining the aborted transaction in the index assertEquals(0, log.collectAbortedTransactions(0L, 100L).size) @@ -549,7 +539,7 @@ class LogCleanerTest extends JUnitSuite { log.roll() cleaner.clean(LogToClean(new TopicPartition("test", 0), log, 0L, log.activeSegment.baseOffset)) - assertEquals(List(1, 2, 3, 4), lastOffsetsPerBatchInLog(log)) + assertEquals(List(1, 3, 4), lastOffsetsPerBatchInLog(log)) assertEquals(Map(1L -> 0, 2L -> 1, 3L -> 0), lastSequencesInLog(log)) assertEquals(List(0, 1), keysInLog(log)) assertEquals(List(3, 4), offsetsInLog(log)) @@ -572,24 +562,20 @@ class LogCleanerTest extends JUnitSuite { log.roll() cleaner.clean(LogToClean(new TopicPartition("test", 0), log, 0L, log.activeSegment.baseOffset)) - assertEquals(List(0, 2, 3), lastOffsetsPerBatchInLog(log)) + assertEquals(List(2, 3), lastOffsetsPerBatchInLog(log)) assertEquals(Map(producerId -> 2), lastSequencesInLog(log)) assertEquals(List(), keysInLog(log)) assertEquals(List(3), offsetsInLog(log)) - // Append new entries from the producer and verify that the empty batches are cleaned up - for (i <- Range(0, ProducerIdEntry.NumBatchesToRetain)) { - appendProducer(Seq(1, 5)) - } - + // Append a new entry from the producer and verify that the empty batch is cleaned up + appendProducer(Seq(1, 5)) log.roll() cleaner.clean(LogToClean(new TopicPartition("test", 0), log, 0L, log.activeSegment.baseOffset)) - // We now retain the last 5 batches. - assertEquals(List(3, 5, 7, 9, 11, 13), lastOffsetsPerBatchInLog(log)) - assertEquals(Map(producerId -> 12), lastSequencesInLog(log)) + assertEquals(List(3, 5), lastOffsetsPerBatchInLog(log)) + assertEquals(Map(producerId -> 4), lastSequencesInLog(log)) assertEquals(List(1, 5), keysInLog(log)) - assertEquals(List(3, 12, 13), offsetsInLog(log)) + assertEquals(List(3, 4, 5), offsetsInLog(log)) } @Test From 7dc9eff4f25143381aae3ba2c9a54b4a01a90b37 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 7 Sep 2017 22:36:42 -0700 Subject: [PATCH 16/40] Addressed a few more PR comments --- .../main/scala/kafka/log/ProducerStateManager.scala | 11 +---------- core/src/main/scala/kafka/tools/DumpLogSegments.scala | 4 ++-- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index 72b0637a2fb14..8871574c1db83 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -115,16 +115,7 @@ private[log] case class ProducerIdEntry(producerId: Long, batchMetadata: mutable val duplicate = batchMetadata.filter { case(metadata) => firstSeq == metadata.firstSeq && lastSeq == metadata.lastSeq } - - if (duplicate.size > 1) - throw new IllegalStateException(s"Found ${duplicate.size} duplicate batches in the cached producer metadata " + - s"for producerId: $producerId and producerEpoch: $producerEpoch. This means that it's likely there are " + - s"duplicates in the log as well.") - - if (duplicate.size == 1) - Some(duplicate.head) - else - None + duplicate.headOption } override def toString: String = { diff --git a/core/src/main/scala/kafka/tools/DumpLogSegments.scala b/core/src/main/scala/kafka/tools/DumpLogSegments.scala index 86d19c7c9aae8..c4f7ce0218b08 100755 --- a/core/src/main/scala/kafka/tools/DumpLogSegments.scala +++ b/core/src/main/scala/kafka/tools/DumpLogSegments.scala @@ -115,7 +115,7 @@ object DumpLogSegments { case Log.TimeIndexFileSuffix => dumpTimeIndex(file, indexSanityOnly, verifyOnly, timeIndexDumpErrors, maxMessageSize) case Log.PidSnapshotFileSuffix => - dumpPidSnapshot(file) + dumpProducerIdSnapshot(file) case Log.TxnIndexFileSuffix => dumpTxnIndex(file) case _ => @@ -152,7 +152,7 @@ object DumpLogSegments { } } - private def dumpPidSnapshot(file: File): Unit = { + private def dumpProducerIdSnapshot(file: File): Unit = { try { ProducerStateManager.readSnapshot(file).foreach { entry => println(s"producerId: ${entry.producerId} producerEpoch: ${entry.producerEpoch} " + From 0ad49aa582af31e93126c6bdbeb44aaf3ac6db0a Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Fri, 8 Sep 2017 17:28:24 -0700 Subject: [PATCH 17/40] Address review comments for the broker side chanegs. Added tests for the new duplicate detection logic. --- .../producer/internals/ProducerBatch.java | 3 ++- .../producer/internals/RecordAccumulator.java | 3 +++ .../clients/producer/internals/Sender.java | 5 +--- .../internals/TransactionManager.java | 8 ++++--- ...n.java => DuplicateSequenceException.java} | 4 ++-- .../apache/kafka/common/protocol/Errors.java | 4 ++-- .../kafka/log/ProducerStateManager.scala | 23 ++++++++----------- .../scala/unit/kafka/log/LogSegmentTest.scala | 2 +- .../test/scala/unit/kafka/log/LogTest.scala | 18 ++++++++++----- .../kafka/log/ProducerStateManagerTest.scala | 2 +- 10 files changed, 39 insertions(+), 33 deletions(-) rename clients/src/main/java/org/apache/kafka/common/errors/{DuplicateSequenceNumberException.java => DuplicateSequenceException.java} (86%) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java index a3e120a996b6d..f803c4a9b6859 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java @@ -388,8 +388,9 @@ public void setProducerState(ProducerIdAndEpoch producerIdAndEpoch, int baseSequ recordsBuilder.setProducerState(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, baseSequence, isTransactional); } - public void reopenBatchAndResetProducerState() { + public void resetProducerState(ProducerIdAndEpoch producerIdAndEpoch, int baseSequence, boolean isTransactional) { recordsBuilder.unsetProducerState(); + recordsBuilder.setProducerState(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, baseSequence, isTransactional); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index e866093e6214e..58695b42e3a9f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -343,6 +343,9 @@ public int splitAndReenqueue(ProducerBatch bigBatch) { return numSplitBatches; } + // The deque for the partition may have to be reordered in situations where leadership changes in between + // batch drains. Since the requests are on different connections, we no longer have any guarantees about ordering + // of the responses. Hence we will have to check if there is anything out of order and correct the order in the queue. private void maybeEnsureQueueIsOrdered(Deque deque, ProducerBatch batch) { if (transactionManager != null) { // When we are requeing and have enabled idempotence, the reenqueued batch must always have a sequence. diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 7f18c7047b74a..5b37200c305de 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -230,7 +230,7 @@ private long sendProducerData(long now) { Cluster cluster = metadata.fetch(); - if (transactionManager != null && transactionManager.maybeResolveSequences()) { + if (transactionManager != null && transactionManager.shouldResetProducerStateAfterResolvingSequences()) { transactionManager.resetProducerId(); return 0; } @@ -556,9 +556,6 @@ else if (error == Errors.CLUSTER_AUTHORIZATION_FAILED) } private void reenqueueBatch(ProducerBatch batch, long currentTimeMs) { - if (transactionManager != null) - // Reset the sequence number for the retried batch. It will be set again on the next drain. - batch.reopenBatchAndResetProducerState(); this.accumulator.reenqueue(batch, currentTimeMs); this.sensors.recordRetries(batch.topicPartition.topic(), batch.recordCount); } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index e9489c0b94428..3465a93fb67a8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -459,6 +459,7 @@ synchronized void adjustSequencesDueToFailedBatch(ProducerBatch batch) { // Sequence numbers are not being tracked for this partition. This could happen if the producer id was just // reset due to a previous OutOfOrderSequenceException. return; + int currentSequence = sequenceNumber(batch.topicPartition); currentSequence -= batch.recordCount; if (currentSequence < 0) @@ -467,13 +468,15 @@ synchronized void adjustSequencesDueToFailedBatch(ProducerBatch batch) { setNextSequence(batch.topicPartition, currentSequence); for (ProducerBatch inFlightBatch : inflightBatchesBySequence.get(batch.topicPartition)) { + if (inFlightBatch.baseSequence() < batch.baseSequence()) + continue; int newSequence = inFlightBatch.baseSequence() - batch.recordCount; if (newSequence < 0) throw new IllegalStateException("Sequence number for batch with sequence " + inFlightBatch.baseSequence() + " for partition " + batch.topicPartition + " is going to become negative :" + newSequence); log.debug("Setting sequence number of batch with current sequence {} for partition {} to {}", batch.baseSequence(), batch.topicPartition, newSequence); - inFlightBatch.setProducerState(new ProducerIdAndEpoch(inFlightBatch.producerId(), inFlightBatch.producerEpoch()), newSequence, inFlightBatch.isTransactional()); + inFlightBatch.resetProducerState(new ProducerIdAndEpoch(inFlightBatch.producerId(), inFlightBatch.producerEpoch()), newSequence, inFlightBatch.isTransactional()); } } @@ -491,8 +494,7 @@ synchronized void markSequenceUnresolved(TopicPartition topicPartition) { // Checks if there are any partitions with unresolved partitions which may now be resolved. Returns true if // the producer id needs a reset, false otherwise. - - synchronized boolean maybeResolveSequences() { + synchronized boolean shouldResetProducerStateAfterResolvingSequences() { for (TopicPartition topicPartition : partitionsWithUnresolvedSequences) { if (!hasInflightBatches(topicPartition)) { // The partition has been fully drained. At this point, the last ack'd sequence should be once less than diff --git a/clients/src/main/java/org/apache/kafka/common/errors/DuplicateSequenceNumberException.java b/clients/src/main/java/org/apache/kafka/common/errors/DuplicateSequenceException.java similarity index 86% rename from clients/src/main/java/org/apache/kafka/common/errors/DuplicateSequenceNumberException.java rename to clients/src/main/java/org/apache/kafka/common/errors/DuplicateSequenceException.java index 469ba98f445e9..91da067d29d9f 100644 --- a/clients/src/main/java/org/apache/kafka/common/errors/DuplicateSequenceNumberException.java +++ b/clients/src/main/java/org/apache/kafka/common/errors/DuplicateSequenceException.java @@ -16,9 +16,9 @@ */ package org.apache.kafka.common.errors; -public class DuplicateSequenceNumberException extends RetriableException { +public class DuplicateSequenceException extends RetriableException { - public DuplicateSequenceNumberException(String message) { + public DuplicateSequenceException(String message) { super(message); } } diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java index 9decef277626b..bbc348643e0cc 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java @@ -25,7 +25,7 @@ import org.apache.kafka.common.errors.CoordinatorNotAvailableException; import org.apache.kafka.common.errors.CorruptRecordException; import org.apache.kafka.common.errors.LogDirNotFoundException; -import org.apache.kafka.common.errors.DuplicateSequenceNumberException; +import org.apache.kafka.common.errors.DuplicateSequenceException; import org.apache.kafka.common.errors.GroupAuthorizationException; import org.apache.kafka.common.errors.IllegalGenerationException; import org.apache.kafka.common.errors.IllegalSaslStateException; @@ -432,7 +432,7 @@ public ApiException build(String message) { new ApiExceptionBuilder() { @Override public ApiException build(String message) { - return new DuplicateSequenceNumberException(message); + return new DuplicateSequenceException(message); } }), INVALID_PRODUCER_EPOCH(47, "Producer attempted an operation with an old epoch. Either there is a newer producer " + diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index 8871574c1db83..12eea8edce756 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -66,9 +66,12 @@ private[log] case class BatchMetadata(lastSeq: Int, lastOffset: Long, offsetDelt } } -private[log] case class ProducerIdEntry(producerId: Long, batchMetadata: mutable.Queue[BatchMetadata], - var producerEpoch: Short, var coordinatorEpoch: Int, - var currentTxnFirstOffset: Option[Long]) { +// the batchMetadata is ordered such that the batch with the lowest sequence is at the head of the queue while the +// batch with the highest sequence is at the tail of the queue. We will retain at most ProducerIdEntry.NumBatchesToRetain +// elements in the queue. When the queue is at capacity, we remove the first element to make space for the incoming batch. +private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: mutable.Queue[BatchMetadata], + var producerEpoch: Short, var coordinatorEpoch: Int, + var currentTxnFirstOffset: Option[Long]) { def firstSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.front.firstSeq def firstOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.front.firstOffset @@ -95,13 +98,7 @@ private[log] case class ProducerIdEntry(producerId: Long, batchMetadata: mutable false } - def removeBatchesOlderThan(offset: Long) = { - val retainedMetadata = batchMetadata.filter(_.lastOffset >= offset) - if (retainedMetadata.size != batchMetadata.size) { - batchMetadata.clear() - retainedMetadata.foreach(batchMetadata.enqueue(_)) - } - } + def removeBatchesOlderThan(offset: Long) = batchMetadata.dropWhile(_.lastOffset < offset) def duplicateOf(batch: RecordBatch): Option[BatchMetadata] = { if (batch.producerEpoch() != producerEpoch) @@ -169,8 +166,8 @@ private[log] class ProducerAppendInfo(val producerId: Long, // the epoch was bumped by a control record, so we expect the sequence number to be reset throw new OutOfOrderSequenceException(s"Out of order sequence number for producerId $producerId: found $firstSeq " + s"(incoming seq. number), but expected 0") - } else if (currentEntry.batchWithSequenceRange(firstSeq, lastSeq).isDefined) { - throw new DuplicateSequenceNumberException(s"Duplicate sequence number for producerId $producerId: (incomingBatch.firstSeq, " + + } else if (lastSeq < currentEntry.firstSeq || currentEntry.batchWithSequenceRange(firstSeq, lastSeq).isDefined) { + throw new DuplicateSequenceException(s"Duplicate sequence number for producerId $producerId: (incomingBatch.firstSeq, " + s"incomingBatch.lastSeq): ($firstSeq, $lastSeq).") } else if (!inSequence(firstSeq, lastSeq)) { throw new OutOfOrderSequenceException(s"Out of order sequence number for producerId $producerId: $firstSeq " + @@ -341,7 +338,7 @@ object ProducerStateManager { val offsetDelta = producerEntryStruct.getInt(OffsetDeltaField) val coordinatorEpoch = producerEntryStruct.getInt(CoordinatorEpochField) val currentTxnFirstOffset = producerEntryStruct.getLong(CurrentTxnFirstOffsetField) - val newEntry = ProducerIdEntry(producerId, mutable.Queue[BatchMetadata](BatchMetadata(seq, offset, offsetDelta, timestamp)), producerEpoch, + val newEntry = new ProducerIdEntry(producerId, mutable.Queue[BatchMetadata](BatchMetadata(seq, offset, offsetDelta, timestamp)), producerEpoch, coordinatorEpoch, if (currentTxnFirstOffset >= 0) Some(currentTxnFirstOffset) else None) newEntry } diff --git a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala index a0fa48110b672..aa09acae1aa19 100644 --- a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala @@ -317,7 +317,7 @@ class LogSegmentTest { // recover again, but this time assuming the transaction from pid2 began on a previous segment stateManager = new ProducerStateManager(topicPartition, logDir) - stateManager.loadProducerEntry(ProducerIdEntry(pid2, mutable.Queue[BatchMetadata](BatchMetadata(10, 90L, 5, RecordBatch.NO_TIMESTAMP)), + stateManager.loadProducerEntry(new ProducerIdEntry(pid2, mutable.Queue[BatchMetadata](BatchMetadata(10, 90L, 5, RecordBatch.NO_TIMESTAMP)), producerEpoch, 0, Some(75L))) segment.recover(stateManager) assertEquals(108L, stateManager.mapEndOffset) diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala index ad64e39a0a57c..da4706e2b6b90 100755 --- a/core/src/test/scala/unit/kafka/log/LogTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogTest.scala @@ -805,16 +805,22 @@ class LogTest { case _: OutOfOrderSequenceException => // Good! } - // Append a Duplicate of an entry in the middle of the log. This is not allowed. + // Append a duplicate of the batch which is 4th from the tail. This should succeed without error since we + // retain the batch metadata of the last 5 batches. + val duplicateOfFourth = TestUtils.records(List(new SimpleRecord(mockTime.milliseconds, "key".getBytes, "value".getBytes)), + producerId = pid, producerEpoch = epoch, sequence = 2) + log.appendAsLeader(duplicateOfFourth, leaderEpoch = 0) + + // Append a Duplicate of an entry older than the last 5 appended batches. This should result in a DuplicateSequenceNumberException. try { val records = TestUtils.records( List(new SimpleRecord(mockTime.milliseconds, s"key-1".getBytes, s"value-1".getBytes)), producerId = pid, producerEpoch = epoch, sequence = 1) log.appendAsLeader(records, leaderEpoch = 0) - fail ("Should have received an OutOfOrderSequenceException since we attempted to append a duplicate of a records " + - "in the middle of the log.") + fail ("Should have received an DuplicateSequenceNumberException since we attempted to append a duplicate of a batch" + + "which is older than the last 5 appended batches.") } catch { - case _: OutOfOrderSequenceException => // Good! + case _: DuplicateSequenceException => // Good! } // Append a duplicate entry with a single records at the tail of the log. This should return the appendInfo of the original entry. @@ -872,7 +878,7 @@ class LogTest { } } - @Test(expected = classOf[DuplicateSequenceNumberException]) + @Test(expected = classOf[DuplicateSequenceException]) def testDuplicateAppendToFollower() : Unit = { val logConfig = createLogConfig(segmentBytes = 1024 * 1024 * 5) val log = createLog(logDir, logConfig) @@ -888,7 +894,7 @@ class LogTest { partitionLeaderEpoch, new SimpleRecord("a".getBytes), new SimpleRecord("b".getBytes))) } - @Test(expected = classOf[DuplicateSequenceNumberException]) + @Test(expected = classOf[DuplicateSequenceException]) def testMultipleProducersWithDuplicatesInSingleAppend() : Unit = { val logConfig = createLogConfig(segmentBytes = 1024 * 1024 * 5) val log = createLog(logDir, logConfig) diff --git a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala index f2dee315fbab5..99649c9e3c8af 100644 --- a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala +++ b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala @@ -63,7 +63,7 @@ class ProducerStateManagerTest extends JUnitSuite { append(stateManager, producerId, epoch, 1, 0L, 1L) // Duplicate sequence number (matches previous sequence number) - assertThrows[DuplicateSequenceNumberException] { + assertThrows[DuplicateSequenceException] { append(stateManager, producerId, epoch, 1, 0L, 1L) } From 9136c790d57ca75fe87a99fe3d61cd770ae8d69e Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Sun, 10 Sep 2017 21:56:18 -0700 Subject: [PATCH 18/40] Handle duplicates during wraparound --- core/src/main/scala/kafka/log/ProducerStateManager.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index 12eea8edce756..7da501452d142 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -108,7 +108,7 @@ private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: muta } // Return the batch metadata of the cached batch having the exact sequence range, if any. - def batchWithSequenceRange(firstSeq: Int, lastSeq:Int) : Option[BatchMetadata] = { + def batchWithSequenceRange(firstSeq: Int, lastSeq: Int) : Option[BatchMetadata] = { val duplicate = batchMetadata.filter { case(metadata) => firstSeq == metadata.firstSeq && lastSeq == metadata.lastSeq } @@ -166,7 +166,7 @@ private[log] class ProducerAppendInfo(val producerId: Long, // the epoch was bumped by a control record, so we expect the sequence number to be reset throw new OutOfOrderSequenceException(s"Out of order sequence number for producerId $producerId: found $firstSeq " + s"(incoming seq. number), but expected 0") - } else if (lastSeq < currentEntry.firstSeq || currentEntry.batchWithSequenceRange(firstSeq, lastSeq).isDefined) { + } else if (isDuplicate(firstSeq, lastSeq)) { throw new DuplicateSequenceException(s"Duplicate sequence number for producerId $producerId: (incomingBatch.firstSeq, " + s"incomingBatch.lastSeq): ($firstSeq, $lastSeq).") } else if (!inSequence(firstSeq, lastSeq)) { @@ -176,6 +176,11 @@ private[log] class ProducerAppendInfo(val producerId: Long, } } + private def isDuplicate(firstSeq: Int, lastSeq: Int): Boolean = { + ((lastSeq != 0 && currentEntry.firstSeq != Int.MaxValue && lastSeq < currentEntry.firstSeq) + || currentEntry.batchWithSequenceRange(firstSeq, lastSeq).isDefined) + } + private def inSequence(firstSeq: Int, lastSeq: Int): Boolean = { firstSeq == currentEntry.lastSeq + 1L || (firstSeq == 0 && currentEntry.lastSeq == Int.MaxValue) } From 5efcf2d1257f0fe2777287d1c67d0398b6f3d364 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Sun, 10 Sep 2017 22:23:42 -0700 Subject: [PATCH 19/40] Two changes: 1. A solution for detecting duplicates when sequence numbers wraparound. This is essentially the same as the solutior for detecing out of order sequences with wraparound, but both are limited. 2. Correctly handle duplicate sequence exceptions on the client. --- .../clients/producer/internals/Sender.java | 10 +++- .../internals/TransactionManager.java | 2 +- .../errors/DuplicateSequenceException.java | 2 +- .../producer/internals/SenderTest.java | 47 +++++++++++++++++++ 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 5b37200c305de..dc1a61f24e543 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -528,6 +528,13 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons "batch but the producer id changed from " + batch.producerId() + " to " + transactionManager.producerIdAndEpoch().producerId + " in the mean time. This batch will be dropped."), false); } + } else if (error == Errors.DUPLICATE_SEQUENCE_NUMBER) { + // If we have received a duplicate sequence error, it means that the sequence number has advanced beyond + // the sequence of the current batch, and we haven't retained batch metadata on the broker to return + // the correct offset and timestamp. + // + // The only thing we can do is to return success to the user and not return a valid offset and timestamp. + completeBatch(batch, response); } else { final RuntimeException exception; if (error == Errors.TOPIC_AUTHORIZATION_FAILED) @@ -563,7 +570,7 @@ private void reenqueueBatch(ProducerBatch batch, long currentTimeMs) { private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response) { if (transactionManager != null) { if (transactionManager.hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch())) { - transactionManager.setLastAckedSequence(batch.topicPartition, batch.baseSequence() + batch.recordCount - 1); + transactionManager.maybeUpdateLastAckedSequence(batch.topicPartition, batch.baseSequence() + batch.recordCount - 1); log.debug("ProducerId: {}; Set last ack'd sequence number for topic-partition {} to {}", batch.producerId(), batch.topicPartition, transactionManager.lastAckedSequence(batch.topicPartition)); } @@ -571,7 +578,6 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons } batch.done(response.baseOffset, response.logAppendTime, null); - this.sensors.recordErrors(batch.topicPartition.topic(), batch.recordCount); this.accumulator.deallocate(batch); } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index 3465a93fb67a8..d7a0480dfa97a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -437,7 +437,7 @@ synchronized void stopTrackingBatch(ProducerBatch batch) { inflightBatchesBySequence.get(batch.topicPartition).remove(batch); } - synchronized void setLastAckedSequence(TopicPartition topicPartition, int sequence) { + synchronized void maybeUpdateLastAckedSequence(TopicPartition topicPartition, int sequence) { if (sequence > lastAckedSequence(topicPartition)) lastAckedSequence.put(topicPartition, sequence); } diff --git a/clients/src/main/java/org/apache/kafka/common/errors/DuplicateSequenceException.java b/clients/src/main/java/org/apache/kafka/common/errors/DuplicateSequenceException.java index 91da067d29d9f..11f81af5bb868 100644 --- a/clients/src/main/java/org/apache/kafka/common/errors/DuplicateSequenceException.java +++ b/clients/src/main/java/org/apache/kafka/common/errors/DuplicateSequenceException.java @@ -16,7 +16,7 @@ */ package org.apache.kafka.common.errors; -public class DuplicateSequenceException extends RetriableException { +public class DuplicateSequenceException extends ApiException { public DuplicateSequenceException(String message) { super(message); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 0d4067d34a6be..12e96bcd109bf 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -959,6 +959,53 @@ public void testExpiryOfAllSentBatchesShouldCauseUnresolvedSequences() throws Ex assertFalse(transactionManager.hasProducerId()); } + @Test + public void testCorrectHandlingOfDuplicateSequenceError() throws Exception { + final long producerId = 343434L; + TransactionManager transactionManager = new TransactionManager(); + setupWithTransactionState(transactionManager); + prepareAndReceiveInitProducerId(producerId, Errors.NONE); + assertTrue(transactionManager.hasProducerId()); + + assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); + + // Send first ProduceRequest + Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + String nodeId = client.requests().peek().destination(); + Node node = new Node(Integer.valueOf(nodeId), "localhost", 0); + assertEquals(1, client.inFlightRequestCount()); + assertEquals(1, transactionManager.sequenceNumber(tp0).longValue()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + + // Send second ProduceRequest + Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + assertEquals(2, client.inFlightRequestCount()); + assertEquals(2, transactionManager.sequenceNumber(tp0).longValue()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + assertFalse(request1.isDone()); + assertFalse(request2.isDone()); + assertTrue(client.isReady(node, time.milliseconds())); + + ClientRequest firstClientRequest = client.requests().peek(); + ClientRequest secondClientRequest = (ClientRequest) client.requests().toArray()[1]; + + client.respondToRequest(secondClientRequest, produceResponse(tp0, 1, Errors.NONE, -1)); + + sender.run(time.milliseconds()); // receive response 1 + + assertEquals(1, transactionManager.lastAckedSequence(tp0)); + + client.respondToRequest(firstClientRequest, produceResponse(tp0, -1, Errors.DUPLICATE_SEQUENCE_NUMBER, -1)); + + sender.run(time.milliseconds()); // receive response 0 + + // Make sure that the last ack'd sequence doesn't change. + assertEquals(1, transactionManager.lastAckedSequence(tp0)); + assertFalse(client.hasInFlightRequests()); + } + void sendIdempotentProducerResponse(final int expectedSequence, TopicPartition tp, Errors responseError, long responseOffset) { client.respond(new MockClient.RequestMatcher() { @Override From eb7905919c2317a37a7802d377c0ebedb4891730 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Sun, 10 Sep 2017 22:54:53 -0700 Subject: [PATCH 20/40] Addressed more PR comments --- .../producer/internals/RecordAccumulator.java | 12 ++++++++++-- .../producer/internals/TransactionManager.java | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index 58695b42e3a9f..b1b33c97ac24a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -346,6 +346,9 @@ public int splitAndReenqueue(ProducerBatch bigBatch) { // The deque for the partition may have to be reordered in situations where leadership changes in between // batch drains. Since the requests are on different connections, we no longer have any guarantees about ordering // of the responses. Hence we will have to check if there is anything out of order and correct the order in the queue. + // + // Note that this assumes that all the batches in the queue which have an assigned sequence also have the current + // producer id. We will not attempt to reorder messages if the producer id has changed. private void maybeEnsureQueueIsOrdered(Deque deque, ProducerBatch batch) { if (transactionManager != null) { // When we are requeing and have enabled idempotence, the reenqueued batch must always have a sequence. @@ -353,7 +356,12 @@ private void maybeEnsureQueueIsOrdered(Deque deque, ProducerBatch throw new IllegalStateException("Trying to reenqueue a batch which doesn't have a sequence even " + "though idempotence is enabled."); - if (batch.baseSequence() != transactionManager.nextBatchBySequence(batch.topicPartition).baseSequence()) { + // If there are no inflight batches being tracked by the transaction manager, it means that the producer + // id must have changed and the batches being re enqueued are from the old producer id. In this case + // we don't try to ensure ordering amongst them. They will eventually fail with an OutOfOrderSequence, + // or they will succeed. + if (transactionManager.nextBatchBySequence(batch.topicPartition) != null && + batch.baseSequence() != transactionManager.nextBatchBySequence(batch.topicPartition).baseSequence()) { // The head of the queues have diverged. This means that the incoming batch should be placed somewhere further back. // We need to find the right place for the incoming batch and insert it there. // We will only enter this branch if we have multiple inflights sent to different brokers, perhaps @@ -367,7 +375,7 @@ private void maybeEnsureQueueIsOrdered(Deque deque, ProducerBatch orderedBatches.add(deque.pollFirst()); log.debug("Reordered incoming batch with sequence {} for partition {}. It was placed in the queue at " + - "position {}" + incomingBatch.baseSequence(), incomingBatch.topicPartition, orderedBatches.size()); + "position {}", incomingBatch.baseSequence(), incomingBatch.topicPartition, orderedBatches.size()); // Either we have reached a point where there are batches without a sequence (ie. never been drained // and are hence in order by default), or the batch at the front of the queue has a sequence greater // than the incoming batch. This is the right place to add the incoming batch. diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index d7a0480dfa97a..25699ccb7458e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -519,7 +519,7 @@ synchronized boolean isNextSequence(TopicPartition topicPartition, int sequence) (lastAckedSequence.containsKey(topicPartition) && (sequence - lastAckedSequence.get(topicPartition) == 1)); } - synchronized void setNextSequence(TopicPartition topicPartition, int sequence) { + private synchronized void setNextSequence(TopicPartition topicPartition, int sequence) { if (!nextSequence.containsKey(topicPartition) && sequence != 0) throw new IllegalStateException("Trying to set the sequence number for " + topicPartition + " to " + sequence + ", but the sequence number was never set for this partition."); From d3fa70411ab940001a6ba653ef1cf86934f4da82 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Mon, 11 Sep 2017 13:37:30 -0700 Subject: [PATCH 21/40] Address a few more minor PR comments --- .../producer/internals/RecordAccumulator.java | 4 ++-- .../clients/producer/internals/Sender.java | 18 +++++++++--------- .../producer/internals/TransactionManager.java | 10 ++++++++-- .../common/record/MemoryRecordsBuilder.java | 3 +++ 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index b1b33c97ac24a..808cae7860a9f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -335,7 +335,7 @@ public int splitAndReenqueue(ProducerBatch bigBatch) { partitionDequeue.addFirst(batch); if (transactionManager != null) { // We should track the newly created batches since they already have assigned sequences. - transactionManager.startTrackingBatch(batch); + transactionManager.addInFlightBatch(batch); maybeEnsureQueueIsOrdered(partitionDequeue, batch); } } @@ -557,7 +557,7 @@ public Map> drain(Cluster cluster, "{} being sent to partition {}", producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, batch.baseSequence(), tp); - transactionManager.startTrackingBatch(batch); + transactionManager.addInFlightBatch(batch); } batch.close(); size += batch.records().sizeInBytes(); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index dc1a61f24e543..ff2537d7e48e1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -504,7 +504,7 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons this.retries - batch.attempts(), error); if (transactionManager != null) - transactionManager.stopTrackingBatch(batch); + transactionManager.removeInFlightBatch(batch); this.accumulator.splitAndReenqueue(batch); this.accumulator.deallocate(batch); this.sensors.recordBatchSplit(); @@ -543,8 +543,10 @@ else if (error == Errors.CLUSTER_AUTHORIZATION_FAILED) exception = new ClusterAuthorizationException("The producer is not authorized to do idempotent sends"); else exception = error.exception(); - // tell the user the result of their request - failBatch(batch, response, exception, true); + // tell the user the result of their request. We only adjust sequence numbers if the batch didn't exhaust + // its retries -- if it did, we don't know the whether the sequence number was accepted or not, and + // thus it is not safe to reassign the sequence. + failBatch(batch, response, exception, batch.attempts() < this.retries); } if (error.exception() instanceof InvalidMetadataException) { if (error.exception() instanceof UnknownTopicOrPartitionException) @@ -574,7 +576,7 @@ private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionRespons log.debug("ProducerId: {}; Set last ack'd sequence number for topic-partition {} to {}", batch.producerId(), batch.topicPartition, transactionManager.lastAckedSequence(batch.topicPartition)); } - transactionManager.stopTrackingBatch(batch); + transactionManager.removeInFlightBatch(batch); } batch.done(response.baseOffset, response.logAppendTime, null); @@ -606,7 +608,7 @@ private void failBatch(ProducerBatch batch, long baseOffset, long logAppendTime, } else if (transactionManager.isTransactional()) { transactionManager.transitionToAbortableError(exception); } - transactionManager.stopTrackingBatch(batch); + transactionManager.removeInFlightBatch(batch); if (adjustSequenceNumbers) transactionManager.adjustSequencesDueToFailedBatch(batch); } @@ -624,10 +626,8 @@ private void failBatch(ProducerBatch batch, long baseOffset, long logAppendTime, private boolean canRetry(ProducerBatch batch, Errors error) { return batch.attempts() < this.retries && ((error.exception() instanceof RetriableException) || - (error.exception() instanceof OutOfOrderSequenceException && - transactionManager.hasProducerId(batch.producerId()) && - !transactionManager.partitionHasUnresolvedSequence(batch.topicPartition) && - !transactionManager.isNextSequence(batch.topicPartition, batch.baseSequence()))); + (error.exception() instanceof OutOfOrderSequenceException + && transactionManager.canRetryOutOfOrderSequenceException(batch))); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index 25699ccb7458e..c6299098166ea 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -413,7 +413,7 @@ synchronized void incrementSequenceNumber(TopicPartition topicPartition, int inc nextSequence.put(topicPartition, currentSequenceNumber); } - synchronized void startTrackingBatch(ProducerBatch batch) { + synchronized void addInFlightBatch(ProducerBatch batch) { if (!batch.hasSequence()) throw new IllegalStateException("Can't track batch for partition " + batch.topicPartition + " when sequence is not set."); if (!inflightBatchesBySequence.containsKey(batch.topicPartition)) { @@ -432,7 +432,7 @@ synchronized ProducerBatch nextBatchBySequence(TopicPartition topicPartition) { return inflightBatchesBySequence.get(topicPartition).peek(); } - synchronized void stopTrackingBatch(ProducerBatch batch) { + synchronized void removeInFlightBatch(ProducerBatch batch) { if (inflightBatchesBySequence.containsKey(batch.topicPartition)) inflightBatchesBySequence.get(batch.topicPartition).remove(batch); } @@ -619,6 +619,12 @@ synchronized boolean hasOngoingTransaction() { return currentState == State.IN_TRANSACTION || isCompleting() || hasAbortableError(); } + synchronized boolean canRetryOutOfOrderSequenceException(ProducerBatch batch) { + return hasProducerId(batch.producerId()) + && !partitionHasUnresolvedSequence(batch.topicPartition) + && !isNextSequence(batch.topicPartition, batch.baseSequence()); + } + // visible for testing synchronized boolean isReady() { return isTransactional() && currentState == State.READY; diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java index 72d3e1ac1caca..64defa65f33cf 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java @@ -280,6 +280,9 @@ public void abort() { } public void unsetProducerState() { + if (aborted) + throw new IllegalStateException("Should not reopen a batch which is already aborted."); + builtRecords = null; } From f4bf62cde6038de7ee9b7b36fdeb7028d306f852 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Mon, 11 Sep 2017 14:11:11 -0700 Subject: [PATCH 22/40] Revert changes to log4j.properties --- clients/src/test/resources/log4j.properties | 2 -- core/src/test/resources/log4j.properties | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/clients/src/test/resources/log4j.properties b/clients/src/test/resources/log4j.properties index 30f06eacf74f5..b1d5b7f2b4091 100644 --- a/clients/src/test/resources/log4j.properties +++ b/clients/src/test/resources/log4j.properties @@ -19,5 +19,3 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n log4j.logger.org.apache.kafka=ERROR - -log4j.logger.org.apache.kafka.clients.producer.internals=TRACE \ No newline at end of file diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties index 9ec020eeb0194..51ba7f85d1c94 100644 --- a/core/src/test/resources/log4j.properties +++ b/core/src/test/resources/log4j.properties @@ -19,6 +19,7 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n log4j.logger.kafka=ERROR +log4j.logger.org.apache.kafka=ERROR # zkclient can be verbose, during debugging it is common to adjust it separately log4j.logger.org.I0Itec.zkclient.ZkClient=WARN From c52b1f86920fa9a52b4de4c1b9c3a979e9bf9be9 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Mon, 11 Sep 2017 14:11:57 -0700 Subject: [PATCH 23/40] Delete unnecessary trailing whitespace --- core/src/test/resources/log4j.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties index 51ba7f85d1c94..d394a2abc9495 100644 --- a/core/src/test/resources/log4j.properties +++ b/core/src/test/resources/log4j.properties @@ -19,7 +19,7 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n log4j.logger.kafka=ERROR -log4j.logger.org.apache.kafka=ERROR +log4j.logger.org.apache.kafka=ERROR # zkclient can be verbose, during debugging it is common to adjust it separately log4j.logger.org.I0Itec.zkclient.ZkClient=WARN From 44236b88e87bc32b34ee6da96cc681f331375b9b Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 12 Sep 2017 18:53:17 -0700 Subject: [PATCH 24/40] Let the producerIdEntry have a concurrent structure for batch metadata. This makes accesses from the cleaner threads (log cleaner, producer id expiration, etc.) safe --- .../kafka/log/ProducerStateManager.scala | 50 ++++++++++++------- .../scala/unit/kafka/log/LogSegmentTest.scala | 6 ++- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index 7da501452d142..b6496853dbdf7 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -31,6 +31,8 @@ import org.apache.kafka.common.protocol.types._ import org.apache.kafka.common.record.{ControlRecordType, EndTransactionMarker, RecordBatch} import org.apache.kafka.common.utils.{ByteUtils, Crc32C} +import java.util.concurrent.ConcurrentLinkedDeque +import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer import scala.collection.{immutable, mutable} @@ -49,7 +51,7 @@ private[log] case class TxnMetadata(producerId: Long, var firstOffset: LogOffset private[log] object ProducerIdEntry { private[log] val NumBatchesToRetain = 5 - def empty(producerId: Long) = new ProducerIdEntry(producerId, mutable.Queue[BatchMetadata](), RecordBatch.NO_PRODUCER_EPOCH, -1, None) + def empty(producerId: Long) = new ProducerIdEntry(producerId, new ConcurrentLinkedDeque[BatchMetadata](), RecordBatch.NO_PRODUCER_EPOCH, -1, None) } private[log] case class BatchMetadata(lastSeq: Int, lastOffset: Long, offsetDelta: Int, timestamp: Long) { @@ -69,24 +71,24 @@ private[log] case class BatchMetadata(lastSeq: Int, lastOffset: Long, offsetDelt // the batchMetadata is ordered such that the batch with the lowest sequence is at the head of the queue while the // batch with the highest sequence is at the tail of the queue. We will retain at most ProducerIdEntry.NumBatchesToRetain // elements in the queue. When the queue is at capacity, we remove the first element to make space for the incoming batch. -private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: mutable.Queue[BatchMetadata], +private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: ConcurrentLinkedDeque[BatchMetadata], var producerEpoch: Short, var coordinatorEpoch: Int, var currentTxnFirstOffset: Option[Long]) { - def firstSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.front.firstSeq - def firstOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.front.firstOffset + def firstSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.peekFirst.firstSeq + def firstOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.peekFirst.firstOffset - def lastSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.last.lastSeq - def lastOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.last.lastOffset - def lastTimestamp = if (batchMetadata.isEmpty) RecordBatch.NO_TIMESTAMP else batchMetadata.last.timestamp + def lastSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.peekLast.lastSeq + def lastOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.peekLast.lastOffset + def lastTimestamp = if (batchMetadata.isEmpty) RecordBatch.NO_TIMESTAMP else batchMetadata.peekLast.timestamp def addBatchMetadata(producerEpoch: Short, lastSeq: Int, lastOffset: Long, offsetDelta: Int, timestamp: Long) = { maybeUpdateEpoch(producerEpoch) if (batchMetadata.size == ProducerIdEntry.NumBatchesToRetain) - batchMetadata.dequeue() + batchMetadata.pollFirst() - batchMetadata.enqueue(BatchMetadata(lastSeq, lastOffset, offsetDelta, timestamp)) + batchMetadata.addLast(BatchMetadata(lastSeq, lastOffset, offsetDelta, timestamp)) } def maybeUpdateEpoch(producerEpoch: Short): Boolean = { @@ -98,7 +100,14 @@ private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: muta false } - def removeBatchesOlderThan(offset: Long) = batchMetadata.dropWhile(_.lastOffset < offset) + def removeBatchesOlderThan(offset: Long) = { + val iterator = batchMetadata.iterator() + while (iterator.hasNext) { + val metadata = iterator.next() + if (metadata.lastOffset < offset) + iterator.remove() + } + } def duplicateOf(batch: RecordBatch): Option[BatchMetadata] = { if (batch.producerEpoch() != producerEpoch) @@ -108,11 +117,12 @@ private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: muta } // Return the batch metadata of the cached batch having the exact sequence range, if any. - def batchWithSequenceRange(firstSeq: Int, lastSeq: Int) : Option[BatchMetadata] = { - val duplicate = batchMetadata.filter { case(metadata) => - firstSeq == metadata.firstSeq && lastSeq == metadata.lastSeq + def batchWithSequenceRange(firstSeq: Int, lastSeq: Int): Option[BatchMetadata] = { + for (metadata <- batchMetadata.asScala) { + if (firstSeq == metadata.firstSeq && lastSeq == metadata.lastSeq) + return Some(metadata) } - duplicate.headOption + None } override def toString: String = { @@ -343,7 +353,9 @@ object ProducerStateManager { val offsetDelta = producerEntryStruct.getInt(OffsetDeltaField) val coordinatorEpoch = producerEntryStruct.getInt(CoordinatorEpochField) val currentTxnFirstOffset = producerEntryStruct.getLong(CurrentTxnFirstOffsetField) - val newEntry = new ProducerIdEntry(producerId, mutable.Queue[BatchMetadata](BatchMetadata(seq, offset, offsetDelta, timestamp)), producerEpoch, + val initialMetadata = new ConcurrentLinkedDeque[BatchMetadata]() + initialMetadata.add(BatchMetadata(seq, offset, offsetDelta, timestamp)) + val newEntry = new ProducerIdEntry(producerId, initialMetadata, producerEpoch, coordinatorEpoch, if (currentTxnFirstOffset >= 0) Some(currentTxnFirstOffset) else None) newEntry } @@ -362,10 +374,10 @@ object ProducerStateManager { val producerEntryStruct = struct.instance(ProducerEntriesField) producerEntryStruct.set(ProducerIdField, producerId) .set(ProducerEpochField, entry.producerEpoch) - .set(LastSequenceField, entry.batchMetadata.last.lastSeq) - .set(LastOffsetField, entry.batchMetadata.last.lastOffset) - .set(OffsetDeltaField, entry.batchMetadata.last.offsetDelta) - .set(TimestampField, entry.batchMetadata.last.timestamp) + .set(LastSequenceField, entry.batchMetadata.peekLast.lastSeq) + .set(LastOffsetField, entry.batchMetadata.peekLast.lastOffset) + .set(OffsetDeltaField, entry.batchMetadata.peekLast.offsetDelta) + .set(TimestampField, entry.batchMetadata.peekLast.timestamp) .set(CoordinatorEpochField, entry.coordinatorEpoch) .set(CurrentTxnFirstOffsetField, entry.currentTxnFirstOffset.getOrElse(-1L)) producerEntryStruct diff --git a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala index aa09acae1aa19..cdce8868b857e 100644 --- a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala @@ -17,6 +17,7 @@ package kafka.log import java.io.File +import java.util.concurrent.ConcurrentLinkedDeque import kafka.utils.TestUtils import kafka.utils.TestUtils.checkEquals @@ -317,8 +318,9 @@ class LogSegmentTest { // recover again, but this time assuming the transaction from pid2 began on a previous segment stateManager = new ProducerStateManager(topicPartition, logDir) - stateManager.loadProducerEntry(new ProducerIdEntry(pid2, mutable.Queue[BatchMetadata](BatchMetadata(10, 90L, 5, RecordBatch.NO_TIMESTAMP)), - producerEpoch, 0, Some(75L))) + val initialMetadata = new ConcurrentLinkedDeque[BatchMetadata]() + initialMetadata.add(BatchMetadata(10, 90L, 5, RecordBatch.NO_TIMESTAMP)) + stateManager.loadProducerEntry(new ProducerIdEntry(pid2, initialMetadata, producerEpoch, 0, Some(75L))) segment.recover(stateManager) assertEquals(108L, stateManager.mapEndOffset) From 5597178fd253b421b6aaa384a82f0f449efbd81b Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 12 Sep 2017 23:06:48 -0700 Subject: [PATCH 25/40] Bunch of changes: 1. Fixed a potentioal NPE in ProducerStateManager.writeSnapshot where we could have no batch metadata in memory (for instance, if only a control batch was written), and yet write a snapshot. This was causing tests to fail because an NPE was raised during shutdown, leaking threads. 2. Addressed the remaining PR comments. --- .../producer/internals/ProducerBatch.java | 2 +- .../producer/internals/RecordAccumulator.java | 95 ++++++++++--------- .../internals/TransactionManager.java | 18 ++-- .../common/record/MemoryRecordsBuilder.java | 2 +- .../producer/internals/SenderTest.java | 14 +-- .../kafka/log/ProducerStateManager.scala | 24 ++--- 6 files changed, 84 insertions(+), 71 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java index f803c4a9b6859..a7076598a0824 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java @@ -389,7 +389,7 @@ public void setProducerState(ProducerIdAndEpoch producerIdAndEpoch, int baseSequ } public void resetProducerState(ProducerIdAndEpoch producerIdAndEpoch, int baseSequence, boolean isTransactional) { - recordsBuilder.unsetProducerState(); + recordsBuilder.reopen(); recordsBuilder.setProducerState(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, baseSequence, isTransactional); } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index 808cae7860a9f..8cf64b11e2ca7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -309,8 +309,10 @@ public void reenqueue(ProducerBatch batch, long now) { batch.reenqueued(now); Deque deque = getOrCreateDeque(batch.topicPartition); synchronized (deque) { - deque.addFirst(batch); - maybeEnsureQueueIsOrdered(deque, batch); + if (transactionManager != null) + insertInSequenceOrder(deque, batch); + else + deque.addFirst(batch); } } @@ -332,11 +334,12 @@ public int splitAndReenqueue(ProducerBatch bigBatch) { incomplete.add(batch); // We treat the newly split batches as if they are not even tried. synchronized (partitionDequeue) { - partitionDequeue.addFirst(batch); if (transactionManager != null) { // We should track the newly created batches since they already have assigned sequences. transactionManager.addInFlightBatch(batch); - maybeEnsureQueueIsOrdered(partitionDequeue, batch); + insertInSequenceOrder(partitionDequeue, batch); + } else { + partitionDequeue.addFirst(batch); } } } @@ -345,49 +348,51 @@ public int splitAndReenqueue(ProducerBatch bigBatch) { // The deque for the partition may have to be reordered in situations where leadership changes in between // batch drains. Since the requests are on different connections, we no longer have any guarantees about ordering - // of the responses. Hence we will have to check if there is anything out of order and correct the order in the queue. + // of the responses. Hence we will have to check if there is anything out of order and ensure the batch is queued + // in the correct sequence order. // // Note that this assumes that all the batches in the queue which have an assigned sequence also have the current // producer id. We will not attempt to reorder messages if the producer id has changed. - private void maybeEnsureQueueIsOrdered(Deque deque, ProducerBatch batch) { - if (transactionManager != null) { - // When we are requeing and have enabled idempotence, the reenqueued batch must always have a sequence. - if (batch.baseSequence() == RecordBatch.NO_SEQUENCE) - throw new IllegalStateException("Trying to reenqueue a batch which doesn't have a sequence even " + - "though idempotence is enabled."); - - // If there are no inflight batches being tracked by the transaction manager, it means that the producer - // id must have changed and the batches being re enqueued are from the old producer id. In this case - // we don't try to ensure ordering amongst them. They will eventually fail with an OutOfOrderSequence, - // or they will succeed. - if (transactionManager.nextBatchBySequence(batch.topicPartition) != null && - batch.baseSequence() != transactionManager.nextBatchBySequence(batch.topicPartition).baseSequence()) { - // The head of the queues have diverged. This means that the incoming batch should be placed somewhere further back. - // We need to find the right place for the incoming batch and insert it there. - // We will only enter this branch if we have multiple inflights sent to different brokers, perhaps - // because a leadership change occurred in between the drains. In this scenario, responses can come - // back out of order, requiring us to re order the batches ourselves rather than relying on the - // implicit ordering guarantees of the network client which are only on a per connection basis. - - ProducerBatch incomingBatch = deque.pollFirst(); - List orderedBatches = new ArrayList<>(); - while (deque.peekFirst() != null && deque.peekFirst().hasSequence() && deque.peekFirst().baseSequence() < incomingBatch.baseSequence()) - orderedBatches.add(deque.pollFirst()); - - log.debug("Reordered incoming batch with sequence {} for partition {}. It was placed in the queue at " + - "position {}", incomingBatch.baseSequence(), incomingBatch.topicPartition, orderedBatches.size()); - // Either we have reached a point where there are batches without a sequence (ie. never been drained - // and are hence in order by default), or the batch at the front of the queue has a sequence greater - // than the incoming batch. This is the right place to add the incoming batch. - deque.addFirst(incomingBatch); - - // Now we have to re insert the previously queued batches in the right order. - for (int i = orderedBatches.size() - 1; i >= 0; --i) { - deque.addFirst(orderedBatches.get(i)); - } - // At this point, the incoming batch has been queued in the correct place according to its sequence. + private void insertInSequenceOrder(Deque deque, ProducerBatch batch) { + // When we are requeing and have enabled idempotence, the reenqueued batch must always have a sequence. + if (batch.baseSequence() == RecordBatch.NO_SEQUENCE) + throw new IllegalStateException("Trying to reenqueue a batch which doesn't have a sequence even " + + "though idempotence is enabled."); + + // If there are no inflight batches being tracked by the transaction manager, it means that the producer + // id must have changed and the batches being re enqueued are from the old producer id. In this case + // we don't try to ensure ordering amongst them. They will eventually fail with an OutOfOrderSequence, + // or they will succeed. + if (transactionManager.nextBatchBySequence(batch.topicPartition) != null && + batch.baseSequence() != transactionManager.nextBatchBySequence(batch.topicPartition).baseSequence()) { + // The incoming batch can't be inserted at the front of the queue without violating the sequence ordering. + // This means that the incoming batch should be placed somewhere further back. + // We need to find the right place for the incoming batch and insert it there. + // We will only enter this branch if we have multiple inflights sent to different brokers, perhaps + // because a leadership change occurred in between the drains. In this scenario, responses can come + // back out of order, requiring us to re order the batches ourselves rather than relying on the + // implicit ordering guarantees of the network client which are only on a per connection basis. + + List orderedBatches = new ArrayList<>(); + while (deque.peekFirst() != null && deque.peekFirst().hasSequence() && deque.peekFirst().baseSequence() < batch.baseSequence()) + orderedBatches.add(deque.pollFirst()); + + log.debug("Reordered incoming batch with sequence {} for partition {}. It was placed in the queue at " + + "position {}", batch.baseSequence(), batch.topicPartition, orderedBatches.size()); + // Either we have reached a point where there are batches without a sequence (ie. never been drained + // and are hence in order by default), or the batch at the front of the queue has a sequence greater + // than the incoming batch. This is the right place to add the incoming batch. + deque.addFirst(batch); + + // Now we have to re insert the previously queued batches in the right order. + for (int i = orderedBatches.size() - 1; i >= 0; --i) { + deque.addFirst(orderedBatches.get(i)); } + + // At this point, the incoming batch has been queued in the correct place according to its sequence. + } else { + deque.addFirst(batch); } } @@ -524,13 +529,15 @@ public Map> drain(Cluster cluster, isTransactional = transactionManager.isTransactional(); - if (!first.hasSequence() && transactionManager.partitionHasUnresolvedSequence(first.topicPartition)) + if (!first.hasSequence() && transactionManager.hasUnresolvedSequence(first.topicPartition)) // Don't drain any new batches while the state of previous sequence numbers // is unknown. The previous batches would be unknown if they were aborted // on the client after being sent to the broker at least once. break; - if (first.hasSequence() && !first.equals(transactionManager.nextBatchBySequence(first.topicPartition))) + if (first.hasSequence() && + (transactionManager.nextBatchBySequence(first.topicPartition) == null + || !first.equals(transactionManager.nextBatchBySequence(first.topicPartition)))) // If the queued batch already has an assigned sequence, then it is being // retried. In this case, we wait until the next immediate batch is ready // and drain that. We only move on when the next in line batch is complete (either successfully diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index c6299098166ea..29e55bee2483d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -429,12 +429,17 @@ public int compare(ProducerBatch o1, ProducerBatch o2) { synchronized ProducerBatch nextBatchBySequence(TopicPartition topicPartition) { - return inflightBatchesBySequence.get(topicPartition).peek(); + PriorityQueue queue = inflightBatchesBySequence.get(topicPartition); + if (queue == null) + return null; + return queue.peek(); } synchronized void removeInFlightBatch(ProducerBatch batch) { - if (inflightBatchesBySequence.containsKey(batch.topicPartition)) - inflightBatchesBySequence.get(batch.topicPartition).remove(batch); + PriorityQueue queue = inflightBatchesBySequence.get(batch.topicPartition); + if (queue == null) + return; + queue.remove(batch); } synchronized void maybeUpdateLastAckedSequence(TopicPartition topicPartition, int sequence) { @@ -484,7 +489,7 @@ synchronized boolean hasInflightBatches(TopicPartition topicPartition) { return inflightBatchesBySequence.containsKey(topicPartition) && !inflightBatchesBySequence.get(topicPartition).isEmpty(); } - synchronized boolean partitionHasUnresolvedSequence(TopicPartition topicPartition) { + synchronized boolean hasUnresolvedSequence(TopicPartition topicPartition) { return partitionsWithUnresolvedSequences.contains(topicPartition); } @@ -515,8 +520,7 @@ synchronized boolean shouldResetProducerStateAfterResolvingSequences() { } synchronized boolean isNextSequence(TopicPartition topicPartition, int sequence) { - return (!lastAckedSequence.containsKey(topicPartition) && sequence == 0) || - (lastAckedSequence.containsKey(topicPartition) && (sequence - lastAckedSequence.get(topicPartition) == 1)); + return sequence - lastAckedSequence(topicPartition) == 1; } private synchronized void setNextSequence(TopicPartition topicPartition, int sequence) { @@ -621,7 +625,7 @@ synchronized boolean hasOngoingTransaction() { synchronized boolean canRetryOutOfOrderSequenceException(ProducerBatch batch) { return hasProducerId(batch.producerId()) - && !partitionHasUnresolvedSequence(batch.topicPartition) + && !hasUnresolvedSequence(batch.topicPartition) && !isNextSequence(batch.topicPartition, batch.baseSequence()); } diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java index 64defa65f33cf..7e49b3fb873cf 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java @@ -279,7 +279,7 @@ public void abort() { aborted = true; } - public void unsetProducerState() { + public void reopen() { if (aborted) throw new IllegalStateException("Should not reopen a batch which is already aborted."); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 12e96bcd109bf..ad040e49808b9 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -792,7 +792,7 @@ public void testExpiryOfUnsentBatchesShouldNotCauseUnresolvedSequences() throws } catch (ExecutionException e) { assertTrue(e.getCause() instanceof TimeoutException); } - assertFalse(transactionManager.partitionHasUnresolvedSequence(tp0)); + assertFalse(transactionManager.hasUnresolvedSequence(tp0)); } @Test @@ -830,7 +830,7 @@ public void testExpiryOfFirstBatchShouldNotCauseUnresolvedSequencesIfFutureBatch } catch (ExecutionException e) { assertTrue(e.getCause() instanceof TimeoutException); } - assertTrue(transactionManager.partitionHasUnresolvedSequence(tp0)); + assertTrue(transactionManager.hasUnresolvedSequence(tp0)); // let's enqueue another batch, which should not be dequeued until the unresolved state is clear. Future request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; @@ -848,10 +848,10 @@ public void testExpiryOfFirstBatchShouldNotCauseUnresolvedSequencesIfFutureBatch assertFalse(batches.peekFirst().hasSequence()); assertFalse(client.hasInFlightRequests()); assertEquals(2L, transactionManager.sequenceNumber(tp0).longValue()); - assertTrue(transactionManager.partitionHasUnresolvedSequence(tp0)); + assertTrue(transactionManager.hasUnresolvedSequence(tp0)); sender.run(time.milliseconds()); // clear the unresolved state, send the pending request. - assertFalse(transactionManager.partitionHasUnresolvedSequence(tp0)); + assertFalse(transactionManager.hasUnresolvedSequence(tp0)); assertTrue(transactionManager.hasProducerId()); assertEquals(0, batches.size()); assertEquals(1, client.inFlightRequestCount()); @@ -893,7 +893,7 @@ public void testExpiryOfFirstBatchShouldCauseResetIfFutureBatchesFail() throws E } catch (ExecutionException e) { assertTrue(e.getCause() instanceof TimeoutException); } - assertTrue(transactionManager.partitionHasUnresolvedSequence(tp0)); + assertTrue(transactionManager.hasUnresolvedSequence(tp0)); // let's enqueue another batch, which should not be dequeued until the unresolved state is clear. Future request3 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; @@ -914,7 +914,7 @@ public void testExpiryOfFirstBatchShouldCauseResetIfFutureBatchesFail() throws E // The producer state should be reset. assertFalse(transactionManager.hasProducerId()); - assertFalse(transactionManager.partitionHasUnresolvedSequence(tp0)); + assertFalse(transactionManager.hasUnresolvedSequence(tp0)); } @Test @@ -949,7 +949,7 @@ public void testExpiryOfAllSentBatchesShouldCauseUnresolvedSequences() throws Ex } catch (ExecutionException e) { assertTrue(e.getCause() instanceof TimeoutException); } - assertTrue(transactionManager.partitionHasUnresolvedSequence(tp0)); + assertTrue(transactionManager.hasUnresolvedSequence(tp0)); assertFalse(client.hasInFlightRequests()); Deque batches = accumulator.batches().get(tp0); assertEquals(0, batches.size()); diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index b6496853dbdf7..a672440a91ef1 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -79,8 +79,9 @@ private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: Conc def firstOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.peekFirst.firstOffset def lastSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.peekLast.lastSeq - def lastOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.peekLast.lastOffset + def lastDataOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.peekLast.lastOffset def lastTimestamp = if (batchMetadata.isEmpty) RecordBatch.NO_TIMESTAMP else batchMetadata.peekLast.timestamp + def lastOffsetDelta : Int = if (batchMetadata.isEmpty) 0 else batchMetadata.peekLast.offsetDelta def addBatchMetadata(producerEpoch: Short, lastSeq: Int, lastOffset: Long, offsetDelta: Int, timestamp: Long) = { maybeUpdateEpoch(producerEpoch) @@ -95,9 +96,10 @@ private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: Conc if (this.producerEpoch != producerEpoch) { batchMetadata.clear() this.producerEpoch = producerEpoch - return true + true + } else { + false } - false } def removeBatchesOlderThan(offset: Long) = { @@ -111,9 +113,9 @@ private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: Conc def duplicateOf(batch: RecordBatch): Option[BatchMetadata] = { if (batch.producerEpoch() != producerEpoch) - return None - - batchWithSequenceRange(batch.baseSequence(), batch.lastSequence()) + None + else + batchWithSequenceRange(batch.baseSequence(), batch.lastSequence()) } // Return the batch metadata of the cached batch having the exact sequence range, if any. @@ -374,10 +376,10 @@ object ProducerStateManager { val producerEntryStruct = struct.instance(ProducerEntriesField) producerEntryStruct.set(ProducerIdField, producerId) .set(ProducerEpochField, entry.producerEpoch) - .set(LastSequenceField, entry.batchMetadata.peekLast.lastSeq) - .set(LastOffsetField, entry.batchMetadata.peekLast.lastOffset) - .set(OffsetDeltaField, entry.batchMetadata.peekLast.offsetDelta) - .set(TimestampField, entry.batchMetadata.peekLast.timestamp) + .set(LastSequenceField, entry.lastSeq) + .set(LastOffsetField, entry.lastDataOffset) + .set(OffsetDeltaField, entry.lastOffsetDelta) + .set(TimestampField, entry.lastTimestamp) .set(CoordinatorEpochField, entry.coordinatorEpoch) .set(CurrentTxnFirstOffsetField, entry.currentTxnFirstOffset.getOrElse(-1L)) producerEntryStruct @@ -609,7 +611,7 @@ class ProducerStateManager(val topicPartition: TopicPartition, private def isProducerRetained(producerIdEntry: ProducerIdEntry, logStartOffset: Long): Boolean = { producerIdEntry.removeBatchesOlderThan(logStartOffset) - producerIdEntry.lastOffset >= logStartOffset + producerIdEntry.lastDataOffset >= logStartOffset } /** From 2ad47d4010afe0f5bff0abe4f91036d21d3c30d9 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 12 Sep 2017 23:15:43 -0700 Subject: [PATCH 26/40] Rename ProducerIdEntry.lastOffset to ProducerIdEntry.lastDataOffset everywhere --- core/src/test/scala/unit/kafka/log/LogTest.scala | 6 +++--- .../unit/kafka/log/ProducerStateManagerTest.scala | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala index da4706e2b6b90..a0d86fff77730 100755 --- a/core/src/test/scala/unit/kafka/log/LogTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogTest.scala @@ -425,7 +425,7 @@ class LogTest { assertEquals(0, entry.firstSeq) assertEquals(baseOffset, entry.firstOffset) assertEquals(3, entry.lastSeq) - assertEquals(baseOffset + 3, entry.lastOffset) + assertEquals(baseOffset + 3, entry.lastDataOffset) } @Test @@ -469,7 +469,7 @@ class LogTest { assertEquals(0, entry.firstSeq) assertEquals(baseOffset, entry.firstOffset) assertEquals(1, entry.lastSeq) - assertEquals(baseOffset + 1, entry.lastOffset) + assertEquals(baseOffset + 1, entry.lastDataOffset) } @Test @@ -505,7 +505,7 @@ class LogTest { assertEquals(0, entry.firstSeq) assertEquals(baseOffset, entry.firstOffset) assertEquals(3, entry.lastSeq) - assertEquals(baseOffset + 3, entry.lastOffset) + assertEquals(baseOffset + 3, entry.lastDataOffset) } @Test diff --git a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala index 99649c9e3c8af..5b7ef3d5a5596 100644 --- a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala +++ b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala @@ -129,7 +129,7 @@ class ProducerStateManagerTest extends JUnitSuite { assertEquals(epoch, lastEntry.producerEpoch) assertEquals(sequence, lastEntry.firstSeq) assertEquals(sequence, lastEntry.lastSeq) - assertEquals(offset, lastEntry.lastOffset) + assertEquals(offset, lastEntry.lastDataOffset) assertEquals(offset, lastEntry.firstOffset) } @@ -210,7 +210,7 @@ class ProducerStateManagerTest extends JUnitSuite { assertEquals(0, lastEntry.firstSeq) assertEquals(5, lastEntry.lastSeq) assertEquals(9L, lastEntry.firstOffset) - assertEquals(20L, lastEntry.lastOffset) + assertEquals(20L, lastEntry.lastDataOffset) assertEquals(Some(16L), lastEntry.currentTxnFirstOffset) assertEquals(List(new TxnMetadata(producerId, 16L)), appendInfo.startedTransactions) @@ -220,7 +220,7 @@ class ProducerStateManagerTest extends JUnitSuite { assertEquals(0, lastEntry.firstSeq) assertEquals(10, lastEntry.lastSeq) assertEquals(9L, lastEntry.firstOffset) - assertEquals(30L, lastEntry.lastOffset) + assertEquals(30L, lastEntry.lastDataOffset) assertEquals(Some(16L), lastEntry.currentTxnFirstOffset) assertEquals(List(new TxnMetadata(producerId, 16L)), appendInfo.startedTransactions) @@ -237,7 +237,7 @@ class ProducerStateManagerTest extends JUnitSuite { assertEquals(0, lastEntry.firstSeq) assertEquals(10, lastEntry.lastSeq) assertEquals(9L, lastEntry.firstOffset) - assertEquals(30L, lastEntry.lastOffset) + assertEquals(30L, lastEntry.lastDataOffset) assertEquals(coordinatorEpoch, lastEntry.coordinatorEpoch) assertEquals(None, lastEntry.currentTxnFirstOffset) assertEquals(List(new TxnMetadata(producerId, 16L)), appendInfo.startedTransactions) @@ -429,7 +429,7 @@ class ProducerStateManagerTest extends JUnitSuite { val maybeEntry = stateManager.lastEntry(anotherPid) assertTrue(maybeEntry.isDefined) - assertEquals(3L, maybeEntry.get.lastOffset) + assertEquals(3L, maybeEntry.get.lastDataOffset) stateManager.truncateHead(3) assertEquals(Set(anotherPid), stateManager.activeProducers.keySet) @@ -460,7 +460,7 @@ class ProducerStateManagerTest extends JUnitSuite { val entry = stateManager.lastEntry(pid2) assertTrue(entry.isDefined) assertEquals(0, entry.get.lastSeq) - assertEquals(1L, entry.get.lastOffset) + assertEquals(1L, entry.get.lastDataOffset) } @Test @@ -671,7 +671,7 @@ class ProducerStateManagerTest extends JUnitSuite { assertFalse(snapshotToTruncate.exists()) val loadedProducerState = reloadedStateManager.activeProducers(producerId) - assertEquals(0L, loadedProducerState.lastOffset) + assertEquals(0L, loadedProducerState.lastDataOffset) } private def appendEndTxnMarker(mapping: ProducerStateManager, From 66bf9faeac10f23ec0d562254dc37edc78f507f3 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 12 Sep 2017 23:24:38 -0700 Subject: [PATCH 27/40] Address more PR comments --- .../org/apache/kafka/clients/producer/internals/Sender.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index ff2537d7e48e1..9cb41f4be68d2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -544,7 +544,7 @@ else if (error == Errors.CLUSTER_AUTHORIZATION_FAILED) else exception = error.exception(); // tell the user the result of their request. We only adjust sequence numbers if the batch didn't exhaust - // its retries -- if it did, we don't know the whether the sequence number was accepted or not, and + // its retries -- if it did, we don't know whether the sequence number was accepted or not, and // thus it is not safe to reassign the sequence. failBatch(batch, response, exception, batch.attempts() < this.retries); } From 8bc2df21e8ff35f0769a5b5b36ad4790c2f4bd4d Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Tue, 12 Sep 2017 23:32:51 -0700 Subject: [PATCH 28/40] Fix previous commit which introduced a bug in RecordAccmulator.drain. If a batch has a sequence, but there are no recorded inflight requests to that partition, it means that the producer id has changed, and the batch under consideration was from a previous producerId. It should always be drained (if it returns again, it will fail fatally). The code before this patch would ensure that it would _never_ drain. --- .../kafka/clients/producer/internals/RecordAccumulator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index 8cf64b11e2ca7..d77e66c9b4512 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -535,9 +535,9 @@ public Map> drain(Cluster cluster, // on the client after being sent to the broker at least once. break; - if (first.hasSequence() && - (transactionManager.nextBatchBySequence(first.topicPartition) == null - || !first.equals(transactionManager.nextBatchBySequence(first.topicPartition)))) + if (first.hasSequence() + && transactionManager.nextBatchBySequence(first.topicPartition) != null + && !first.equals(transactionManager.nextBatchBySequence(first.topicPartition))) // If the queued batch already has an assigned sequence, then it is being // retried. In this case, we wait until the next immediate batch is ready // and drain that. We only move on when the next in line batch is complete (either successfully From f394747f8017a05d9cc35965c15efd40f31a9928 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 13 Sep 2017 10:36:49 -0700 Subject: [PATCH 29/40] PR comment: return currentLastSequence instead of doing another map lookup --- .../kafka/clients/producer/internals/TransactionManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index 29e55bee2483d..560689b420ab1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -451,7 +451,7 @@ synchronized int lastAckedSequence(TopicPartition topicPartition) { Integer currentLastAckedSequence = lastAckedSequence.get(topicPartition); if (currentLastAckedSequence == null) return -1; - return lastAckedSequence.get(topicPartition); + return currentLastAckedSequence; } // If a batch is failed fatally, the sequence numbers for future batches bound for the partition must be adjusted From 04d9b3bc7bea83e9da0ec121c47c4974136d94d5 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 13 Sep 2017 11:45:44 -0700 Subject: [PATCH 30/40] Remove the concurrent queue from ProducerIdEntry and instead return just the lastSeq per producer to the LogCleaner, hence simpliying the concurrency model --- .../internals/TransactionManager.java | 3 + core/src/main/scala/kafka/log/Log.scala | 6 +- .../src/main/scala/kafka/log/LogCleaner.scala | 6 +- .../kafka/log/ProducerStateManager.scala | 48 ++++------- .../scala/unit/kafka/log/LogSegmentTest.scala | 4 +- .../test/scala/unit/kafka/log/LogTest.scala | 79 ++++++++----------- 6 files changed, 61 insertions(+), 85 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index 560689b420ab1..9c0c5acc114f7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -500,6 +500,9 @@ synchronized void markSequenceUnresolved(TopicPartition topicPartition) { // Checks if there are any partitions with unresolved partitions which may now be resolved. Returns true if // the producer id needs a reset, false otherwise. synchronized boolean shouldResetProducerStateAfterResolvingSequences() { + if (isTransactional()) + // We should not reset producer state if we are transactional. We should transition to a fatal error instead. + return false; for (TopicPartition topicPartition : partitionsWithUnresolvedSequences) { if (!hasInflightBatches(topicPartition)) { // The partition has been fully drained. At this point, the last ack'd sequence should be once less than diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala index 11db803389eef..a16b29ca226b4 100644 --- a/core/src/main/scala/kafka/log/Log.scala +++ b/core/src/main/scala/kafka/log/Log.scala @@ -514,8 +514,10 @@ class Log(@volatile var dir: File, completedTxns.foreach(producerStateManager.completeTxn) } - private[log] def activeProducers: Map[Long, ProducerIdEntry] = lock synchronized { - producerStateManager.activeProducers + private[log] def activeProducersWithLastSequence: Map[Long, Int] = lock synchronized { + producerStateManager.activeProducers.map { case (producerId, producerIdEntry) => + (producerId, producerIdEntry.lastSeq) + } } /** diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala index 4f53b41df4fc0..61dd0fc47c874 100644 --- a/core/src/main/scala/kafka/log/LogCleaner.scala +++ b/core/src/main/scala/kafka/log/LogCleaner.scala @@ -450,7 +450,7 @@ private[log] class Cleaner(val id: Int, info("Cleaning segment %s in log %s (largest timestamp %s) into %s, %s deletes." .format(startOffset, log.name, new Date(oldSegmentOpt.largestTimestamp), cleaned.baseOffset, if(retainDeletes) "retaining" else "discarding")) cleanInto(log.topicPartition, oldSegmentOpt, cleaned, map, retainDeletes, log.config.maxMessageSize, transactionMetadata, - log.activeProducers, stats) + log.activeProducersWithLastSequence, stats) currentSegmentOpt = nextSegmentOpt } @@ -503,7 +503,7 @@ private[log] class Cleaner(val id: Int, retainDeletes: Boolean, maxLogMessageSize: Int, transactionMetadata: CleanedTransactionMetadata, - activeProducers: Map[Long, ProducerIdEntry], + activeProducers: Map[Long, Int], stats: CleanerStats) { val logCleanerFilter = new RecordFilter { var discardBatchRecords: Boolean = _ @@ -515,7 +515,7 @@ private[log] class Cleaner(val id: Int, // check if the batch contains the last sequence number for the producer. if so, we cannot // remove the batch just yet or the producer may see an out of sequence error. - if (batch.hasProducerId && activeProducers.get(batch.producerId).exists(_.lastSeq == batch.lastSequence)) + if (batch.hasProducerId && activeProducers.get(batch.producerId).contains(batch.lastSequence)) BatchRetention.RETAIN_EMPTY else if (discardBatchRecords) BatchRetention.DELETE diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index a672440a91ef1..790765987432f 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -51,7 +51,7 @@ private[log] case class TxnMetadata(producerId: Long, var firstOffset: LogOffset private[log] object ProducerIdEntry { private[log] val NumBatchesToRetain = 5 - def empty(producerId: Long) = new ProducerIdEntry(producerId, new ConcurrentLinkedDeque[BatchMetadata](), RecordBatch.NO_PRODUCER_EPOCH, -1, None) + def empty(producerId: Long) = new ProducerIdEntry(producerId, mutable.Queue[BatchMetadata](), RecordBatch.NO_PRODUCER_EPOCH, -1, None) } private[log] case class BatchMetadata(lastSeq: Int, lastOffset: Long, offsetDelta: Int, timestamp: Long) { @@ -71,25 +71,25 @@ private[log] case class BatchMetadata(lastSeq: Int, lastOffset: Long, offsetDelt // the batchMetadata is ordered such that the batch with the lowest sequence is at the head of the queue while the // batch with the highest sequence is at the tail of the queue. We will retain at most ProducerIdEntry.NumBatchesToRetain // elements in the queue. When the queue is at capacity, we remove the first element to make space for the incoming batch. -private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: ConcurrentLinkedDeque[BatchMetadata], +private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: mutable.Queue[BatchMetadata], var producerEpoch: Short, var coordinatorEpoch: Int, var currentTxnFirstOffset: Option[Long]) { - def firstSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.peekFirst.firstSeq - def firstOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.peekFirst.firstOffset + def firstSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.front.firstSeq + def firstOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.front.firstOffset - def lastSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.peekLast.lastSeq - def lastDataOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.peekLast.lastOffset - def lastTimestamp = if (batchMetadata.isEmpty) RecordBatch.NO_TIMESTAMP else batchMetadata.peekLast.timestamp - def lastOffsetDelta : Int = if (batchMetadata.isEmpty) 0 else batchMetadata.peekLast.offsetDelta + def lastSeq: Int = if (batchMetadata.isEmpty) RecordBatch.NO_SEQUENCE else batchMetadata.last.lastSeq + def lastDataOffset: Long = if (batchMetadata.isEmpty) -1L else batchMetadata.last.lastOffset + def lastTimestamp = if (batchMetadata.isEmpty) RecordBatch.NO_TIMESTAMP else batchMetadata.last.timestamp + def lastOffsetDelta : Int = if (batchMetadata.isEmpty) 0 else batchMetadata.last.offsetDelta def addBatchMetadata(producerEpoch: Short, lastSeq: Int, lastOffset: Long, offsetDelta: Int, timestamp: Long) = { maybeUpdateEpoch(producerEpoch) if (batchMetadata.size == ProducerIdEntry.NumBatchesToRetain) - batchMetadata.pollFirst() + batchMetadata.dequeue() - batchMetadata.addLast(BatchMetadata(lastSeq, lastOffset, offsetDelta, timestamp)) + batchMetadata.enqueue(BatchMetadata(lastSeq, lastOffset, offsetDelta, timestamp)) } def maybeUpdateEpoch(producerEpoch: Short): Boolean = { @@ -102,14 +102,7 @@ private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: Conc } } - def removeBatchesOlderThan(offset: Long) = { - val iterator = batchMetadata.iterator() - while (iterator.hasNext) { - val metadata = iterator.next() - if (metadata.lastOffset < offset) - iterator.remove() - } - } + def removeBatchesOlderThan(offset: Long) = batchMetadata.dropWhile(_.lastOffset < offset) def duplicateOf(batch: RecordBatch): Option[BatchMetadata] = { if (batch.producerEpoch() != producerEpoch) @@ -120,11 +113,10 @@ private[log] class ProducerIdEntry(val producerId: Long, val batchMetadata: Conc // Return the batch metadata of the cached batch having the exact sequence range, if any. def batchWithSequenceRange(firstSeq: Int, lastSeq: Int): Option[BatchMetadata] = { - for (metadata <- batchMetadata.asScala) { - if (firstSeq == metadata.firstSeq && lastSeq == metadata.lastSeq) - return Some(metadata) + val duplicate = batchMetadata.filter { case(metadata) => + firstSeq == metadata.firstSeq && lastSeq == metadata.lastSeq } - None + duplicate.headOption } override def toString: String = { @@ -248,14 +240,6 @@ private[log] class ProducerAppendInfo(val producerId: Long, throw new TransactionCoordinatorFencedException(s"Invalid coordinator epoch: ${endTxnMarker.coordinatorEpoch} " + s"(zombie), ${currentEntry.coordinatorEpoch} (current)") - // TODO(reviewers): The semantics of the ProducerIdEntry have changed so that now explicitly caches the metadata - // of the last 5 RecordBatches appended to the partition by a producer. So we don't need to care about the offset - // of the control batches anymore, since they never need to de-duped. I think that the new code is simpler, but - // it would be worth discussing whether we want to preserve the old behavior of retaining the offset of the control - // batch in the review. - - // it is possible that this control record is the first record seen from a new epoch, for instance if the coordinator - // times out a transaction. In this case, we bump the epoch and reset the sequence numbers currentEntry.maybeUpdateEpoch(producerEpoch) val firstOffset = currentEntry.currentTxnFirstOffset match { @@ -355,9 +339,7 @@ object ProducerStateManager { val offsetDelta = producerEntryStruct.getInt(OffsetDeltaField) val coordinatorEpoch = producerEntryStruct.getInt(CoordinatorEpochField) val currentTxnFirstOffset = producerEntryStruct.getLong(CurrentTxnFirstOffsetField) - val initialMetadata = new ConcurrentLinkedDeque[BatchMetadata]() - initialMetadata.add(BatchMetadata(seq, offset, offsetDelta, timestamp)) - val newEntry = new ProducerIdEntry(producerId, initialMetadata, producerEpoch, + val newEntry = new ProducerIdEntry(producerId, mutable.Queue[BatchMetadata](BatchMetadata(seq, offset, offsetDelta, timestamp)), producerEpoch, coordinatorEpoch, if (currentTxnFirstOffset >= 0) Some(currentTxnFirstOffset) else None) newEntry } diff --git a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala index cdce8868b857e..61a8492db558f 100644 --- a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala @@ -318,9 +318,7 @@ class LogSegmentTest { // recover again, but this time assuming the transaction from pid2 began on a previous segment stateManager = new ProducerStateManager(topicPartition, logDir) - val initialMetadata = new ConcurrentLinkedDeque[BatchMetadata]() - initialMetadata.add(BatchMetadata(10, 90L, 5, RecordBatch.NO_TIMESTAMP)) - stateManager.loadProducerEntry(new ProducerIdEntry(pid2, initialMetadata, producerEpoch, 0, Some(75L))) + stateManager.loadProducerEntry(new ProducerIdEntry(pid2, mutable.Queue[BatchMetadata](BatchMetadata(10, 90L, 5, RecordBatch.NO_TIMESTAMP)), producerEpoch, 0, Some(75L))) segment.recover(stateManager) assertEquals(108L, stateManager.mapEndOffset) diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala index a0d86fff77730..2ae62c5338051 100755 --- a/core/src/test/scala/unit/kafka/log/LogTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogTest.scala @@ -418,14 +418,11 @@ class LogTest { log.truncateTo(baseOffset + 4) - val activeProducers = log.activeProducers + val activeProducers = log.activeProducersWithLastSequence assertTrue(activeProducers.contains(pid)) - val entry = activeProducers(pid) - assertEquals(0, entry.firstSeq) - assertEquals(baseOffset, entry.firstOffset) - assertEquals(3, entry.lastSeq) - assertEquals(baseOffset + 3, entry.lastDataOffset) + val lastSeq = activeProducers(pid) + assertEquals(3, lastSeq) } @Test @@ -462,14 +459,11 @@ class LogTest { log.truncateTo(baseOffset + 2) - val activeProducers = log.activeProducers + val activeProducers = log.activeProducersWithLastSequence assertTrue(activeProducers.contains(pid)) - val entry = activeProducers(pid) - assertEquals(0, entry.firstSeq) - assertEquals(baseOffset, entry.firstOffset) - assertEquals(1, entry.lastSeq) - assertEquals(baseOffset + 1, entry.lastDataOffset) + val lastSeq = activeProducers(pid) + assertEquals(1, lastSeq) } @Test @@ -498,14 +492,11 @@ class LogTest { val filteredRecords = MemoryRecords.readableRecords(filtered) log.appendAsFollower(filteredRecords) - val activeProducers = log.activeProducers + val activeProducers = log.activeProducersWithLastSequence assertTrue(activeProducers.contains(pid)) - val entry = activeProducers(pid) - assertEquals(0, entry.firstSeq) - assertEquals(baseOffset, entry.firstOffset) - assertEquals(3, entry.lastSeq) - assertEquals(baseOffset + 3, entry.lastDataOffset) + val lastSeq = activeProducers(pid) + assertEquals(3, lastSeq) } @Test @@ -547,13 +538,13 @@ class LogTest { } log.truncateTo(1L) - assertEquals(1, log.activeProducers.size) + assertEquals(1, log.activeProducersWithLastSequence.size) - val pidEntryOpt = log.activeProducers.get(pid) - assertTrue(pidEntryOpt.isDefined) + val lastSeqOpt = log.activeProducersWithLastSequence.get(pid) + assertTrue(lastSeqOpt.isDefined) - val pidEntry = pidEntryOpt.get - assertEquals(0, pidEntry.lastSeq) + val lastSeq = lastSeqOpt.get + assertEquals(0, lastSeq) } @Test @@ -568,21 +559,21 @@ class LogTest { producerEpoch = epoch, sequence = 0), leaderEpoch = 0) log.appendAsLeader(TestUtils.records(List(new SimpleRecord(mockTime.milliseconds(), "b".getBytes)), producerId = pid2, producerEpoch = epoch, sequence = 0), leaderEpoch = 0) - assertEquals(2, log.activeProducers.size) + assertEquals(2, log.activeProducersWithLastSequence.size) log.maybeIncrementLogStartOffset(1L) - assertEquals(1, log.activeProducers.size) - val retainedEntryOpt = log.activeProducers.get(pid2) - assertTrue(retainedEntryOpt.isDefined) - assertEquals(0, retainedEntryOpt.get.lastSeq) + assertEquals(1, log.activeProducersWithLastSequence.size) + val retainedLastSeqOpt = log.activeProducersWithLastSequence.get(pid2) + assertTrue(retainedLastSeqOpt.isDefined) + assertEquals(0, retainedLastSeqOpt.get) log.close() val reloadedLog = createLog(logDir, logConfig, logStartOffset = 1L) - assertEquals(1, reloadedLog.activeProducers.size) - val reloadedEntryOpt = log.activeProducers.get(pid2) - assertEquals(retainedEntryOpt, reloadedEntryOpt) + assertEquals(1, reloadedLog.activeProducersWithLastSequence.size) + val reloadedLastSeqOpt = log.activeProducersWithLastSequence.get(pid2) + assertEquals(retainedLastSeqOpt, reloadedLastSeqOpt) } @Test @@ -600,24 +591,24 @@ class LogTest { producerEpoch = epoch, sequence = 0), leaderEpoch = 0) assertEquals(2, log.logSegments.size) - assertEquals(2, log.activeProducers.size) + assertEquals(2, log.activeProducersWithLastSequence.size) log.maybeIncrementLogStartOffset(1L) log.onHighWatermarkIncremented(log.logEndOffset) log.deleteOldSegments() assertEquals(1, log.logSegments.size) - assertEquals(1, log.activeProducers.size) - val retainedEntryOpt = log.activeProducers.get(pid2) - assertTrue(retainedEntryOpt.isDefined) - assertEquals(0, retainedEntryOpt.get.lastSeq) + assertEquals(1, log.activeProducersWithLastSequence.size) + val retainedLastSeqOpt = log.activeProducersWithLastSequence.get(pid2) + assertTrue(retainedLastSeqOpt.isDefined) + assertEquals(0, retainedLastSeqOpt.get) log.close() val reloadedLog = createLog(logDir, logConfig, logStartOffset = 1L) - assertEquals(1, reloadedLog.activeProducers.size) - val reloadedEntryOpt = log.activeProducers.get(pid2) - assertEquals(retainedEntryOpt, reloadedEntryOpt) + assertEquals(1, reloadedLog.activeProducersWithLastSequence.size) + val reloadedEntryOpt = log.activeProducersWithLastSequence.get(pid2) + assertEquals(retainedLastSeqOpt, reloadedEntryOpt) } @Test @@ -659,13 +650,13 @@ class LogTest { log.takeProducerSnapshot() assertEquals(3, log.logSegments.size) - assertEquals(Set(pid1, pid2), log.activeProducers.keySet) + assertEquals(Set(pid1, pid2), log.activeProducersWithLastSequence.keySet) log.onHighWatermarkIncremented(log.logEndOffset) log.deleteOldSegments() assertEquals(2, log.logSegments.size) - assertEquals(Set(pid2), log.activeProducers.keySet) + assertEquals(Set(pid2), log.activeProducersWithLastSequence.keySet) } @Test @@ -749,13 +740,13 @@ class LogTest { val records = Seq(new SimpleRecord(mockTime.milliseconds(), "foo".getBytes)) log.appendAsLeader(TestUtils.records(records, producerId = pid, producerEpoch = 0, sequence = 0), leaderEpoch = 0) - assertEquals(Set(pid), log.activeProducers.keySet) + assertEquals(Set(pid), log.activeProducersWithLastSequence.keySet) mockTime.sleep(producerIdExpirationCheckIntervalMs) - assertEquals(Set(pid), log.activeProducers.keySet) + assertEquals(Set(pid), log.activeProducersWithLastSequence.keySet) mockTime.sleep(producerIdExpirationCheckIntervalMs) - assertEquals(Set(), log.activeProducers.keySet) + assertEquals(Set(), log.activeProducersWithLastSequence.keySet) } @Test From 9c8810ca681b9433b944c60de35dc6389840fb25 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 13 Sep 2017 11:50:21 -0700 Subject: [PATCH 31/40] Address other minor comments --- .../clients/producer/internals/TransactionManager.java | 2 +- .../scala/unit/kafka/log/ProducerStateManagerTest.scala | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index 9c0c5acc114f7..cca0a1a56ed1d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -508,7 +508,7 @@ synchronized boolean shouldResetProducerStateAfterResolvingSequences() { // The partition has been fully drained. At this point, the last ack'd sequence should be once less than // next sequence destined for the partition. If so, the partition is fully resolved. If not, we should // reset the sequence number if necessary. - if (lastAckedSequence(topicPartition) == sequenceNumber(topicPartition) - 1) { + if (isNextSequence(topicPartition, sequenceNumber(topicPartition))) { // This would happen when a batch was expired, but subsequent batches succeeded. partitionsWithUnresolvedSequences.remove(topicPartition); } else { diff --git a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala index 5b7ef3d5a5596..976bbd7e92148 100644 --- a/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala +++ b/core/src/test/scala/unit/kafka/log/ProducerStateManagerTest.scala @@ -95,13 +95,7 @@ class ProducerStateManagerTest extends JUnitSuite { val lastEntry = maybeLastEntry.get assertEquals(epoch, lastEntry.producerEpoch) - // TODO(reviewers): The semantics of the producer state manager have been changed so that we store the - // last N batches. As such, the ProducerIdEntry.firstSeq returns the first seq of the first batch in the cache. - // The ProducerIdEntry.lastSeq is the last seq of the last batch in the cache. As a result, during wraparound - // the firstSeq could be greater than the lastSeq. - // - // This seems reasonable, but it is worth at least discussing the new semantics of these methods when reviewing - // the new code. + assertEquals(Int.MaxValue, lastEntry.firstSeq) assertEquals(0, lastEntry.lastSeq) } From a61730802e0cac1fb0f83561edfd8470d83b69ec Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 13 Sep 2017 12:31:13 -0700 Subject: [PATCH 32/40] Correctly transition to a fatal error state when batches in retry are expired during a transaction --- .../clients/producer/internals/Sender.java | 3 + .../internals/TransactionManager.java | 7 ++- .../internals/TransactionManagerTest.java | 59 +++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 9cb41f4be68d2..dd4145065cf1a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -203,6 +203,9 @@ void run(long now) { if (!transactionManager.isTransactional()) { // this is an idempotent producer, so make sure we have a producer id maybeWaitForProducerId(); + } else if (transactionManager.isTransactional() && transactionManager.hasUnresolvedSequences() && !transactionManager.hasFatalError()) { + transactionManager.transitionToFatalError(new KafkaException("The client hasn't received acknowledgment for " + + "some previously sent messages and can no longer retry them. It isn't safe to continue.")); } else if (transactionManager.hasInFlightRequest() || maybeSendTransactionalRequest(now)) { // as long as there are outstanding transactional requests, we simply wait for them to return client.poll(retryBackoffMs, now); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index cca0a1a56ed1d..58f320400f8ef 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -489,11 +489,16 @@ synchronized boolean hasInflightBatches(TopicPartition topicPartition) { return inflightBatchesBySequence.containsKey(topicPartition) && !inflightBatchesBySequence.get(topicPartition).isEmpty(); } + synchronized boolean hasUnresolvedSequences() { + return !partitionsWithUnresolvedSequences.isEmpty(); + } + synchronized boolean hasUnresolvedSequence(TopicPartition topicPartition) { return partitionsWithUnresolvedSequences.contains(topicPartition); } synchronized void markSequenceUnresolved(TopicPartition topicPartition) { + log.debug("{}Marking partition {} unresolved", logPrefix, topicPartition); partitionsWithUnresolvedSequences.add(topicPartition); } @@ -501,7 +506,7 @@ synchronized void markSequenceUnresolved(TopicPartition topicPartition) { // the producer id needs a reset, false otherwise. synchronized boolean shouldResetProducerStateAfterResolvingSequences() { if (isTransactional()) - // We should not reset producer state if we are transactional. We should transition to a fatal error instead. + // We should not reset producer state if we are transactional. We will transition to a fatal error instead. return false; for (TopicPartition topicPartition : partitionsWithUnresolvedSequences) { if (!hasInflightBatches(topicPartition)) { diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java index d90a18611d8cc..101cf93d36e14 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java @@ -2152,6 +2152,65 @@ public void testDropCommitOnBatchExpiry() throws InterruptedException, Execution assertFalse(transactionManager.transactionContainsPartition(tp0)); } + @Test + public void testTransitionToFatalErrorWhenRetriedBatchIsExpired() throws InterruptedException, ExecutionException { + final long pid = 13131L; + final short epoch = 1; + + doInitTransactions(pid, epoch); + + transactionManager.beginTransaction(); + transactionManager.maybeAddPartitionToTransaction(tp0); + + Future responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), + "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future; + + assertFalse(responseFuture.isDone()); + + prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); + + assertFalse(transactionManager.transactionContainsPartition(tp0)); + assertFalse(transactionManager.isSendToPartitionAllowed(tp0)); + sender.run(time.milliseconds()); // send addPartitions. + // Check that only addPartitions was sent. + assertTrue(transactionManager.transactionContainsPartition(tp0)); + assertTrue(transactionManager.isSendToPartitionAllowed(tp0)); + + prepareProduceResponse(Errors.NOT_LEADER_FOR_PARTITION, pid, epoch); + sender.run(time.milliseconds()); // send the produce request. + + assertFalse(responseFuture.isDone()); + + TransactionalRequestResult commitResult = transactionManager.beginCommit(); + + // Sleep 10 seconds to make sure that the batches in the queue would be expired if they can't be drained. + time.sleep(10000); + // Disconnect the target node for the pending produce request. This will ensure that sender will try to + // expire the batch. + Node clusterNode = this.cluster.nodes().get(0); + client.disconnect(clusterNode.idString()); + client.blackout(clusterNode, 100); + + sender.run(time.milliseconds()); // We should try to flush the produce, but expire it instead without sending anything. + assertTrue(responseFuture.isDone()); + + try { + // make sure the produce was expired. + responseFuture.get(); + fail("Expected to get a TimeoutException since the queued ProducerBatch should have been expired"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof TimeoutException); + } + sender.run(time.milliseconds()); // Transition to fatal error since we have unresolved batches. + sender.run(time.milliseconds()); // Fail the queued transactional requests + + assertTrue(commitResult.isCompleted()); + assertFalse(commitResult.isSuccessful()); // the commit should have been dropped. + + assertTrue(transactionManager.hasFatalError()); + assertFalse(transactionManager.hasOngoingTransaction()); + } + private void verifyAddPartitionsFailsWithPartitionLevelError(final Errors error) throws InterruptedException { final long pid = 1L; final short epoch = 1; From 78cea1ca25793dd70a133fec73fb4ed9eac820a8 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 13 Sep 2017 13:23:15 -0700 Subject: [PATCH 33/40] Added assertions on the send futures for some of the newly added test cases in SenderTest --- .../clients/producer/internals/SenderTest.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index ad040e49808b9..8624d07a24990 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -506,11 +506,14 @@ public void testIdempotenceWithMultipleInflights() throws Exception { assertEquals(1, client.inFlightRequestCount()); assertEquals(0, transactionManager.lastAckedSequence(tp0)); + assertNotNull(request1.get()); + assertFalse(request2.isDone()); sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L); sender.run(time.milliseconds()); // receive response 1 assertEquals(1, transactionManager.lastAckedSequence(tp0)); assertFalse(client.hasInFlightRequests()); + assertNotNull(request2.get()); } @@ -567,11 +570,15 @@ public void testIdempotenceWithMultipleInflightsFirstFails() throws Exception { sender.run(time.milliseconds()); // receive response 0 assertEquals(0, transactionManager.lastAckedSequence(tp0)); assertEquals(0, client.inFlightRequestCount()); + assertFalse(request2.isDone()); + assertNotNull(request1.get()); + sender.run(time.milliseconds()); // send request 1 assertEquals(1, client.inFlightRequestCount()); sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L); sender.run(time.milliseconds()); // receive response 1 + assertNotNull(request2.get()); assertFalse(client.hasInFlightRequests()); assertEquals(1, transactionManager.lastAckedSequence(tp0)); } @@ -607,7 +614,7 @@ public void testMustNotRetryOutOfOrderSequenceForNextBatch() throws Exception { assertEquals(1, client.inFlightRequestCount()); assertEquals(3, transactionManager.sequenceNumber(tp0).longValue()); assertEquals(1, transactionManager.lastAckedSequence(tp0)); - assertTrue(request1.isDone()); + assertNotNull(request1.get()); assertFalse(request2.isDone()); assertTrue(client.isReady(node, time.milliseconds())); @@ -678,6 +685,8 @@ public void testCorrectHandlingOfOutOfOrderResponses() throws Exception { assertEquals(1, queuedBatches.peekLast().baseSequence()); assertEquals(-1, transactionManager.lastAckedSequence(tp0)); assertEquals(0, client.inFlightRequestCount()); + assertFalse(request1.isDone()); + assertFalse(request2.isDone()); sender.run(time.milliseconds()); // send request 0 assertEquals(1, client.inFlightRequestCount()); @@ -691,6 +700,7 @@ public void testCorrectHandlingOfOutOfOrderResponses() throws Exception { sender.run(time.milliseconds()); // receive response 0 assertEquals(0, transactionManager.lastAckedSequence(tp0)); assertEquals(0, client.inFlightRequestCount()); + assertNotNull(request1.get()); sender.run(time.milliseconds()); // send request 1 assertEquals(1, client.inFlightRequestCount()); sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L); @@ -698,6 +708,7 @@ public void testCorrectHandlingOfOutOfOrderResponses() throws Exception { assertFalse(client.hasInFlightRequests()); assertEquals(1, transactionManager.lastAckedSequence(tp0)); + assertNotNull(request2.get()); } @Test From 9629c491b4aaafa9043a054ca49ceff3ee6e6e9f Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 13 Sep 2017 14:17:18 -0700 Subject: [PATCH 34/40] check the response offsets in all the tests --- .../producer/internals/SenderTest.java | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 8624d07a24990..6c31fd4c8af97 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -506,14 +506,16 @@ public void testIdempotenceWithMultipleInflights() throws Exception { assertEquals(1, client.inFlightRequestCount()); assertEquals(0, transactionManager.lastAckedSequence(tp0)); - assertNotNull(request1.get()); + assertTrue(request1.isDone()); + assertEquals(0, request1.get().offset()); assertFalse(request2.isDone()); sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L); sender.run(time.milliseconds()); // receive response 1 assertEquals(1, transactionManager.lastAckedSequence(tp0)); assertFalse(client.hasInFlightRequests()); - assertNotNull(request2.get()); + assertTrue(request2.isDone()); + assertEquals(1, request2.get().offset()); } @@ -570,15 +572,18 @@ public void testIdempotenceWithMultipleInflightsFirstFails() throws Exception { sender.run(time.milliseconds()); // receive response 0 assertEquals(0, transactionManager.lastAckedSequence(tp0)); assertEquals(0, client.inFlightRequestCount()); + assertFalse(request2.isDone()); - assertNotNull(request1.get()); + assertTrue(request1.isDone()); + assertEquals(0, request1.get().offset()); sender.run(time.milliseconds()); // send request 1 assertEquals(1, client.inFlightRequestCount()); sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L); sender.run(time.milliseconds()); // receive response 1 - assertNotNull(request2.get()); + assertTrue(request2.isDone()); + assertEquals(1, request2.get().offset()); assertFalse(client.hasInFlightRequests()); assertEquals(1, transactionManager.lastAckedSequence(tp0)); } @@ -614,7 +619,8 @@ public void testMustNotRetryOutOfOrderSequenceForNextBatch() throws Exception { assertEquals(1, client.inFlightRequestCount()); assertEquals(3, transactionManager.sequenceNumber(tp0).longValue()); assertEquals(1, transactionManager.lastAckedSequence(tp0)); - assertNotNull(request1.get()); + assertTrue(request1.isDone()); + assertEquals(0, request1.get().offset()); assertFalse(request2.isDone()); assertTrue(client.isReady(node, time.milliseconds())); @@ -700,7 +706,9 @@ public void testCorrectHandlingOfOutOfOrderResponses() throws Exception { sender.run(time.milliseconds()); // receive response 0 assertEquals(0, transactionManager.lastAckedSequence(tp0)); assertEquals(0, client.inFlightRequestCount()); - assertNotNull(request1.get()); + assertTrue(request1.isDone()); + assertEquals(0, request1.get().offset()); + sender.run(time.milliseconds()); // send request 1 assertEquals(1, client.inFlightRequestCount()); sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L); @@ -708,7 +716,8 @@ public void testCorrectHandlingOfOutOfOrderResponses() throws Exception { assertFalse(client.hasInFlightRequests()); assertEquals(1, transactionManager.lastAckedSequence(tp0)); - assertNotNull(request2.get()); + assertTrue(request2.isDone()); + assertEquals(1, request2.get().offset()); } @Test @@ -739,10 +748,11 @@ public void testCorrectHandlingOfOutOfOrderResponsesWhenSecondSucceeds() throws ClientRequest firstClientRequest = client.requests().peek(); ClientRequest secondClientRequest = (ClientRequest) client.requests().toArray()[1]; - client.respondToRequest(secondClientRequest, produceResponse(tp0, 1, Errors.NONE, -1)); + client.respondToRequest(secondClientRequest, produceResponse(tp0, 1, Errors.NONE, 1)); sender.run(time.milliseconds()); // receive response 1 - assertNotNull(request2.get()); + assertTrue(request2.isDone()); + assertEquals(1, request2.get().offset()); assertFalse(request1.isDone()); Deque queuedBatches = accumulator.batches().get(tp0); @@ -774,7 +784,8 @@ public void testCorrectHandlingOfOutOfOrderResponsesWhenSecondSucceeds() throws assertEquals(0, client.inFlightRequestCount()); assertFalse(client.hasInFlightRequests()); - assertNotNull(request1.get()); + assertTrue(request1.isDone()); + assertEquals(0, request1.get().offset()); } @Test @@ -852,7 +863,8 @@ public void testExpiryOfFirstBatchShouldNotCauseUnresolvedSequencesIfFutureBatch sender.run(time.milliseconds()); // send second request sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1); sender.run(time.milliseconds()); // receive second response, the third request shouldn't be sent since we are in an unresolved state. - assertNotNull(request2.get()); + assertTrue(request2.isDone()); + assertEquals(1, request2.get().offset()); Deque batches = accumulator.batches().get(tp0); assertEquals(1, batches.size()); From 9bc3ef6b8d8b72331efe8c3a8fe9ff670e5698ca Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 13 Sep 2017 14:35:36 -0700 Subject: [PATCH 35/40] Fix checkstyle failures --- .../org/apache/kafka/clients/producer/internals/SenderTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 6c31fd4c8af97..6040437115c69 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -80,7 +80,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; From 27b58a45184edac4c34e3d8a4556049e5ac3e570 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Wed, 13 Sep 2017 18:00:44 -0700 Subject: [PATCH 36/40] Address further PR comments --- .../clients/producer/internals/ProducerBatch.java | 1 - .../clients/producer/internals/RecordAccumulator.java | 10 ++++++---- .../main/scala/kafka/log/ProducerStateManager.scala | 2 -- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java index a7076598a0824..94d52642829c6 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java @@ -255,7 +255,6 @@ public Deque split(int splitBatchSize) { ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId(), producerEpoch()); for (ProducerBatch newBatch : batches) { newBatch.setProducerState(producerIdAndEpoch, sequence, isTransactional()); - newBatch.retry = true; sequence += newBatch.recordCount; } } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java index d77e66c9b4512..f3cb4faf8573b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java @@ -360,12 +360,15 @@ private void insertInSequenceOrder(Deque deque, ProducerBatch bat throw new IllegalStateException("Trying to reenqueue a batch which doesn't have a sequence even " + "though idempotence is enabled."); + if (transactionManager.nextBatchBySequence(batch.topicPartition) == null) + throw new IllegalStateException("We are reenqueueing a batch which is not tracked as part of the in flight " + + "requests. batch.topicPartition: " + batch.topicPartition + "; batch.baseSequence: " + batch.baseSequence()); + // If there are no inflight batches being tracked by the transaction manager, it means that the producer // id must have changed and the batches being re enqueued are from the old producer id. In this case // we don't try to ensure ordering amongst them. They will eventually fail with an OutOfOrderSequence, // or they will succeed. - if (transactionManager.nextBatchBySequence(batch.topicPartition) != null && - batch.baseSequence() != transactionManager.nextBatchBySequence(batch.topicPartition).baseSequence()) { + if (batch.baseSequence() != transactionManager.nextBatchBySequence(batch.topicPartition).baseSequence()) { // The incoming batch can't be inserted at the front of the queue without violating the sequence ordering. // This means that the incoming batch should be placed somewhere further back. // We need to find the right place for the incoming batch and insert it there. @@ -536,8 +539,7 @@ public Map> drain(Cluster cluster, break; if (first.hasSequence() - && transactionManager.nextBatchBySequence(first.topicPartition) != null - && !first.equals(transactionManager.nextBatchBySequence(first.topicPartition))) + && first.baseSequence() != transactionManager.nextBatchBySequence(first.topicPartition).baseSequence()) // If the queued batch already has an assigned sequence, then it is being // retried. In this case, we wait until the next immediate batch is ready // and drain that. We only move on when the next in line batch is complete (either successfully diff --git a/core/src/main/scala/kafka/log/ProducerStateManager.scala b/core/src/main/scala/kafka/log/ProducerStateManager.scala index 790765987432f..4c3d1a18d712d 100644 --- a/core/src/main/scala/kafka/log/ProducerStateManager.scala +++ b/core/src/main/scala/kafka/log/ProducerStateManager.scala @@ -31,8 +31,6 @@ import org.apache.kafka.common.protocol.types._ import org.apache.kafka.common.record.{ControlRecordType, EndTransactionMarker, RecordBatch} import org.apache.kafka.common.utils.{ByteUtils, Crc32C} -import java.util.concurrent.ConcurrentLinkedDeque -import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer import scala.collection.{immutable, mutable} From 3902de0c40007c5cfdcca469d67aab6643dba91a Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 14 Sep 2017 12:37:07 -0700 Subject: [PATCH 37/40] Address PR comments. One notable update: We didn't have a test for reassiging sequence numbers, and we had a bug in that logic. Essentially a newly reassigned batch would be fail with an out of order sequence exception because it would be the next in line batch after its sequence was adjusted. This is fixed by tracking whether the sequence of a batch has been adjusted since the last drain, in which case the out of order sequence exception for it is always retried. If it fails again, it will be fatal. --- .../producer/internals/ProducerBatch.java | 8 +- .../internals/TransactionManager.java | 10 +-- .../common/record/MemoryRecordsBuilder.java | 14 +++- .../producer/internals/SenderTest.java | 77 ++++++++++++++++++- 4 files changed, 99 insertions(+), 10 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java index 94d52642829c6..b1a86aa1c19c6 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java @@ -388,8 +388,7 @@ public void setProducerState(ProducerIdAndEpoch producerIdAndEpoch, int baseSequ } public void resetProducerState(ProducerIdAndEpoch producerIdAndEpoch, int baseSequence, boolean isTransactional) { - recordsBuilder.reopen(); - recordsBuilder.setProducerState(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, baseSequence, isTransactional); + recordsBuilder.reopenAndResetProducerState(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, baseSequence, isTransactional); } /** @@ -459,4 +458,9 @@ public boolean hasSequence() { public boolean isTransactional() { return recordsBuilder.isTransactional(); } + + public boolean sequenceHasBeenReset() { + return recordsBuilder.isReopened(); + } + } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index 58f320400f8ef..c641c03d5420a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -464,7 +464,8 @@ synchronized void adjustSequencesDueToFailedBatch(ProducerBatch batch) { // Sequence numbers are not being tracked for this partition. This could happen if the producer id was just // reset due to a previous OutOfOrderSequenceException. return; - + log.debug("{}producerId: {}, send to partition {} failed fatally. Reducing future sequence numbers by {}", logPrefix, + batch.producerId(), batch.topicPartition, batch.recordCount); int currentSequence = sequenceNumber(batch.topicPartition); currentSequence -= batch.recordCount; if (currentSequence < 0) @@ -480,7 +481,7 @@ synchronized void adjustSequencesDueToFailedBatch(ProducerBatch batch) { throw new IllegalStateException("Sequence number for batch with sequence " + inFlightBatch.baseSequence() + " for partition " + batch.topicPartition + " is going to become negative :" + newSequence); - log.debug("Setting sequence number of batch with current sequence {} for partition {} to {}", batch.baseSequence(), batch.topicPartition, newSequence); + log.info("Resetting sequence number of batch with current sequence {} for partition {} to {}", inFlightBatch.baseSequence(), batch.topicPartition, newSequence); inFlightBatch.resetProducerState(new ProducerIdAndEpoch(inFlightBatch.producerId(), inFlightBatch.producerEpoch()), newSequence, inFlightBatch.isTransactional()); } } @@ -632,9 +633,8 @@ synchronized boolean hasOngoingTransaction() { } synchronized boolean canRetryOutOfOrderSequenceException(ProducerBatch batch) { - return hasProducerId(batch.producerId()) - && !hasUnresolvedSequence(batch.topicPartition) - && !isNextSequence(batch.topicPartition, batch.baseSequence()); + return hasProducerId(batch.producerId()) && !hasUnresolvedSequence(batch.topicPartition) && + (batch.sequenceHasBeenReset() || !isNextSequence(batch.topicPartition, batch.baseSequence())); } // visible for testing diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java index 7e49b3fb873cf..a033ca016cd1a 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java @@ -75,6 +75,7 @@ public class MemoryRecordsBuilder { private MemoryRecords builtRecords; private boolean aborted = false; + private boolean reopened = false; public MemoryRecordsBuilder(ByteBufferOutputStream bufferStream, byte magic, @@ -279,11 +280,19 @@ public void abort() { aborted = true; } - public void reopen() { + public void reopenAndResetProducerState(long producerId, short producerEpoch, int baseSequence, boolean isTransactional) { if (aborted) throw new IllegalStateException("Should not reopen a batch which is already aborted."); - builtRecords = null; + reopened = true; + this.producerId = producerId; + this.producerEpoch = producerEpoch; + this.baseSequence = baseSequence; + this.isTransactional = isTransactional; + } + + public boolean isReopened() { + return reopened; } public void close() { @@ -311,6 +320,7 @@ else if (compressionType != CompressionType.NONE) buffer.position(initialPosition); builtRecords = MemoryRecords.readableRecords(buffer.slice()); } + reopened = false; } private void validateProducerState() { diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 6040437115c69..4e6dfa104d9e3 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -30,6 +30,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.ClusterAuthorizationException; import org.apache.kafka.common.errors.OutOfOrderSequenceException; +import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.UnsupportedForMessageFormatException; import org.apache.kafka.common.errors.UnsupportedVersionException; @@ -587,6 +588,72 @@ public void testIdempotenceWithMultipleInflightsFirstFails() throws Exception { assertEquals(1, transactionManager.lastAckedSequence(tp0)); } + @Test + public void testIdempotenceWithMultipleInflightsWhereFirstFailsFatallyAndSequenceOfFutureBatchesIsAdjusted() throws Exception { + final long producerId = 343434L; + TransactionManager transactionManager = new TransactionManager(); + setupWithTransactionState(transactionManager); + prepareAndReceiveInitProducerId(producerId, Errors.NONE); + assertTrue(transactionManager.hasProducerId()); + + assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); + + // Send first ProduceRequest + Future request1 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + String nodeId = client.requests().peek().destination(); + Node node = new Node(Integer.valueOf(nodeId), "localhost", 0); + assertEquals(1, client.inFlightRequestCount()); + assertEquals(1, transactionManager.sequenceNumber(tp0).longValue()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + + // Send second ProduceRequest + Future request2 = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; + sender.run(time.milliseconds()); + assertEquals(2, client.inFlightRequestCount()); + assertEquals(2, transactionManager.sequenceNumber(tp0).longValue()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + assertFalse(request1.isDone()); + assertFalse(request2.isDone()); + assertTrue(client.isReady(node, time.milliseconds())); + + sendIdempotentProducerResponse(0, tp0, Errors.MESSAGE_TOO_LARGE, -1L); + + sender.run(time.milliseconds()); // receive response 0, should adjust sequences of future batches. + + assertTrue(request1.isDone()); + try { + request1.get(); + fail("Should have raised an error"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof RecordTooLargeException); + } + + assertEquals(1, client.inFlightRequestCount()); + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + + sendIdempotentProducerResponse(1, tp0, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, -1L); + + sender.run(time.milliseconds()); // receive response 1 + + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + assertEquals(0, client.inFlightRequestCount()); + + sender.run(time.milliseconds()); // resend request 1 + + assertEquals(1, client.inFlightRequestCount()); + + assertEquals(-1, transactionManager.lastAckedSequence(tp0)); + + sendIdempotentProducerResponse(0, tp0, Errors.NONE, 0L); + sender.run(time.milliseconds()); // receive response 1 + assertEquals(0, transactionManager.lastAckedSequence(tp0)); + assertEquals(0, client.inFlightRequestCount()); + + assertTrue(request1.isDone()); + assertEquals(0, request2.get().offset()); + } + @Test public void testMustNotRetryOutOfOrderSequenceForNextBatch() throws Exception { final long producerId = 343434L; @@ -835,7 +902,7 @@ public void testExpiryOfFirstBatchShouldNotCauseUnresolvedSequencesIfFutureBatch assertEquals(2, client.inFlightRequestCount()); - sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_FOR_PARTITION, -1); + sendIdempotentProducerResponse(0, tp0, Errors.REQUEST_TIMED_OUT, -1); sender.run(time.milliseconds()); // receive first response Node node = this.cluster.nodes().get(0); @@ -927,6 +994,14 @@ public void testExpiryOfFirstBatchShouldCauseResetIfFutureBatchesFail() throws E sendIdempotentProducerResponse(1, tp0, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, 1); sender.run(time.milliseconds()); // receive second response, the third request shouldn't be sent since we are in an unresolved state. assertTrue(request2.isDone()); + + try { + request2.get(); + fail("should have failed with an exception"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof OutOfOrderSequenceException); + } + Deque batches = accumulator.batches().get(tp0); // The second request should not be requeued. From 2d6b5de55839b5ce607ea629e31770755f2936ec Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 14 Sep 2017 13:00:32 -0700 Subject: [PATCH 38/40] Fix up build failures after merge --- .../kafka/clients/producer/internals/TransactionManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index c641c03d5420a..c0c4ed9924703 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -464,7 +464,7 @@ synchronized void adjustSequencesDueToFailedBatch(ProducerBatch batch) { // Sequence numbers are not being tracked for this partition. This could happen if the producer id was just // reset due to a previous OutOfOrderSequenceException. return; - log.debug("{}producerId: {}, send to partition {} failed fatally. Reducing future sequence numbers by {}", logPrefix, + log.debug("producerId: {}, send to partition {} failed fatally. Reducing future sequence numbers by {}", batch.producerId(), batch.topicPartition, batch.recordCount); int currentSequence = sequenceNumber(batch.topicPartition); currentSequence -= batch.recordCount; @@ -499,7 +499,7 @@ synchronized boolean hasUnresolvedSequence(TopicPartition topicPartition) { } synchronized void markSequenceUnresolved(TopicPartition topicPartition) { - log.debug("{}Marking partition {} unresolved", logPrefix, topicPartition); + log.debug("Marking partition {} unresolved", topicPartition); partitionsWithUnresolvedSequences.add(topicPartition); } From f7ad085b9f17d5c1d0e24eea5b1715ed4a6123f5 Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 14 Sep 2017 13:52:51 -0700 Subject: [PATCH 39/40] Address more PR comments: 1. Move the 'reopen' flag from MemoryRecordsBuilder to ProducerBatch 2. Move the logic to resolve sequence number state to the top of the run loop. 3. More canonical scala in Log.scala. --- .../producer/internals/ProducerBatch.java | 7 ++++-- .../clients/producer/internals/Sender.java | 10 ++++++--- .../internals/TransactionManager.java | 8 +++---- .../common/record/MemoryRecordsBuilder.java | 8 +------ .../producer/internals/SenderTest.java | 8 +++---- .../internals/TransactionManagerTest.java | 22 +++++++++---------- core/src/main/scala/kafka/log/Log.scala | 14 ++++-------- 7 files changed, 36 insertions(+), 41 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java index b1a86aa1c19c6..93c843b9163a1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java @@ -75,6 +75,7 @@ private enum FinalState { ABORTED, FAILED, SUCCEEDED } private long drainedMs; private String expiryErrorMessage; private boolean retry; + private boolean reopened = false; public ProducerBatch(TopicPartition tp, MemoryRecordsBuilder recordsBuilder, long now) { this(tp, recordsBuilder, now, false); @@ -388,7 +389,8 @@ public void setProducerState(ProducerIdAndEpoch producerIdAndEpoch, int baseSequ } public void resetProducerState(ProducerIdAndEpoch producerIdAndEpoch, int baseSequence, boolean isTransactional) { - recordsBuilder.reopenAndResetProducerState(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, baseSequence, isTransactional); + reopened = true; + recordsBuilder.reopenAndRewriteProducerState(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, baseSequence, isTransactional); } /** @@ -406,6 +408,7 @@ public void close() { recordsBuilder.compressionType(), (float) recordsBuilder.compressionRatio()); } + reopened = false; } /** @@ -460,7 +463,7 @@ public boolean isTransactional() { } public boolean sequenceHasBeenReset() { - return recordsBuilder.isReopened(); + return reopened; } } diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index dd4145065cf1a..0ba908c7e8b2d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -200,13 +200,17 @@ public void run() { */ void run(long now) { if (transactionManager != null) { + if (transactionManager.shouldResetProducerStateAfterResolvingSequences()) + // Check if the previous run expired batches which requires a reset of the producer state. + transactionManager.resetProducerId(); + if (!transactionManager.isTransactional()) { // this is an idempotent producer, so make sure we have a producer id maybeWaitForProducerId(); - } else if (transactionManager.isTransactional() && transactionManager.hasUnresolvedSequences() && !transactionManager.hasFatalError()) { + } else if (transactionManager.hasUnresolvedSequences() && !transactionManager.hasFatalError()) { transactionManager.transitionToFatalError(new KafkaException("The client hasn't received acknowledgment for " + "some previously sent messages and can no longer retry them. It isn't safe to continue.")); - } else if (transactionManager.hasInFlightRequest() || maybeSendTransactionalRequest(now)) { + } else if (transactionManager.hasInFlightTransactionalRequest() || maybeSendTransactionalRequest(now)) { // as long as there are outstanding transactional requests, we simply wait for them to return client.poll(retryBackoffMs, now); return; @@ -348,7 +352,7 @@ private boolean maybeSendTransactionalRequest(long now) { ClientRequest clientRequest = client.newClientRequest(targetNode.idString(), requestBuilder, now, true, nextRequestHandler); - transactionManager.setInFlightRequestCorrelationId(clientRequest.correlationId()); + transactionManager.setInFlightTransactionalRequestCorrelationId(clientRequest.correlationId()); log.debug("Sending transactional request {} to node {}", requestBuilder, targetNode); client.send(clientRequest, now); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java index c0c4ed9924703..b2387a0a74e10 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java @@ -594,15 +594,15 @@ void lookupCoordinator(TxnRequestHandler request) { lookupCoordinator(request.coordinatorType(), request.coordinatorKey()); } - void setInFlightRequestCorrelationId(int correlationId) { + void setInFlightTransactionalRequestCorrelationId(int correlationId) { inFlightRequestCorrelationId = correlationId; } - void clearInFlightRequestCorrelationId() { + void clearInFlightTransactionalRequestCorrelationId() { inFlightRequestCorrelationId = NO_INFLIGHT_REQUEST_CORRELATION_ID; } - boolean hasInFlightRequest() { + boolean hasInFlightTransactionalRequest() { return inFlightRequestCorrelationId != NO_INFLIGHT_REQUEST_CORRELATION_ID; } @@ -787,7 +787,7 @@ public void onComplete(ClientResponse response) { if (response.requestHeader().correlationId() != inFlightRequestCorrelationId) { fatalError(new RuntimeException("Detected more than one in-flight transactional request.")); } else { - clearInFlightRequestCorrelationId(); + clearInFlightTransactionalRequestCorrelationId(); if (response.wasDisconnected()) { log.debug("Disconnected from {}. Will retry.", response.destination()); if (this.needsCoordinator()) diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java index a033ca016cd1a..fc83134ebe4e8 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java @@ -75,7 +75,6 @@ public class MemoryRecordsBuilder { private MemoryRecords builtRecords; private boolean aborted = false; - private boolean reopened = false; public MemoryRecordsBuilder(ByteBufferOutputStream bufferStream, byte magic, @@ -280,20 +279,16 @@ public void abort() { aborted = true; } - public void reopenAndResetProducerState(long producerId, short producerEpoch, int baseSequence, boolean isTransactional) { + public void reopenAndRewriteProducerState(long producerId, short producerEpoch, int baseSequence, boolean isTransactional) { if (aborted) throw new IllegalStateException("Should not reopen a batch which is already aborted."); builtRecords = null; - reopened = true; this.producerId = producerId; this.producerEpoch = producerEpoch; this.baseSequence = baseSequence; this.isTransactional = isTransactional; } - public boolean isReopened() { - return reopened; - } public void close() { if (aborted) @@ -320,7 +315,6 @@ else if (compressionType != CompressionType.NONE) buffer.position(initialPosition); builtRecords = MemoryRecords.readableRecords(buffer.slice()); } - reopened = false; } private void validateProducerState() { diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 4e6dfa104d9e3..26e3e6c49e8cc 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -1050,10 +1050,10 @@ public void testExpiryOfAllSentBatchesShouldCauseUnresolvedSequences() throws Ex assertFalse(client.hasInFlightRequests()); Deque batches = accumulator.batches().get(tp0); assertEquals(0, batches.size()); - assertTrue(transactionManager.hasProducerId()); - - sender.run(time.milliseconds()); // we should reset the producer state. - assertFalse(transactionManager.hasProducerId()); + assertTrue(transactionManager.hasProducerId(producerId)); + // We should now clear the old producerId and get a new one in a single run loop. + prepareAndReceiveInitProducerId(producerId + 1, Errors.NONE); + assertTrue(transactionManager.hasProducerId(producerId + 1)); } @Test diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java index 101cf93d36e14..28f9c820c4a15 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java @@ -1482,14 +1482,14 @@ public void testCommitTransactionWithUnsentProduceRequest() throws Exception { sender.run(time.milliseconds()); assertFalse(accumulator.hasUndrained()); assertTrue(accumulator.hasIncomplete()); - assertFalse(transactionManager.hasInFlightRequest()); + assertFalse(transactionManager.hasInFlightTransactionalRequest()); assertFalse(responseFuture.isDone()); // until the produce future returns, we will not send EndTxn sender.run(time.milliseconds()); assertFalse(accumulator.hasUndrained()); assertTrue(accumulator.hasIncomplete()); - assertFalse(transactionManager.hasInFlightRequest()); + assertFalse(transactionManager.hasInFlightTransactionalRequest()); assertFalse(responseFuture.isDone()); // now the produce response returns @@ -1498,14 +1498,14 @@ public void testCommitTransactionWithUnsentProduceRequest() throws Exception { assertTrue(responseFuture.isDone()); assertFalse(accumulator.hasUndrained()); assertFalse(accumulator.hasIncomplete()); - assertFalse(transactionManager.hasInFlightRequest()); + assertFalse(transactionManager.hasInFlightTransactionalRequest()); // now we send EndTxn sender.run(time.milliseconds()); - assertTrue(transactionManager.hasInFlightRequest()); + assertTrue(transactionManager.hasInFlightTransactionalRequest()); sendEndTxnResponse(Errors.NONE, TransactionResult.COMMIT, pid, epoch); sender.run(time.milliseconds()); - assertFalse(transactionManager.hasInFlightRequest()); + assertFalse(transactionManager.hasInFlightTransactionalRequest()); assertTrue(transactionManager.isReady()); } @@ -1530,21 +1530,21 @@ public void testCommitTransactionWithInFlightProduceRequest() throws Exception { sender.run(time.milliseconds()); assertFalse(accumulator.hasUndrained()); assertTrue(accumulator.hasIncomplete()); - assertFalse(transactionManager.hasInFlightRequest()); + assertFalse(transactionManager.hasInFlightTransactionalRequest()); // now we begin the commit with the produce request still pending transactionManager.beginCommit(); sender.run(time.milliseconds()); assertFalse(accumulator.hasUndrained()); assertTrue(accumulator.hasIncomplete()); - assertFalse(transactionManager.hasInFlightRequest()); + assertFalse(transactionManager.hasInFlightTransactionalRequest()); assertFalse(responseFuture.isDone()); // until the produce future returns, we will not send EndTxn sender.run(time.milliseconds()); assertFalse(accumulator.hasUndrained()); assertTrue(accumulator.hasIncomplete()); - assertFalse(transactionManager.hasInFlightRequest()); + assertFalse(transactionManager.hasInFlightTransactionalRequest()); assertFalse(responseFuture.isDone()); // now the produce response returns @@ -1553,14 +1553,14 @@ public void testCommitTransactionWithInFlightProduceRequest() throws Exception { assertTrue(responseFuture.isDone()); assertFalse(accumulator.hasUndrained()); assertFalse(accumulator.hasIncomplete()); - assertFalse(transactionManager.hasInFlightRequest()); + assertFalse(transactionManager.hasInFlightTransactionalRequest()); // now we send EndTxn sender.run(time.milliseconds()); - assertTrue(transactionManager.hasInFlightRequest()); + assertTrue(transactionManager.hasInFlightTransactionalRequest()); sendEndTxnResponse(Errors.NONE, TransactionResult.COMMIT, pid, epoch); sender.run(time.milliseconds()); - assertFalse(transactionManager.hasInFlightRequest()); + assertFalse(transactionManager.hasInFlightTransactionalRequest()); assertTrue(transactionManager.isReady()); } diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala index a16b29ca226b4..d98f443e2629d 100644 --- a/core/src/main/scala/kafka/log/Log.scala +++ b/core/src/main/scala/kafka/log/Log.scala @@ -775,17 +775,11 @@ class Log(@volatile var dir: File, // if this is a client produce request, there will be upto 5 batches which could have been duplicated. // If we find a duplicate, we return the metadata of the appended batch to the client. - if (isFromClient) { - maybeLastEntry match { - case Some(lastEntry) => - lastEntry.duplicateOf(batch) match { - case Some(duplicateBatch) => - return (updatedProducers, completedTxns.toList, Option(duplicateBatch)) - case _ => - } - case _ => + if (isFromClient) + maybeLastEntry.flatMap(_.duplicateOf(batch)).foreach { duplicate => + return (updatedProducers, completedTxns.toList, Some(duplicate)) } - } + val maybeCompletedTxn = updateProducers(batch, updatedProducers, loadingFromLog = false) maybeCompletedTxn.foreach(completedTxns += _) } From b1f653004d0de4776178c466a48c3a126d5d2e0e Mon Sep 17 00:00:00 2001 From: Apurva Mehta Date: Thu, 14 Sep 2017 14:31:00 -0700 Subject: [PATCH 40/40] Address final PR comment --- .../org/apache/kafka/clients/producer/internals/Sender.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 0ba908c7e8b2d..2da47a8430a17 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -236,11 +236,6 @@ void run(long now) { private long sendProducerData(long now) { Cluster cluster = metadata.fetch(); - - if (transactionManager != null && transactionManager.shouldResetProducerStateAfterResolvingSequences()) { - transactionManager.resetProducerId(); - return 0; - } // get the list of partitions with data ready to send RecordAccumulator.ReadyCheckResult result = this.accumulator.ready(cluster, now);