From dba6ee106a34644f82c43b3dfd12c170d14cd274 Mon Sep 17 00:00:00 2001 From: David Mao <47232755+splett2@users.noreply.github.com> Date: Mon, 20 Jul 2020 17:41:13 -0500 Subject: [PATCH] Revert "Merge AK Trunk -> CCS master - July 20 (#368)" This reverts commit 8b5f70f9e3682570aad0537b33bb82bb0bee8ce9. --- .../clients/consumer/internals/Fetcher.java | 4 +- .../OffsetsForLeaderEpochClient.java | 2 +- .../kafka/clients/producer/MockProducer.java | 2 +- .../NotLeaderForPartitionException.java | 4 +- .../errors/NotLeaderOrFollowerException.java | 48 ----- .../errors/ReplicaNotAvailableException.java | 8 +- .../apache/kafka/common/protocol/Errors.java | 10 +- .../requests/AlterClientQuotasResponse.java | 13 +- .../kafka/common/requests/ApiError.java | 5 +- .../requests/DeleteRecordsResponse.java | 2 +- .../DescribeClientQuotasResponse.java | 11 +- .../kafka/common/requests/FetchRequest.java | 2 +- .../kafka/common/requests/FetchResponse.java | 10 +- .../common/requests/ListOffsetResponse.java | 4 +- .../OffsetsForLeaderEpochResponse.java | 4 +- .../kafka/common/requests/ProduceRequest.java | 2 +- .../common/requests/ProduceResponse.java | 8 +- .../requests/SaslAuthenticateRequest.java | 5 +- .../requests/WriteTxnMarkersResponse.java | 2 +- .../kafka/common/security/JaasContext.java | 6 +- .../SaslClientAuthenticator.java | 4 +- .../security/ssl/DefaultSslEngineFactory.java | 8 +- .../security/ssl/SslPrincipalMapper.java | 3 +- .../clients/admin/KafkaAdminClientTest.java | 12 +- .../clients/consumer/KafkaConsumerTest.java | 2 +- .../consumer/internals/FetcherTest.java | 20 +- .../producer/internals/SenderTest.java | 20 +- .../internals/TransactionManagerTest.java | 12 +- .../GarbageCollectedMemoryPoolTest.java | 1 - .../common/network/SslTransportLayerTest.java | 1 - .../requests/LeaderAndIsrResponseTest.java | 2 +- .../common/requests/RequestResponseTest.java | 200 +++++++----------- .../requests/StopReplicaResponseTest.java | 2 +- ...ExpiringCredentialRefreshingLoginTest.java | 1 - .../scala/kafka/admin/ConfigCommand.scala | 9 - .../main/scala/kafka/cluster/Partition.scala | 21 +- .../group/GroupMetadataManager.scala | 4 +- ...actionMarkerRequestCompletionHandler.scala | 2 +- .../transaction/TransactionStateManager.scala | 2 +- .../kafka/server/AbstractFetcherThread.scala | 4 +- .../scala/kafka/server/AdminManager.scala | 12 +- .../kafka/server/DelayedDeleteRecords.scala | 2 +- .../scala/kafka/server/DelayedFetch.scala | 10 +- .../scala/kafka/server/DelayedProduce.scala | 2 +- .../scala/kafka/server/DynamicConfig.scala | 13 +- .../main/scala/kafka/server/KafkaApis.scala | 6 +- .../server/ReplicaAlterLogDirsThread.scala | 10 +- .../scala/kafka/server/ReplicaManager.scala | 56 ++--- .../api/PlaintextAdminIntegrationTest.scala | 2 +- .../api/ProducerFailureHandlingTest.scala | 2 +- .../kafka/server/DelayedFetchTest.scala | 12 +- .../unit/kafka/admin/ConfigCommandTest.scala | 21 -- .../unit/kafka/cluster/PartitionTest.scala | 22 +- .../group/GroupMetadataManagerTest.scala | 4 +- ...onMarkerRequestCompletionHandlerTest.scala | 4 +- .../TransactionStateManagerTest.scala | 2 +- .../server/ClientQuotasRequestTest.scala | 30 +-- .../unit/kafka/server/FetchRequestTest.scala | 2 +- .../unit/kafka/server/KafkaApisTest.scala | 4 +- .../kafka/server/ListOffsetsRequestTest.scala | 14 +- .../unit/kafka/server/LogDirFailureTest.scala | 10 +- .../OffsetsForLeaderEpochRequestTest.scala | 8 +- .../kafka/server/ProduceRequestTest.scala | 2 +- .../ReplicaAlterLogDirsThreadTest.scala | 24 +-- .../server/ReplicaFetcherThreadTest.scala | 2 +- .../server/ReplicaManagerQuotasTest.scala | 3 +- .../kafka/server/ReplicaManagerTest.scala | 54 ++--- .../epoch/LeaderEpochIntegrationTest.scala | 2 +- .../epoch/OffsetsForLeaderEpochTest.scala | 2 +- docs/upgrade.html | 4 - .../internals/StreamsPartitionAssignor.java | 51 ++--- .../internals/assignment/ClientState.java | 121 +++++------ .../assignment/AssignmentTestUtils.java | 7 - .../internals/assignment/ClientStateTest.java | 88 ++------ tests/kafkatest/services/connect.py | 25 +-- .../client_compatibility_features_test.py | 3 +- ...ient_compatibility_produce_consume_test.py | 3 +- tests/kafkatest/tests/client/consumer_test.py | 4 +- .../tests/connect/connect_distributed_test.py | 14 -- .../compatibility_test_new_broker_test.py | 3 +- tests/kafkatest/tests/core/upgrade_test.py | 4 +- 81 files changed, 404 insertions(+), 742 deletions(-) delete mode 100644 clients/src/main/java/org/apache/kafka/common/errors/NotLeaderOrFollowerException.java diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 3251f4fa804e8..52c5155fe1fe0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -1043,7 +1043,7 @@ private void handleListOffsetResponse(Map implements Producer { private boolean producerFenced; private boolean sentOffsets; private long commitCount = 0L; - private final Map mockMetrics; + private Map mockMetrics; public RuntimeException initTransactionException = null; public RuntimeException beginTransactionException = null; diff --git a/clients/src/main/java/org/apache/kafka/common/errors/NotLeaderForPartitionException.java b/clients/src/main/java/org/apache/kafka/common/errors/NotLeaderForPartitionException.java index 6ecce02a2a84b..718277905751d 100644 --- a/clients/src/main/java/org/apache/kafka/common/errors/NotLeaderForPartitionException.java +++ b/clients/src/main/java/org/apache/kafka/common/errors/NotLeaderForPartitionException.java @@ -17,10 +17,8 @@ package org.apache.kafka.common.errors; /** - * This server is not the leader for the given partition. - * @deprecated since 2.7. Use {@link NotLeaderOrFollowerException}. + * This server is not the leader for the given partition */ -@Deprecated public class NotLeaderForPartitionException extends InvalidMetadataException { private static final long serialVersionUID = 1L; diff --git a/clients/src/main/java/org/apache/kafka/common/errors/NotLeaderOrFollowerException.java b/clients/src/main/java/org/apache/kafka/common/errors/NotLeaderOrFollowerException.java deleted file mode 100644 index 2db960b738e07..0000000000000 --- a/clients/src/main/java/org/apache/kafka/common/errors/NotLeaderOrFollowerException.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.common.errors; - -/** - * Broker returns this error if a request could not be processed because the broker is not the leader - * or follower for a topic partition. This could be a transient exception during leader elections and - * reassignments. For `Produce` and other requests which are intended only for the leader, this exception - * indicates that the broker is not the current leader. For consumer `Fetch` requests which may be - * satisfied by a leader or follower, this exception indicates that the broker is not a replica - * of the topic partition. - */ -@SuppressWarnings("deprecation") -public class NotLeaderOrFollowerException extends NotLeaderForPartitionException { - - private static final long serialVersionUID = 1L; - - public NotLeaderOrFollowerException() { - super(); - } - - public NotLeaderOrFollowerException(String message) { - super(message); - } - - public NotLeaderOrFollowerException(Throwable cause) { - super(cause); - } - - public NotLeaderOrFollowerException(String message, Throwable cause) { - super(message, cause); - } - -} diff --git a/clients/src/main/java/org/apache/kafka/common/errors/ReplicaNotAvailableException.java b/clients/src/main/java/org/apache/kafka/common/errors/ReplicaNotAvailableException.java index 07971cd57e18a..b94d400d8fe31 100644 --- a/clients/src/main/java/org/apache/kafka/common/errors/ReplicaNotAvailableException.java +++ b/clients/src/main/java/org/apache/kafka/common/errors/ReplicaNotAvailableException.java @@ -16,13 +16,7 @@ */ package org.apache.kafka.common.errors; -/** - * The replica is not available for the requested topic partition. This may be - * a transient exception during reassignments. From version 2.6 onwards, Fetch requests - * and other requests intended only for the leader or follower of the topic partition return - * {@link NotLeaderOrFollowerException} if the broker is a not a replica of the partition. - */ -public class ReplicaNotAvailableException extends InvalidMetadataException { +public class ReplicaNotAvailableException extends ApiException { private static final long serialVersionUID = 1L; diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java index 2b091f32e34f9..526b4dfc0ba04 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java @@ -72,7 +72,7 @@ import org.apache.kafka.common.errors.NotCoordinatorException; import org.apache.kafka.common.errors.NotEnoughReplicasAfterAppendException; import org.apache.kafka.common.errors.NotEnoughReplicasException; -import org.apache.kafka.common.errors.NotLeaderOrFollowerException; +import org.apache.kafka.common.errors.NotLeaderForPartitionException; import org.apache.kafka.common.errors.OffsetMetadataTooLarge; import org.apache.kafka.common.errors.OffsetNotAvailableException; import org.apache.kafka.common.errors.OffsetOutOfRangeException; @@ -139,15 +139,13 @@ public enum Errors { InvalidFetchSizeException::new), LEADER_NOT_AVAILABLE(5, "There is no leader for this topic-partition as we are in the middle of a leadership election.", LeaderNotAvailableException::new), - NOT_LEADER_OR_FOLLOWER(6, "For requests intended only for the leader, this error indicates that the broker is not the current leader. " + - "For requests intended for any replica, this error indicates that the broker is not a replica of the topic partition.", - NotLeaderOrFollowerException::new), + NOT_LEADER_FOR_PARTITION(6, "This server is not the leader for that topic-partition.", + NotLeaderForPartitionException::new), REQUEST_TIMED_OUT(7, "The request timed out.", TimeoutException::new), BROKER_NOT_AVAILABLE(8, "The broker is not available.", BrokerNotAvailableException::new), - REPLICA_NOT_AVAILABLE(9, "The replica is not available for the requested topic-partition. Produce/Fetch requests and other requests " + - "intended only for the leader or follower return NOT_LEADER_OR_FOLLOWER if the broker is not a replica of the topic-partition.", + REPLICA_NOT_AVAILABLE(9, "The replica is not available for the requested topic-partition.", ReplicaNotAvailableException::new), MESSAGE_TOO_LARGE(10, "The request included a message larger than the max message size the server will accept.", RecordTooLargeException::new), diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java index 3e95671a6f939..a3b01bf3c1b2b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java @@ -20,12 +20,10 @@ import org.apache.kafka.common.message.AlterClientQuotasResponseData; import org.apache.kafka.common.message.AlterClientQuotasResponseData.EntityData; import org.apache.kafka.common.message.AlterClientQuotasResponseData.EntryData; -import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.quota.ClientQuotaEntity; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -52,13 +50,14 @@ public AlterClientQuotasResponse(Map result, int th } public AlterClientQuotasResponse(Collection entities, int throttleTimeMs, Throwable e) { - ApiError apiError = ApiError.fromThrowable(e); + short errorCode = Errors.forException(e).code(); + String errorMessage = e.getMessage(); List entries = new ArrayList<>(entities.size()); for (ClientQuotaEntity entity : entities) { entries.add(new EntryData() - .setErrorCode(apiError.error().code()) - .setErrorMessage(apiError.message()) + .setErrorCode(errorCode) + .setErrorMessage(errorMessage) .setEntity(toEntityData(entity))); } @@ -121,8 +120,4 @@ private static List toEntityData(ClientQuotaEntity entity) { } return entityData; } - - public static AlterClientQuotasResponse parse(ByteBuffer buffer, short version) { - return new AlterClientQuotasResponse(ApiKeys.ALTER_CLIENT_QUOTAS.parseResponse(version, buffer), version); - } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiError.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiError.java index 5c9ca7b0fa6e7..6cb09f078ecca 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiError.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiError.java @@ -38,10 +38,9 @@ public class ApiError { private final String message; public static ApiError fromThrowable(Throwable t) { - // Avoid populating the error message if it's a generic one. Also don't populate error - // message for UNKNOWN_SERVER_ERROR to ensure we don't leak sensitive information. + // Avoid populating the error message if it's a generic one Errors error = Errors.forException(t); - String message = error == Errors.UNKNOWN_SERVER_ERROR || error.message().equals(t.getMessage()) ? null : t.getMessage(); + String message = error.message().equals(t.getMessage()) ? null : t.getMessage(); return new ApiError(error, message); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java index 968a35bd32c59..3266bf89a4238 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java @@ -36,7 +36,7 @@ public class DeleteRecordsResponse extends AbstractResponse { * * OFFSET_OUT_OF_RANGE (1) * UNKNOWN_TOPIC_OR_PARTITION (3) - * NOT_LEADER_OR_FOLLOWER (6) + * NOT_LEADER_FOR_PARTITION (6) * REQUEST_TIMED_OUT (7) * UNKNOWN (-1) */ diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java index bda3673c27f2c..cb54b019dba88 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java @@ -21,12 +21,10 @@ import org.apache.kafka.common.message.DescribeClientQuotasResponseData.EntityData; import org.apache.kafka.common.message.DescribeClientQuotasResponseData.EntryData; import org.apache.kafka.common.message.DescribeClientQuotasResponseData.ValueData; -import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.quota.ClientQuotaEntity; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -68,11 +66,10 @@ public DescribeClientQuotasResponse(Map> } public DescribeClientQuotasResponse(int throttleTimeMs, Throwable e) { - ApiError apiError = ApiError.fromThrowable(e); this.data = new DescribeClientQuotasResponseData() .setThrottleTimeMs(throttleTimeMs) - .setErrorCode(apiError.error().code()) - .setErrorMessage(apiError.message()) + .setErrorCode(Errors.forException(e).code()) + .setErrorMessage(e.getMessage()) .setEntries(null); } @@ -118,8 +115,4 @@ public Map errorCounts() { protected Struct toStruct(short version) { return data.toStruct(version); } - - public static DescribeClientQuotasResponse parse(ByteBuffer buffer, short version) { - return new DescribeClientQuotasResponse(ApiKeys.DESCRIBE_CLIENT_QUOTAS.parseResponse(version, buffer), version); - } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java index 29cbb6010b5be..502e32b712da5 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java @@ -146,7 +146,7 @@ public class FetchRequest extends AbstractRequest { TOPICS_V5); // V6 bumped up to indicate that the client supports KafkaStorageException. The KafkaStorageException will be - // translated to NotLeaderOrFollowerException in the response if version <= 5 + // translated to NotLeaderForPartitionException in the response if version <= 5 private static final Schema FETCH_REQUEST_V6 = FETCH_REQUEST_V5; // V7 added incremental fetch requests. diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java index 984c50ca724bd..21edcc5419515 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java @@ -58,8 +58,8 @@ * * - {@link Errors#OFFSET_OUT_OF_RANGE} If the fetch offset is out of range for a requested partition * - {@link Errors#TOPIC_AUTHORIZATION_FAILED} If the user does not have READ access to a requested topic - * - {@link Errors#REPLICA_NOT_AVAILABLE} If the request is received by a broker with version < 2.6 which is not a replica - * - {@link Errors#NOT_LEADER_OR_FOLLOWER} If the broker is not a leader or follower and either the provided leader epoch + * - {@link Errors#REPLICA_NOT_AVAILABLE} If the request is received by a broker which is not a replica + * - {@link Errors#NOT_LEADER_FOR_PARTITION} If the broker is not a leader and either the provided leader epoch * matches the known leader epoch on the broker or is empty * - {@link Errors#FENCED_LEADER_EPOCH} If the epoch is lower than the broker's epoch * - {@link Errors#UNKNOWN_LEADER_EPOCH} If the epoch is larger than the broker's epoch @@ -190,7 +190,7 @@ public class FetchResponse extends AbstractResponse { new Field(RESPONSES_KEY_NAME, new ArrayOf(FETCH_RESPONSE_TOPIC_V5))); // V6 bumped up to indicate that the client supports KafkaStorageException. The KafkaStorageException will - // be translated to NotLeaderOrFollowerException in the response if version <= 5 + // be translated to NotLeaderForPartitionException in the response if version <= 5 private static final Schema FETCH_RESPONSE_V6 = FETCH_RESPONSE_V5; // V7 added incremental fetch responses and a top-level error code. @@ -549,9 +549,9 @@ private static Struct toStruct(short version, int thrott // If consumer sends FetchRequest V5 or earlier, the client library is not guaranteed to recognize the error code // for KafkaStorageException. In this case the client library will translate KafkaStorageException to // UnknownServerException which is not retriable. We can ensure that consumer will update metadata and retry - // by converting the KafkaStorageException to NotLeaderOrFollowerException in the response if FetchRequest version <= 5 + // by converting the KafkaStorageException to NotLeaderForPartitionException in the response if FetchRequest version <= 5 if (errorCode == Errors.KAFKA_STORAGE_ERROR.code() && version <= 5) - errorCode = Errors.NOT_LEADER_OR_FOLLOWER.code(); + errorCode = Errors.NOT_LEADER_FOR_PARTITION.code(); Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME); Struct partitionDataHeader = partitionData.instance(PARTITION_HEADER_KEY_NAME); partitionDataHeader.set(PARTITION_ID, partitionEntry.getKey()); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java index 3fe14b0c27162..63319411d6d9d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java @@ -44,8 +44,8 @@ * * - {@link Errors#UNSUPPORTED_FOR_MESSAGE_FORMAT} If the message format does not support lookup by timestamp * - {@link Errors#TOPIC_AUTHORIZATION_FAILED} If the user does not have DESCRIBE access to a requested topic - * - {@link Errors#REPLICA_NOT_AVAILABLE} If the request is received by a broker with version < 2.6 which is not a replica - * - {@link Errors#NOT_LEADER_OR_FOLLOWER} If the broker is not a leader or follower and either the provided leader epoch + * - {@link Errors#REPLICA_NOT_AVAILABLE} If the request is received by a broker which is not a replica + * - {@link Errors#NOT_LEADER_FOR_PARTITION} If the broker is not a leader and either the provided leader epoch * matches the known leader epoch on the broker or is empty * - {@link Errors#FENCED_LEADER_EPOCH} If the epoch is lower than the broker's epoch * - {@link Errors#UNKNOWN_LEADER_EPOCH} If the epoch is larger than the broker's epoch diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java index 6573aca4e914f..c547c0951a70a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java @@ -41,8 +41,8 @@ * Possible error codes: * * - {@link Errors#TOPIC_AUTHORIZATION_FAILED} If the user does not have DESCRIBE access to a requested topic - * - {@link Errors#REPLICA_NOT_AVAILABLE} If the request is received by a broker with version < 2.6 which is not a replica - * - {@link Errors#NOT_LEADER_OR_FOLLOWER} If the broker is not a leader or follower and either the provided leader epoch + * - {@link Errors#REPLICA_NOT_AVAILABLE} If the request is received by a broker which is not a replica + * - {@link Errors#NOT_LEADER_FOR_PARTITION} If the broker is not a leader and either the provided leader epoch * matches the known leader epoch on the broker or is empty * - {@link Errors#FENCED_LEADER_EPOCH} If the epoch is lower than the broker's epoch * - {@link Errors#UNKNOWN_LEADER_EPOCH} If the epoch is larger than the broker's epoch diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java index 210b71da22158..485145f3ffdac 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java @@ -99,7 +99,7 @@ public class ProduceRequest extends AbstractRequest { /** * The body of PRODUCE_REQUEST_V4 is the same as PRODUCE_REQUEST_V3. * The version number is bumped up to indicate that the client supports KafkaStorageException. - * The KafkaStorageException will be translated to NotLeaderOrFollowerException in the response if version <= 3 + * The KafkaStorageException will be translated to NotLeaderForPartitionException in the response if version <= 3 */ private static final Schema PRODUCE_REQUEST_V4 = PRODUCE_REQUEST_V3; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java index 205d9aba5a683..2bb1e86bb4750 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java @@ -56,7 +56,7 @@ public class ProduceResponse extends AbstractResponse { * * {@link Errors#CORRUPT_MESSAGE} * {@link Errors#UNKNOWN_TOPIC_OR_PARTITION} - * {@link Errors#NOT_LEADER_OR_FOLLOWER} + * {@link Errors#NOT_LEADER_FOR_PARTITION} * {@link Errors#MESSAGE_TOO_LARGE} * {@link Errors#INVALID_TOPIC_EXCEPTION} * {@link Errors#RECORD_LIST_TOO_LARGE} @@ -126,7 +126,7 @@ public class ProduceResponse extends AbstractResponse { /** * The body of PRODUCE_RESPONSE_V4 is the same as PRODUCE_RESPONSE_V3. * The version number is bumped up to indicate that the client supports KafkaStorageException. - * The KafkaStorageException will be translated to NotLeaderOrFollowerException in the response if version <= 3 + * The KafkaStorageException will be translated to NotLeaderForPartitionException in the response if version <= 3 */ private static final Schema PRODUCE_RESPONSE_V4 = PRODUCE_RESPONSE_V3; @@ -265,9 +265,9 @@ protected Struct toStruct(short version) { // If producer sends ProduceRequest V3 or earlier, the client library is not guaranteed to recognize the error code // for KafkaStorageException. In this case the client library will translate KafkaStorageException to // UnknownServerException which is not retriable. We can ensure that producer will update metadata and retry - // by converting the KafkaStorageException to NotLeaderOrFollowerException in the response if ProduceRequest version <= 3 + // by converting the KafkaStorageException to NotLeaderForPartitionException in the response if ProduceRequest version <= 3 if (errorCode == Errors.KAFKA_STORAGE_ERROR.code() && version <= 3) - errorCode = Errors.NOT_LEADER_OR_FOLLOWER.code(); + errorCode = Errors.NOT_LEADER_FOR_PARTITION.code(); Struct partStruct = topicData.instance(PARTITION_RESPONSES_KEY_NAME) .set(PARTITION_ID, partitionEntry.getKey()) .set(ERROR_CODE, errorCode) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java index 5fdfbec66a923..0ed512584d22f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java @@ -77,10 +77,9 @@ public SaslAuthenticateRequestData data() { @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { - ApiError apiError = ApiError.fromThrowable(e); SaslAuthenticateResponseData response = new SaslAuthenticateResponseData() - .setErrorCode(apiError.error().code()) - .setErrorMessage(apiError.message()); + .setErrorCode(ApiError.fromThrowable(e).error().code()) + .setErrorMessage(e.getMessage()); return new SaslAuthenticateResponse(response); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java index 8fdcb99c9333a..783ef3b44e80f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java @@ -37,7 +37,7 @@ * - {@link Errors#CORRUPT_MESSAGE} * - {@link Errors#INVALID_PRODUCER_EPOCH} * - {@link Errors#UNKNOWN_TOPIC_OR_PARTITION} - * - {@link Errors#NOT_LEADER_OR_FOLLOWER} + * - {@link Errors#NOT_LEADER_FOR_PARTITION} * - {@link Errors#MESSAGE_TOO_LARGE} * - {@link Errors#RECORD_LIST_TOO_LARGE} * - {@link Errors#NOT_ENOUGH_REPLICAS} diff --git a/clients/src/main/java/org/apache/kafka/common/security/JaasContext.java b/clients/src/main/java/org/apache/kafka/common/security/JaasContext.java index 48216a8a90c4c..231ee167110b3 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/JaasContext.java +++ b/clients/src/main/java/org/apache/kafka/common/security/JaasContext.java @@ -62,11 +62,12 @@ public static JaasContext loadServerContext(ListenerName listenerName, String me throw new IllegalArgumentException("listenerName should not be null for SERVER"); if (mechanism == null) throw new IllegalArgumentException("mechanism should not be null for SERVER"); + String globalContextName = GLOBAL_CONTEXT_NAME_SERVER; String listenerContextName = listenerName.value().toLowerCase(Locale.ROOT) + "." + GLOBAL_CONTEXT_NAME_SERVER; Password dynamicJaasConfig = (Password) configs.get(mechanism.toLowerCase(Locale.ROOT) + "." + SaslConfigs.SASL_JAAS_CONFIG); if (dynamicJaasConfig == null && configs.get(SaslConfigs.SASL_JAAS_CONFIG) != null) LOG.warn("Server config {} should be prefixed with SASL mechanism name, ignoring config", SaslConfigs.SASL_JAAS_CONFIG); - return load(Type.SERVER, listenerContextName, GLOBAL_CONTEXT_NAME_SERVER, dynamicJaasConfig); + return load(Type.SERVER, listenerContextName, globalContextName, dynamicJaasConfig); } /** @@ -78,8 +79,9 @@ public static JaasContext loadServerContext(ListenerName listenerName, String me * */ public static JaasContext loadClientContext(Map configs) { + String globalContextName = GLOBAL_CONTEXT_NAME_CLIENT; Password dynamicJaasConfig = (Password) configs.get(SaslConfigs.SASL_JAAS_CONFIG); - return load(JaasContext.Type.CLIENT, null, GLOBAL_CONTEXT_NAME_CLIENT, dynamicJaasConfig); + return load(JaasContext.Type.CLIENT, null, globalContextName, dynamicJaasConfig); } static JaasContext load(JaasContext.Type contextType, String listenerContextName, diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java index 6d279ac9f4f2b..8b32e81451760 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java @@ -672,7 +672,7 @@ public void setAuthenticationEndAndSessionReauthenticationTimes(long nowNanos) { double pctWindowJitterToAvoidReauthenticationStormAcrossManyChannelsSimultaneously = 0.10; double pctToUse = pctWindowFactorToTakeNetworkLatencyAndClockDriftIntoAccount + RNG.nextDouble() * pctWindowJitterToAvoidReauthenticationStormAcrossManyChannelsSimultaneously; - sessionLifetimeMsToUse = (long) (positiveSessionLifetimeMs * pctToUse); + sessionLifetimeMsToUse = (long) (positiveSessionLifetimeMs.longValue() * pctToUse); clientSessionReauthenticationTimeNanos = authenticationEndNanos + 1000 * 1000 * sessionLifetimeMsToUse; log.debug( "Finished {} with session expiration in {} ms and session re-authentication on or after {} ms", @@ -684,7 +684,7 @@ public void setAuthenticationEndAndSessionReauthenticationTimes(long nowNanos) { public Long reauthenticationLatencyMs() { return reauthenticating() - ? Math.round((authenticationEndNanos - reauthenticationBeginNanos) / 1000.0 / 1000.0) + ? Long.valueOf(Math.round((authenticationEndNanos - reauthenticationBeginNanos) / 1000.0 / 1000.0)) : null; } diff --git a/clients/src/main/java/org/apache/kafka/common/security/ssl/DefaultSslEngineFactory.java b/clients/src/main/java/org/apache/kafka/common/security/ssl/DefaultSslEngineFactory.java index 02294e5b403ef..f71adaf62a82a 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/ssl/DefaultSslEngineFactory.java +++ b/clients/src/main/java/org/apache/kafka/common/security/ssl/DefaultSslEngineFactory.java @@ -115,14 +115,14 @@ public void configure(Map configs) { List cipherSuitesList = (List) configs.get(SslConfigs.SSL_CIPHER_SUITES_CONFIG); if (cipherSuitesList != null && !cipherSuitesList.isEmpty()) { - this.cipherSuites = cipherSuitesList.toArray(new String[0]); + this.cipherSuites = cipherSuitesList.toArray(new String[cipherSuitesList.size()]); } else { this.cipherSuites = null; } List enabledProtocolsList = (List) configs.get(SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG); if (enabledProtocolsList != null && !enabledProtocolsList.isEmpty()) { - this.enabledProtocols = enabledProtocolsList.toArray(new String[0]); + this.enabledProtocols = enabledProtocolsList.toArray(new String[enabledProtocolsList.size()]); } else { this.enabledProtocols = null; } @@ -149,7 +149,7 @@ public void configure(Map configs) { } @Override - public void close() { + public void close() throws IOException { this.sslContext = null; } @@ -194,7 +194,7 @@ private static SslClientAuth createSslClientAuth(String key) { log.warn("Unrecognized client authentication configuration {}. Falling " + "back to NONE. Recognized client authentication configurations are {}.", key, String.join(", ", SslClientAuth.VALUES.stream(). - map(Enum::name).collect(Collectors.toList()))); + map(a -> a.name()).collect(Collectors.toList()))); return SslClientAuth.NONE; } diff --git a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslPrincipalMapper.java b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslPrincipalMapper.java index 33da9642d97b2..977fdc332cbfa 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslPrincipalMapper.java +++ b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslPrincipalMapper.java @@ -163,7 +163,8 @@ private String escapeLiteralBackReferences(final String unescaped, final int num if (backRefNum.startsWith("0")) { continue; } - int backRefIndex = Integer.parseInt(backRefNum); + final int originalBackRefIndex = Integer.parseInt(backRefNum); + int backRefIndex = originalBackRefIndex; // if we have a replacement value like $123, and we have less than 123 capturing groups, then diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index a1fa30766a462..f83624501a213 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -54,7 +54,7 @@ import org.apache.kafka.common.errors.LeaderNotAvailableException; import org.apache.kafka.common.errors.LogDirNotFoundException; import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.errors.NotLeaderOrFollowerException; +import org.apache.kafka.common.errors.NotLeaderForPartitionException; import org.apache.kafka.common.errors.OffsetOutOfRangeException; import org.apache.kafka.common.errors.SaslAuthenticationException; import org.apache.kafka.common.errors.SecurityDisabledException; @@ -1201,7 +1201,7 @@ public void testDeleteRecords() throws Exception { new DeleteRecordsResponseData.DeleteRecordsPartitionResult() .setPartitionIndex(myTopicPartition3.partition()) .setLowWatermark(DeleteRecordsResponse.INVALID_LOW_WATERMARK) - .setErrorCode(Errors.NOT_LEADER_OR_FOLLOWER.code()), + .setErrorCode(Errors.NOT_LEADER_FOR_PARTITION.code()), new DeleteRecordsResponseData.DeleteRecordsPartitionResult() .setPartitionIndex(myTopicPartition4.partition()) .setLowWatermark(DeleteRecordsResponse.INVALID_LOW_WATERMARK) @@ -1270,7 +1270,7 @@ public void testDeleteRecords() throws Exception { myTopicPartition3Result.get(); fail("get() should throw ExecutionException"); } catch (ExecutionException e1) { - assertTrue(e1.getCause() instanceof NotLeaderOrFollowerException); + assertTrue(e1.getCause() instanceof NotLeaderForPartitionException); } // "unknown topic or partition" failure on records deletion for partition 4 @@ -3270,7 +3270,7 @@ public void testListOffsetsWithMultiplePartitionsLeaderChange() throws Exception env.kafkaClient().prepareResponse(prepareMetadataResponse(oldCluster, Errors.NONE)); Map responseData = new HashMap<>(); - responseData.put(tp0, new PartitionData(Errors.NOT_LEADER_OR_FOLLOWER, -1L, 345L, Optional.of(543))); + responseData.put(tp0, new PartitionData(Errors.NOT_LEADER_FOR_PARTITION, -1L, 345L, Optional.of(543))); responseData.put(tp1, new PartitionData(Errors.LEADER_NOT_AVAILABLE, -2L, 123L, Optional.of(456))); env.kafkaClient().prepareResponseFrom(new ListOffsetResponse(responseData), node0); @@ -3328,10 +3328,10 @@ public void testListOffsetsWithLeaderChange() throws Exception { env.kafkaClient().prepareResponse(prepareMetadataResponse(oldCluster, Errors.NONE)); Map responseData = new HashMap<>(); - responseData.put(tp0, new PartitionData(Errors.NOT_LEADER_OR_FOLLOWER, -1L, 345L, Optional.of(543))); + responseData.put(tp0, new PartitionData(Errors.NOT_LEADER_FOR_PARTITION, -1L, 345L, Optional.of(543))); env.kafkaClient().prepareResponseFrom(new ListOffsetResponse(responseData), node0); - // updating leader from node0 to node1 and metadata refresh because of NOT_LEADER_OR_FOLLOWER + // updating leader from node0 to node1 and metadata refresh because of NOT_LEADER_FOR_PARTITION final PartitionInfo newPartitionInfo = new PartitionInfo("foo", 0, node1, new Node[]{node0, node1, node2}, new Node[]{node0, node1, node2}); final Cluster newCluster = new Cluster("mockClusterId", nodes, singletonList(newPartitionInfo), diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index 9fc54bf7548f0..0abcdd00e424e 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -591,7 +591,7 @@ public void testFetchProgressWithMissingPartitionPosition() { return timestamps.get(tp0).timestamp == ListOffsetRequest.LATEST_TIMESTAMP && timestamps.get(tp1).timestamp == ListOffsetRequest.EARLIEST_TIMESTAMP; }, listOffsetsResponse(Collections.singletonMap(tp0, 50L), - Collections.singletonMap(tp1, Errors.NOT_LEADER_OR_FOLLOWER))); + Collections.singletonMap(tp1, Errors.NOT_LEADER_FOR_PARTITION))); client.prepareResponse( body -> { FetchRequest request = (FetchRequest) body; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index edc7c8548652e..ece5781645f81 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -406,7 +406,7 @@ public void testFetchError() { assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NOT_LEADER_FOR_PARTITION, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -1097,13 +1097,13 @@ public void testFetchDiscardedAfterPausedPartitionResumedAndSeekedToNewOffset() } @Test - public void testFetchNotLeaderOrFollower() { + public void testFetchNotLeaderForPartition() { buildFetcher(); assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); + client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NOT_LEADER_FOR_PARTITION, 100L, 0)); consumerClient.poll(time.timer(0)); assertEquals(0, fetcher.fetchedRecords().size()); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); @@ -1579,7 +1579,7 @@ public void testResetOffsetsMetadataRefresh() { // First fetch fails with stale metadata client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP, - Optional.of(validLeaderEpoch)), listOffsetResponse(Errors.NOT_LEADER_OR_FOLLOWER, 1L, 5L), false); + Optional.of(validLeaderEpoch)), listOffsetResponse(Errors.NOT_LEADER_FOR_PARTITION, 1L, 5L), false); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.hasValidPosition(tp0)); @@ -2445,11 +2445,11 @@ public void testGetOffsetsForTimes() { // Error code none with known offset testGetOffsetsForTimesWithError(Errors.NONE, Errors.NONE, 10L, 100L, 10L, 100L); // Test both of partition has error. - testGetOffsetsForTimesWithError(Errors.NOT_LEADER_OR_FOLLOWER, Errors.INVALID_REQUEST, 10L, 100L, 10L, 100L); + testGetOffsetsForTimesWithError(Errors.NOT_LEADER_FOR_PARTITION, Errors.INVALID_REQUEST, 10L, 100L, 10L, 100L); // Test the second partition has error. - testGetOffsetsForTimesWithError(Errors.NONE, Errors.NOT_LEADER_OR_FOLLOWER, 10L, 100L, 10L, 100L); + testGetOffsetsForTimesWithError(Errors.NONE, Errors.NOT_LEADER_FOR_PARTITION, 10L, 100L, 10L, 100L); // Test different errors. - testGetOffsetsForTimesWithError(Errors.NOT_LEADER_OR_FOLLOWER, Errors.NONE, 10L, 100L, 10L, 100L); + testGetOffsetsForTimesWithError(Errors.NOT_LEADER_FOR_PARTITION, Errors.NONE, 10L, 100L, 10L, 100L); testGetOffsetsForTimesWithError(Errors.UNKNOWN_TOPIC_OR_PARTITION, Errors.NONE, 10L, 100L, 10L, 100L); testGetOffsetsForTimesWithError(Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, Errors.NONE, 10L, 100L, null, 100L); testGetOffsetsForTimesWithError(Errors.BROKER_NOT_AVAILABLE, Errors.NONE, 10L, 100L, 10L, 100L); @@ -2475,7 +2475,7 @@ public void testGetOffsetsFencedLeaderEpoch() { @Test public void testGetOffsetByTimeWithPartitionsRetryCouldTriggerMetadataUpdate() { - List retriableErrors = Arrays.asList(Errors.NOT_LEADER_OR_FOLLOWER, + List retriableErrors = Arrays.asList(Errors.NOT_LEADER_FOR_PARTITION, Errors.REPLICA_NOT_AVAILABLE, Errors.KAFKA_STORAGE_ERROR, Errors.OFFSET_NOT_AVAILABLE, Errors.LEADER_NOT_AVAILABLE, Errors.FENCED_LEADER_EPOCH, Errors.UNKNOWN_LEADER_EPOCH); @@ -2522,7 +2522,7 @@ public void testGetOffsetByTimeWithPartitionsRetryCouldTriggerMetadataUpdate() { // We will count the answered future response in the end to verify if this is the case. Map paritionDataWithFatalError = new HashMap<>(allPartitionData); paritionDataWithFatalError.put(tp1, new ListOffsetResponse.PartitionData( - Errors.NOT_LEADER_OR_FOLLOWER, ListOffsetRequest.LATEST_TIMESTAMP, -1L, Optional.empty())); + Errors.NOT_LEADER_FOR_PARTITION, ListOffsetRequest.LATEST_TIMESTAMP, -1L, Optional.empty())); client.prepareResponseFrom(new ListOffsetResponse(paritionDataWithFatalError), originalLeader); // The request to new leader must only contain one partition tp1 with error. @@ -2691,7 +2691,7 @@ public void testBatchedListOffsetsMetadataErrors() { buildFetcher(); Map partitionData = new HashMap<>(); - partitionData.put(tp0, new ListOffsetResponse.PartitionData(Errors.NOT_LEADER_OR_FOLLOWER, + partitionData.put(tp0, new ListOffsetResponse.PartitionData(Errors.NOT_LEADER_FOR_PARTITION, ListOffsetResponse.UNKNOWN_TIMESTAMP, ListOffsetResponse.UNKNOWN_OFFSET, Optional.empty())); partitionData.put(tp1, new ListOffsetResponse.PartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION, diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index a55d6d6849cbc..cada9655e4ebf 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -870,7 +870,7 @@ public void testCorrectHandlingOfOutOfOrderResponses() throws Exception { assertEquals(1, client.inFlightRequestCount()); assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0)); - client.respondToRequest(firstClientRequest, produceResponse(tp0, -1, Errors.NOT_LEADER_OR_FOLLOWER, -1)); + client.respondToRequest(firstClientRequest, produceResponse(tp0, -1, Errors.NOT_LEADER_FOR_PARTITION, -1)); sender.runOnce(); // receive response 0 @@ -1086,7 +1086,7 @@ public void testExpiryOfFirstBatchShouldCauseEpochBumpIfFutureBatchesFail() thro assertEquals(2, client.inFlightRequestCount()); - sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_OR_FOLLOWER, -1); + sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_FOR_PARTITION, -1); sender.runOnce(); // receive first response Node node = metadata.fetch().nodes().get(0); @@ -1142,7 +1142,7 @@ public void testUnresolvedSequencesAreNotFatal() throws Exception { assertEquals(2, client.inFlightRequestCount()); - sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_OR_FOLLOWER, -1); + sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_FOR_PARTITION, -1); sender.runOnce(); // receive first response Node node = metadata.fetch().nodes().get(0); @@ -1172,7 +1172,7 @@ public void testExpiryOfAllSentBatchesShouldCauseUnresolvedSequences() throws Ex // Send first ProduceRequest Future request1 = appendToAccumulator(tp0, 0L, "key", "value"); sender.runOnce(); // send request - sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_OR_FOLLOWER, -1); + sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_FOR_PARTITION, -1); sender.runOnce(); // receive response assertEquals(1L, transactionManager.sequenceNumber(tp0).longValue()); @@ -1219,7 +1219,7 @@ public void testResetOfProducerStateShouldAllowQueuedBatchesToDrain() throws Exc assertEquals(1, client.inFlightRequestCount()); Map responses = new LinkedHashMap<>(); - responses.put(tp1, new OffsetAndError(-1, Errors.NOT_LEADER_OR_FOLLOWER)); + responses.put(tp1, new OffsetAndError(-1, Errors.NOT_LEADER_FOR_PARTITION)); responses.put(tp0, new OffsetAndError(-1, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER)); client.respond(produceResponse(responses)); @@ -1260,7 +1260,7 @@ public void testCloseWithProducerIdReset() throws Exception { assertEquals(1, client.inFlightRequestCount()); Map responses = new LinkedHashMap<>(); - responses.put(tp1, new OffsetAndError(-1, Errors.NOT_LEADER_OR_FOLLOWER)); + responses.put(tp1, new OffsetAndError(-1, Errors.NOT_LEADER_FOR_PARTITION)); responses.put(tp0, new OffsetAndError(-1, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER)); client.respond(produceResponse(responses)); sender.initiateClose(); // initiate close @@ -1293,7 +1293,7 @@ public void testForceCloseWithProducerIdReset() throws Exception { assertEquals(1, client.inFlightRequestCount()); Map responses = new LinkedHashMap<>(); - responses.put(tp1, new OffsetAndError(-1, Errors.NOT_LEADER_OR_FOLLOWER)); + responses.put(tp1, new OffsetAndError(-1, Errors.NOT_LEADER_FOR_PARTITION)); responses.put(tp0, new OffsetAndError(-1, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER)); client.respond(produceResponse(responses)); sender.runOnce(); // out of order sequence error triggers producer ID reset because epoch is maxed out @@ -1327,7 +1327,7 @@ public void testBatchesDrainedWithOldProducerIdShouldSucceedOnSubsequentRetry() assertEquals(1, client.inFlightRequestCount()); Map responses = new LinkedHashMap<>(); - responses.put(tp1, new OffsetAndError(-1, Errors.NOT_LEADER_OR_FOLLOWER)); + responses.put(tp1, new OffsetAndError(-1, Errors.NOT_LEADER_FOR_PARTITION)); responses.put(tp0, new OffsetAndError(-1, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER)); client.respond(produceResponse(responses)); sender.runOnce(); @@ -1338,7 +1338,7 @@ public void testBatchesDrainedWithOldProducerIdShouldSucceedOnSubsequentRetry() assertFalse(successfulResponse.isDone()); // The response comes back with a retriable error. - client.respond(produceResponse(tp1, 0, Errors.NOT_LEADER_OR_FOLLOWER, -1)); + client.respond(produceResponse(tp1, 0, Errors.NOT_LEADER_FOR_PARTITION, -1)); sender.runOnce(); // The response @@ -2149,7 +2149,7 @@ public void testExpiredBatchDoesNotRetry() throws Exception { assertEquals(1, client.inFlightRequestCount()); time.sleep(deliverTimeoutMs); - client.respond(produceResponse(tp0, -1, Errors.NOT_LEADER_OR_FOLLOWER, -1)); // return a retriable error + client.respond(produceResponse(tp0, -1, Errors.NOT_LEADER_FOR_PARTITION, -1)); // return a retriable error sender.runOnce(); // expire the batch assertTrue(request1.isDone()); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java index 1d146427258e8..126d4473dbd5d 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java @@ -2192,7 +2192,7 @@ public void resendFailedProduceRequestAfterAbortableError() throws Exception { Future responseFuture = appendToAccumulator(tp0); prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, producerId); - prepareProduceResponse(Errors.NOT_LEADER_OR_FOLLOWER, producerId, epoch); + prepareProduceResponse(Errors.NOT_LEADER_FOR_PARTITION, producerId, epoch); runUntil(() -> !client.hasPendingResponses()); assertFalse(responseFuture.isDone()); @@ -2385,7 +2385,7 @@ public void testTransitionToFatalErrorWhenRetriedBatchIsExpired() throws Interru runUntil(() -> transactionManager.transactionContainsPartition(tp0)); assertTrue(transactionManager.isSendToPartitionAllowed(tp0)); - prepareProduceResponse(Errors.NOT_LEADER_OR_FOLLOWER, producerId, epoch); + prepareProduceResponse(Errors.NOT_LEADER_FOR_PARTITION, producerId, epoch); runUntil(() -> !client.hasPendingResponses()); assertFalse(responseFuture.isDone()); @@ -2920,10 +2920,10 @@ MAX_REQUEST_SIZE, ACKS_ALL, MAX_RETRIES, new SenderMetricsRegistry(new Metrics(t sender.runOnce(); assertEquals(1, accumulator.batches().get(tp1).size()); - // Partition failover occurs and tp1 returns a NOT_LEADER_OR_FOLLOWER error + // Partition failover occurs and tp1 returns a NOT_LEADER_FOR_PARTITION error // Despite having the old epoch, the batch should retry ProduceResponse.PartitionResponse t1b2Response = new ProduceResponse.PartitionResponse( - Errors.NOT_LEADER_OR_FOLLOWER, -1, -1, 600L); + Errors.NOT_LEADER_FOR_PARTITION, -1, -1, 600L); assertTrue(transactionManager.canRetry(t1b2Response, tp1b2)); accumulator.reenqueue(tp1b2, time.milliseconds()); @@ -3042,10 +3042,10 @@ MAX_REQUEST_SIZE, ACKS_ALL, MAX_RETRIES, new SenderMetricsRegistry(new Metrics(t sender.runOnce(); assertEquals(1, accumulator.batches().get(tp1).size()); - // Partition failover occurs and tp1 returns a NOT_LEADER_OR_FOLLOWER error + // Partition failover occurs and tp1 returns a NOT_LEADER_FOR_PARTITION error // Despite having the old epoch, the batch should retry ProduceResponse.PartitionResponse t1b2Response = new ProduceResponse.PartitionResponse( - Errors.NOT_LEADER_OR_FOLLOWER, -1, -1, 600L); + Errors.NOT_LEADER_FOR_PARTITION, -1, -1, 600L); assertTrue(transactionManager.canRetry(t1b2Response, tp1b2)); accumulator.reenqueue(tp1b2, time.milliseconds()); diff --git a/clients/src/test/java/org/apache/kafka/common/memory/GarbageCollectedMemoryPoolTest.java b/clients/src/test/java/org/apache/kafka/common/memory/GarbageCollectedMemoryPoolTest.java index 02b38564e30a3..788d447551125 100644 --- a/clients/src/test/java/org/apache/kafka/common/memory/GarbageCollectedMemoryPoolTest.java +++ b/clients/src/test/java/org/apache/kafka/common/memory/GarbageCollectedMemoryPoolTest.java @@ -80,7 +80,6 @@ public void testReleaseForeignBuffer() throws Exception { GarbageCollectedMemoryPool pool = new GarbageCollectedMemoryPool(1000, 10, true, null); ByteBuffer fellOffATruck = ByteBuffer.allocate(1); pool.release(fellOffATruck); - pool.close(); } @Test diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java index 5f4f27b323fd2..ac94817dc8ffb 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java @@ -903,7 +903,6 @@ SecurityProtocol.SSL, new TestSecurityConfig(sslServerConfigs), NetworkTestUtils.waitForChannelClose(selector, node, ChannelState.State.AUTHENTICATION_FAILED); server.close(); selector.close(); - serverChannelBuilder.close(); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java index bf2fe13fb4ed6..edb962b2c5805 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java @@ -66,7 +66,7 @@ public void testErrorCountsFromGetErrorResponse() { @Test public void testErrorCountsWithTopLevelError() { List partitions = createPartitions("foo", - asList(Errors.NONE, Errors.NOT_LEADER_OR_FOLLOWER)); + asList(Errors.NONE, Errors.NOT_LEADER_FOR_PARTITION)); LeaderAndIsrResponse response = new LeaderAndIsrResponse(new LeaderAndIsrResponseData() .setErrorCode(Errors.UNKNOWN_SERVER_ERROR.code()) .setPartitionErrors(partitions)); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 723b67118c96a..ad7a1502d8825 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -147,9 +147,6 @@ import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.SchemaException; import org.apache.kafka.common.protocol.types.Struct; -import org.apache.kafka.common.quota.ClientQuotaAlteration; -import org.apache.kafka.common.quota.ClientQuotaEntity; -import org.apache.kafka.common.quota.ClientQuotaFilter; import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.RecordBatch; @@ -200,21 +197,18 @@ public class RequestResponseTest { - // Exception includes a message that we verify is not included in error responses - private final UnknownServerException unknownServerException = new UnknownServerException("secret"); - @Test public void testSerialization() throws Exception { checkRequest(createFindCoordinatorRequest(0), true); checkRequest(createFindCoordinatorRequest(1), true); - checkErrorResponse(createFindCoordinatorRequest(0), unknownServerException, true); - checkErrorResponse(createFindCoordinatorRequest(1), unknownServerException, true); + checkErrorResponse(createFindCoordinatorRequest(0), new UnknownServerException(), true); + checkErrorResponse(createFindCoordinatorRequest(1), new UnknownServerException(), true); checkResponse(createFindCoordinatorResponse(), 0, true); checkResponse(createFindCoordinatorResponse(), 1, true); checkRequest(createControlledShutdownRequest(), true); checkResponse(createControlledShutdownResponse(), 1, true); - checkErrorResponse(createControlledShutdownRequest(), unknownServerException, true); - checkErrorResponse(createControlledShutdownRequest(0), unknownServerException, true); + checkErrorResponse(createControlledShutdownRequest(), new UnknownServerException(), true); + checkErrorResponse(createControlledShutdownRequest(0), new UnknownServerException(), true); checkRequest(createFetchRequest(4), true); checkResponse(createFetchResponse(), 4, true); List toForgetTopics = new ArrayList<>(); @@ -224,53 +218,53 @@ public void testSerialization() throws Exception { checkRequest(createFetchRequest(7, new FetchMetadata(123, 456), toForgetTopics), true); checkResponse(createFetchResponse(123), 7, true); checkResponse(createFetchResponse(Errors.FETCH_SESSION_ID_NOT_FOUND, 123), 7, true); - checkErrorResponse(createFetchRequest(4), unknownServerException, true); + checkErrorResponse(createFetchRequest(4), new UnknownServerException(), true); checkRequest(createHeartBeatRequest(), true); - checkErrorResponse(createHeartBeatRequest(), unknownServerException, true); + checkErrorResponse(createHeartBeatRequest(), new UnknownServerException(), true); checkResponse(createHeartBeatResponse(), 0, true); for (int v = ApiKeys.JOIN_GROUP.oldestVersion(); v <= ApiKeys.JOIN_GROUP.latestVersion(); v++) { checkRequest(createJoinGroupRequest(v), true); - checkErrorResponse(createJoinGroupRequest(v), unknownServerException, true); + checkErrorResponse(createJoinGroupRequest(v), new UnknownServerException(), true); checkResponse(createJoinGroupResponse(v), v, true); } for (int v = ApiKeys.SYNC_GROUP.oldestVersion(); v <= ApiKeys.SYNC_GROUP.latestVersion(); v++) { checkRequest(createSyncGroupRequest(v), true); - checkErrorResponse(createSyncGroupRequest(v), unknownServerException, true); + checkErrorResponse(createSyncGroupRequest(v), new UnknownServerException(), true); checkResponse(createSyncGroupResponse(v), v, true); } checkRequest(createLeaveGroupRequest(), true); - checkErrorResponse(createLeaveGroupRequest(), unknownServerException, true); + checkErrorResponse(createLeaveGroupRequest(), new UnknownServerException(), true); checkResponse(createLeaveGroupResponse(), 0, true); for (short v = ApiKeys.LIST_GROUPS.oldestVersion(); v <= ApiKeys.LIST_GROUPS.latestVersion(); v++) { checkRequest(createListGroupsRequest(v), false); - checkErrorResponse(createListGroupsRequest(v), unknownServerException, true); + checkErrorResponse(createListGroupsRequest(v), new UnknownServerException(), true); checkResponse(createListGroupsResponse(v), v, true); } checkRequest(createDescribeGroupRequest(), true); - checkErrorResponse(createDescribeGroupRequest(), unknownServerException, true); + checkErrorResponse(createDescribeGroupRequest(), new UnknownServerException(), true); checkResponse(createDescribeGroupResponse(), 0, true); checkRequest(createDeleteGroupsRequest(), true); - checkErrorResponse(createDeleteGroupsRequest(), unknownServerException, true); + checkErrorResponse(createDeleteGroupsRequest(), new UnknownServerException(), true); checkResponse(createDeleteGroupsResponse(), 0, true); for (int i = 0; i < ApiKeys.LIST_OFFSETS.latestVersion(); i++) { checkRequest(createListOffsetRequest(i), true); - checkErrorResponse(createListOffsetRequest(i), unknownServerException, true); + checkErrorResponse(createListOffsetRequest(i), new UnknownServerException(), true); checkResponse(createListOffsetResponse(i), i, true); } checkRequest(MetadataRequest.Builder.allTopics().build((short) 2), true); checkRequest(createMetadataRequest(1, Collections.singletonList("topic1")), true); - checkErrorResponse(createMetadataRequest(1, Collections.singletonList("topic1")), unknownServerException, true); + checkErrorResponse(createMetadataRequest(1, Collections.singletonList("topic1")), new UnknownServerException(), true); checkResponse(createMetadataResponse(), 2, true); - checkErrorResponse(createMetadataRequest(2, Collections.singletonList("topic1")), unknownServerException, true); + checkErrorResponse(createMetadataRequest(2, Collections.singletonList("topic1")), new UnknownServerException(), true); checkResponse(createMetadataResponse(), 3, true); - checkErrorResponse(createMetadataRequest(3, Collections.singletonList("topic1")), unknownServerException, true); + checkErrorResponse(createMetadataRequest(3, Collections.singletonList("topic1")), new UnknownServerException(), true); checkResponse(createMetadataResponse(), 4, true); - checkErrorResponse(createMetadataRequest(4, Collections.singletonList("topic1")), unknownServerException, true); + checkErrorResponse(createMetadataRequest(4, Collections.singletonList("topic1")), new UnknownServerException(), true); checkRequest(createOffsetFetchRequestForAllPartition("group1", false), true); checkRequest(createOffsetFetchRequestForAllPartition("group1", true), true); checkErrorResponse(createOffsetFetchRequestForAllPartition("group1", false), new NotCoordinatorException("Not Coordinator"), true); @@ -281,42 +275,42 @@ public void testSerialization() throws Exception { checkRequest(createOffsetFetchRequest(7, true), true); checkRequest(createOffsetFetchRequestForAllPartition("group1", false), true); checkRequest(createOffsetFetchRequestForAllPartition("group1", true), true); - checkErrorResponse(createOffsetFetchRequest(0, false), unknownServerException, true); - checkErrorResponse(createOffsetFetchRequest(1, false), unknownServerException, true); - checkErrorResponse(createOffsetFetchRequest(2, false), unknownServerException, true); - checkErrorResponse(createOffsetFetchRequest(7, true), unknownServerException, true); + checkErrorResponse(createOffsetFetchRequest(0, false), new UnknownServerException(), true); + checkErrorResponse(createOffsetFetchRequest(1, false), new UnknownServerException(), true); + checkErrorResponse(createOffsetFetchRequest(2, false), new UnknownServerException(), true); + checkErrorResponse(createOffsetFetchRequest(7, true), new UnknownServerException(), true); checkResponse(createOffsetFetchResponse(), 0, true); checkRequest(createProduceRequest(2), true); - checkErrorResponse(createProduceRequest(2), unknownServerException, true); + checkErrorResponse(createProduceRequest(2), new UnknownServerException(), true); checkRequest(createProduceRequest(3), true); - checkErrorResponse(createProduceRequest(3), unknownServerException, true); + checkErrorResponse(createProduceRequest(3), new UnknownServerException(), true); checkResponse(createProduceResponse(), 2, true); checkResponse(createProduceResponseWithErrorMessage(), 8, true); for (int v = ApiKeys.STOP_REPLICA.oldestVersion(); v <= ApiKeys.STOP_REPLICA.latestVersion(); v++) { checkRequest(createStopReplicaRequest(v, true), true); checkRequest(createStopReplicaRequest(v, false), true); - checkErrorResponse(createStopReplicaRequest(v, true), unknownServerException, true); - checkErrorResponse(createStopReplicaRequest(v, false), unknownServerException, true); + checkErrorResponse(createStopReplicaRequest(v, true), new UnknownServerException(), true); + checkErrorResponse(createStopReplicaRequest(v, false), new UnknownServerException(), true); checkResponse(createStopReplicaResponse(), v, true); } checkRequest(createLeaderAndIsrRequest(0), true); - checkErrorResponse(createLeaderAndIsrRequest(0), unknownServerException, false); + checkErrorResponse(createLeaderAndIsrRequest(0), new UnknownServerException(), false); checkRequest(createLeaderAndIsrRequest(1), true); - checkErrorResponse(createLeaderAndIsrRequest(1), unknownServerException, false); + checkErrorResponse(createLeaderAndIsrRequest(1), new UnknownServerException(), false); checkRequest(createLeaderAndIsrRequest(2), true); - checkErrorResponse(createLeaderAndIsrRequest(2), unknownServerException, false); + checkErrorResponse(createLeaderAndIsrRequest(2), new UnknownServerException(), false); checkResponse(createLeaderAndIsrResponse(), 0, true); checkRequest(createSaslHandshakeRequest(), true); - checkErrorResponse(createSaslHandshakeRequest(), unknownServerException, true); + checkErrorResponse(createSaslHandshakeRequest(), new UnknownServerException(), true); checkResponse(createSaslHandshakeResponse(), 0, true); checkRequest(createSaslAuthenticateRequest(), true); - checkErrorResponse(createSaslAuthenticateRequest(), unknownServerException, true); + checkErrorResponse(createSaslAuthenticateRequest(), new UnknownServerException(), true); checkResponse(createSaslAuthenticateResponse(), 0, true); checkResponse(createSaslAuthenticateResponse(), 1, true); checkRequest(createApiVersionRequest(), true); - checkErrorResponse(createApiVersionRequest(), unknownServerException, true); + checkErrorResponse(createApiVersionRequest(), new UnknownServerException(), true); checkErrorResponse(createApiVersionRequest(), new UnsupportedVersionException("Not Supported"), true); checkResponse(createApiVersionResponse(), 0, true); checkResponse(createApiVersionResponse(), 1, true); @@ -328,107 +322,107 @@ public void testSerialization() throws Exception { checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 3, true); checkRequest(createCreateTopicRequest(0), true); - checkErrorResponse(createCreateTopicRequest(0), unknownServerException, true); + checkErrorResponse(createCreateTopicRequest(0), new UnknownServerException(), true); checkResponse(createCreateTopicResponse(), 0, true); checkRequest(createCreateTopicRequest(1), true); - checkErrorResponse(createCreateTopicRequest(1), unknownServerException, true); + checkErrorResponse(createCreateTopicRequest(1), new UnknownServerException(), true); checkResponse(createCreateTopicResponse(), 1, true); checkRequest(createCreateTopicRequest(2), true); - checkErrorResponse(createCreateTopicRequest(2), unknownServerException, true); + checkErrorResponse(createCreateTopicRequest(2), new UnknownServerException(), true); checkResponse(createCreateTopicResponse(), 2, true); checkRequest(createCreateTopicRequest(3), true); - checkErrorResponse(createCreateTopicRequest(3), unknownServerException, true); + checkErrorResponse(createCreateTopicRequest(3), new UnknownServerException(), true); checkResponse(createCreateTopicResponse(), 3, true); checkRequest(createCreateTopicRequest(4), true); - checkErrorResponse(createCreateTopicRequest(4), unknownServerException, true); + checkErrorResponse(createCreateTopicRequest(4), new UnknownServerException(), true); checkResponse(createCreateTopicResponse(), 4, true); checkRequest(createCreateTopicRequest(5), true); - checkErrorResponse(createCreateTopicRequest(5), unknownServerException, true); + checkErrorResponse(createCreateTopicRequest(5), new UnknownServerException(), true); checkResponse(createCreateTopicResponse(), 5, true); checkRequest(createDeleteTopicsRequest(), true); - checkErrorResponse(createDeleteTopicsRequest(), unknownServerException, true); + checkErrorResponse(createDeleteTopicsRequest(), new UnknownServerException(), true); checkResponse(createDeleteTopicsResponse(), 0, true); checkRequest(createInitPidRequest(), true); - checkErrorResponse(createInitPidRequest(), unknownServerException, true); + checkErrorResponse(createInitPidRequest(), new UnknownServerException(), true); checkResponse(createInitPidResponse(), 0, true); checkRequest(createAddPartitionsToTxnRequest(), true); checkResponse(createAddPartitionsToTxnResponse(), 0, true); - checkErrorResponse(createAddPartitionsToTxnRequest(), unknownServerException, true); + checkErrorResponse(createAddPartitionsToTxnRequest(), new UnknownServerException(), true); checkRequest(createAddOffsetsToTxnRequest(), true); checkResponse(createAddOffsetsToTxnResponse(), 0, true); - checkErrorResponse(createAddOffsetsToTxnRequest(), unknownServerException, true); + checkErrorResponse(createAddOffsetsToTxnRequest(), new UnknownServerException(), true); checkRequest(createEndTxnRequest(), true); checkResponse(createEndTxnResponse(), 0, true); - checkErrorResponse(createEndTxnRequest(), unknownServerException, true); + checkErrorResponse(createEndTxnRequest(), new UnknownServerException(), true); checkRequest(createWriteTxnMarkersRequest(), true); checkResponse(createWriteTxnMarkersResponse(), 0, true); - checkErrorResponse(createWriteTxnMarkersRequest(), unknownServerException, true); + checkErrorResponse(createWriteTxnMarkersRequest(), new UnknownServerException(), true); checkOlderFetchVersions(); checkResponse(createMetadataResponse(), 0, true); checkResponse(createMetadataResponse(), 1, true); - checkErrorResponse(createMetadataRequest(1, Collections.singletonList("topic1")), unknownServerException, true); + checkErrorResponse(createMetadataRequest(1, Collections.singletonList("topic1")), new UnknownServerException(), true); checkRequest(createOffsetCommitRequest(0), true); - checkErrorResponse(createOffsetCommitRequest(0), unknownServerException, true); + checkErrorResponse(createOffsetCommitRequest(0), new UnknownServerException(), true); checkRequest(createOffsetCommitRequest(1), true); - checkErrorResponse(createOffsetCommitRequest(1), unknownServerException, true); + checkErrorResponse(createOffsetCommitRequest(1), new UnknownServerException(), true); checkRequest(createOffsetCommitRequest(2), true); - checkErrorResponse(createOffsetCommitRequest(2), unknownServerException, true); + checkErrorResponse(createOffsetCommitRequest(2), new UnknownServerException(), true); checkRequest(createOffsetCommitRequest(3), true); - checkErrorResponse(createOffsetCommitRequest(3), unknownServerException, true); + checkErrorResponse(createOffsetCommitRequest(3), new UnknownServerException(), true); checkRequest(createOffsetCommitRequest(4), true); - checkErrorResponse(createOffsetCommitRequest(4), unknownServerException, true); + checkErrorResponse(createOffsetCommitRequest(4), new UnknownServerException(), true); checkResponse(createOffsetCommitResponse(), 4, true); checkRequest(createOffsetCommitRequest(5), true); - checkErrorResponse(createOffsetCommitRequest(5), unknownServerException, true); + checkErrorResponse(createOffsetCommitRequest(5), new UnknownServerException(), true); checkResponse(createOffsetCommitResponse(), 5, true); checkRequest(createJoinGroupRequest(0), true); checkRequest(createUpdateMetadataRequest(0, null), false); - checkErrorResponse(createUpdateMetadataRequest(0, null), unknownServerException, true); + checkErrorResponse(createUpdateMetadataRequest(0, null), new UnknownServerException(), true); checkRequest(createUpdateMetadataRequest(1, null), false); checkRequest(createUpdateMetadataRequest(1, "rack1"), false); - checkErrorResponse(createUpdateMetadataRequest(1, null), unknownServerException, true); + checkErrorResponse(createUpdateMetadataRequest(1, null), new UnknownServerException(), true); checkRequest(createUpdateMetadataRequest(2, "rack1"), false); checkRequest(createUpdateMetadataRequest(2, null), false); - checkErrorResponse(createUpdateMetadataRequest(2, "rack1"), unknownServerException, true); + checkErrorResponse(createUpdateMetadataRequest(2, "rack1"), new UnknownServerException(), true); checkRequest(createUpdateMetadataRequest(3, "rack1"), false); checkRequest(createUpdateMetadataRequest(3, null), false); - checkErrorResponse(createUpdateMetadataRequest(3, "rack1"), unknownServerException, true); + checkErrorResponse(createUpdateMetadataRequest(3, "rack1"), new UnknownServerException(), true); checkRequest(createUpdateMetadataRequest(4, "rack1"), false); checkRequest(createUpdateMetadataRequest(4, null), false); - checkErrorResponse(createUpdateMetadataRequest(4, "rack1"), unknownServerException, true); + checkErrorResponse(createUpdateMetadataRequest(4, "rack1"), new UnknownServerException(), true); checkRequest(createUpdateMetadataRequest(5, "rack1"), false); checkRequest(createUpdateMetadataRequest(5, null), false); - checkErrorResponse(createUpdateMetadataRequest(5, "rack1"), unknownServerException, true); + checkErrorResponse(createUpdateMetadataRequest(5, "rack1"), new UnknownServerException(), true); checkResponse(createUpdateMetadataResponse(), 0, true); checkRequest(createListOffsetRequest(0), true); - checkErrorResponse(createListOffsetRequest(0), unknownServerException, true); + checkErrorResponse(createListOffsetRequest(0), new UnknownServerException(), true); checkResponse(createListOffsetResponse(0), 0, true); checkRequest(createLeaderEpochRequestForReplica(0, 1), true); checkRequest(createLeaderEpochRequestForConsumer(), true); checkResponse(createLeaderEpochResponse(), 0, true); - checkErrorResponse(createLeaderEpochRequestForConsumer(), unknownServerException, true); + checkErrorResponse(createLeaderEpochRequestForConsumer(), new UnknownServerException(), true); checkRequest(createAddPartitionsToTxnRequest(), true); - checkErrorResponse(createAddPartitionsToTxnRequest(), unknownServerException, true); + checkErrorResponse(createAddPartitionsToTxnRequest(), new UnknownServerException(), true); checkResponse(createAddPartitionsToTxnResponse(), 0, true); checkRequest(createAddOffsetsToTxnRequest(), true); - checkErrorResponse(createAddOffsetsToTxnRequest(), unknownServerException, true); + checkErrorResponse(createAddOffsetsToTxnRequest(), new UnknownServerException(), true); checkResponse(createAddOffsetsToTxnResponse(), 0, true); checkRequest(createEndTxnRequest(), true); - checkErrorResponse(createEndTxnRequest(), unknownServerException, true); + checkErrorResponse(createEndTxnRequest(), new UnknownServerException(), true); checkResponse(createEndTxnResponse(), 0, true); checkRequest(createWriteTxnMarkersRequest(), true); - checkErrorResponse(createWriteTxnMarkersRequest(), unknownServerException, true); + checkErrorResponse(createWriteTxnMarkersRequest(), new UnknownServerException(), true); checkResponse(createWriteTxnMarkersResponse(), 0, true); checkRequest(createTxnOffsetCommitRequest(0), true); checkRequest(createTxnOffsetCommitRequest(3), true); checkRequest(createTxnOffsetCommitRequestWithAutoDowngrade(2), true); - checkErrorResponse(createTxnOffsetCommitRequest(0), unknownServerException, true); - checkErrorResponse(createTxnOffsetCommitRequest(3), unknownServerException, true); - checkErrorResponse(createTxnOffsetCommitRequestWithAutoDowngrade(2), unknownServerException, true); + checkErrorResponse(createTxnOffsetCommitRequest(0), new UnknownServerException(), true); + checkErrorResponse(createTxnOffsetCommitRequest(3), new UnknownServerException(), true); + checkErrorResponse(createTxnOffsetCommitRequestWithAutoDowngrade(2), new UnknownServerException(), true); checkResponse(createTxnOffsetCommitResponse(), 0, true); checkRequest(createDescribeAclsRequest(), true); checkErrorResponse(createDescribeAclsRequest(), new SecurityDisabledException("Security is not enabled."), true); @@ -440,18 +434,18 @@ public void testSerialization() throws Exception { checkErrorResponse(createDeleteAclsRequest(), new SecurityDisabledException("Security is not enabled."), true); checkResponse(createDeleteAclsResponse(), ApiKeys.DELETE_ACLS.latestVersion(), true); checkRequest(createAlterConfigsRequest(), false); - checkErrorResponse(createAlterConfigsRequest(), unknownServerException, true); + checkErrorResponse(createAlterConfigsRequest(), new UnknownServerException(), true); checkResponse(createAlterConfigsResponse(), 0, false); checkRequest(createDescribeConfigsRequest(0), true); checkRequest(createDescribeConfigsRequestWithConfigEntries(0), false); - checkErrorResponse(createDescribeConfigsRequest(0), unknownServerException, true); + checkErrorResponse(createDescribeConfigsRequest(0), new UnknownServerException(), true); checkResponse(createDescribeConfigsResponse((short) 0), 0, false); checkRequest(createDescribeConfigsRequest(1), true); checkRequest(createDescribeConfigsRequestWithConfigEntries(1), false); checkRequest(createDescribeConfigsRequestWithDocumentation(1), false); checkRequest(createDescribeConfigsRequestWithDocumentation(2), false); checkRequest(createDescribeConfigsRequestWithDocumentation(3), false); - checkErrorResponse(createDescribeConfigsRequest(1), unknownServerException, true); + checkErrorResponse(createDescribeConfigsRequest(1), new UnknownServerException(), true); checkResponse(createDescribeConfigsResponse((short) 1), 1, false); checkDescribeConfigsResponseVersions(); checkRequest(createCreatePartitionsRequest(), true); @@ -459,43 +453,36 @@ public void testSerialization() throws Exception { checkErrorResponse(createCreatePartitionsRequest(), new InvalidTopicException(), true); checkResponse(createCreatePartitionsResponse(), 0, true); checkRequest(createCreateTokenRequest(), true); - checkErrorResponse(createCreateTokenRequest(), unknownServerException, true); + checkErrorResponse(createCreateTokenRequest(), new UnknownServerException(), true); checkResponse(createCreateTokenResponse(), 0, true); checkRequest(createDescribeTokenRequest(), true); - checkErrorResponse(createDescribeTokenRequest(), unknownServerException, true); + checkErrorResponse(createDescribeTokenRequest(), new UnknownServerException(), true); checkResponse(createDescribeTokenResponse(), 0, true); checkRequest(createExpireTokenRequest(), true); - checkErrorResponse(createExpireTokenRequest(), unknownServerException, true); + checkErrorResponse(createExpireTokenRequest(), new UnknownServerException(), true); checkResponse(createExpireTokenResponse(), 0, true); checkRequest(createRenewTokenRequest(), true); - checkErrorResponse(createRenewTokenRequest(), unknownServerException, true); + checkErrorResponse(createRenewTokenRequest(), new UnknownServerException(), true); checkResponse(createRenewTokenResponse(), 0, true); checkRequest(createElectLeadersRequest(), true); checkRequest(createElectLeadersRequestNullPartitions(), true); - checkErrorResponse(createElectLeadersRequest(), unknownServerException, true); + checkErrorResponse(createElectLeadersRequest(), new UnknownServerException(), true); checkResponse(createElectLeadersResponse(), 1, true); checkRequest(createIncrementalAlterConfigsRequest(), true); - checkErrorResponse(createIncrementalAlterConfigsRequest(), unknownServerException, true); + checkErrorResponse(createIncrementalAlterConfigsRequest(), new UnknownServerException(), true); checkResponse(createIncrementalAlterConfigsResponse(), 0, true); checkRequest(createAlterPartitionReassignmentsRequest(), true); - checkErrorResponse(createAlterPartitionReassignmentsRequest(), unknownServerException, true); + checkErrorResponse(createAlterPartitionReassignmentsRequest(), new UnknownServerException(), true); checkResponse(createAlterPartitionReassignmentsResponse(), 0, true); checkRequest(createListPartitionReassignmentsRequest(), true); - checkErrorResponse(createListPartitionReassignmentsRequest(), unknownServerException, true); + checkErrorResponse(createListPartitionReassignmentsRequest(), new UnknownServerException(), true); checkResponse(createListPartitionReassignmentsResponse(), 0, true); checkRequest(createOffsetDeleteRequest(), true); - checkErrorResponse(createOffsetDeleteRequest(), unknownServerException, true); + checkErrorResponse(createOffsetDeleteRequest(), new UnknownServerException(), true); checkResponse(createOffsetDeleteResponse(), 0, true); checkRequest(createAlterReplicaLogDirsRequest(), true); - checkErrorResponse(createAlterReplicaLogDirsRequest(), unknownServerException, true); + checkErrorResponse(createAlterReplicaLogDirsRequest(), new UnknownServerException(), true); checkResponse(createAlterReplicaLogDirsResponse(), 0, true); - - checkRequest(createDescribeClientQuotasRequest(), true); - checkErrorResponse(createDescribeClientQuotasRequest(), unknownServerException, true); - checkResponse(createDescribeClientQuotasResponse(), 0, true); - checkRequest(createAlterClientQuotasRequest(), true); - checkErrorResponse(createAlterClientQuotasRequest(), unknownServerException, true); - checkResponse(createAlterClientQuotasResponse(), 0, true); } @Test @@ -509,7 +496,7 @@ public void testResponseHeader() { private void checkOlderFetchVersions() throws Exception { int latestVersion = FETCH.latestVersion(); for (int i = 0; i < latestVersion; ++i) { - checkErrorResponse(createFetchRequest(i), unknownServerException, true); + checkErrorResponse(createFetchRequest(i), new UnknownServerException(), true); checkRequest(createFetchRequest(i), true); checkResponse(createFetchResponse(), i, true); } @@ -564,13 +551,7 @@ private void checkDescribeConfigsResponseVersions() throws Exception { } private void checkErrorResponse(AbstractRequest req, Throwable e, boolean checkEqualityAndHashCode) { - AbstractResponse response = req.getErrorResponse(e); - checkResponse(response, req.version(), checkEqualityAndHashCode); - if (e instanceof UnknownServerException) { - String responseStr = response.toStruct(req.version()).toString(); - assertFalse(String.format("Unknown message included in response for %s: %s ", req.api, responseStr), - responseStr.contains(e.getMessage())); - } + checkResponse(req.getErrorResponse(e), req.version(), checkEqualityAndHashCode); } private void checkRequest(AbstractRequest req, boolean checkEqualityAndHashCode) { @@ -2344,25 +2325,4 @@ private AlterReplicaLogDirsResponse createAlterReplicaLogDirsResponse() { return new AlterReplicaLogDirsResponse(data); } - private DescribeClientQuotasRequest createDescribeClientQuotasRequest() { - ClientQuotaFilter filter = ClientQuotaFilter.all(); - return new DescribeClientQuotasRequest.Builder(filter).build((short) 0); - } - - private DescribeClientQuotasResponse createDescribeClientQuotasResponse() { - ClientQuotaEntity entity = new ClientQuotaEntity(Collections.singletonMap(ClientQuotaEntity.USER, "user")); - return new DescribeClientQuotasResponse(Collections.singletonMap(entity, Collections.singletonMap("request_percentage", 1.0)), 0); - } - - private AlterClientQuotasRequest createAlterClientQuotasRequest() { - ClientQuotaEntity entity = new ClientQuotaEntity(Collections.singletonMap(ClientQuotaEntity.USER, "user")); - ClientQuotaAlteration.Op op = new ClientQuotaAlteration.Op("request_percentage", 2.0); - ClientQuotaAlteration alteration = new ClientQuotaAlteration(entity, Collections.singleton(op)); - return new AlterClientQuotasRequest.Builder(Collections.singleton(alteration), false).build((short) 0); - } - - private AlterClientQuotasResponse createAlterClientQuotasResponse() { - ClientQuotaEntity entity = new ClientQuotaEntity(Collections.singletonMap(ClientQuotaEntity.USER, "user")); - return new AlterClientQuotasResponse(Collections.singletonMap(entity, ApiError.NONE), 0); - } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java index 5a3af47b8dd56..d8d8f4f56a77e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java @@ -59,7 +59,7 @@ public void testErrorCountsWithTopLevelError() { List errors = new ArrayList<>(); errors.add(new StopReplicaPartitionError().setTopicName("foo").setPartitionIndex(0)); errors.add(new StopReplicaPartitionError().setTopicName("foo").setPartitionIndex(1) - .setErrorCode(Errors.NOT_LEADER_OR_FOLLOWER.code())); + .setErrorCode(Errors.NOT_LEADER_FOR_PARTITION.code())); StopReplicaResponse response = new StopReplicaResponse(new StopReplicaResponseData() .setErrorCode(Errors.UNKNOWN_SERVER_ERROR.code()) .setPartitionErrors(errors)); diff --git a/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/expiring/ExpiringCredentialRefreshingLoginTest.java b/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/expiring/ExpiringCredentialRefreshingLoginTest.java index f87405a1f8a77..3a39242927d43 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/expiring/ExpiringCredentialRefreshingLoginTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/expiring/ExpiringCredentialRefreshingLoginTest.java @@ -368,7 +368,6 @@ public void testRefresh() throws Exception { inOrder.verify(mockLoginContext).login(); } } - testExpiringCredentialRefreshingLogin.close(); } } } diff --git a/core/src/main/scala/kafka/admin/ConfigCommand.scala b/core/src/main/scala/kafka/admin/ConfigCommand.scala index ddd3a8861bae0..ea7021c409303 100644 --- a/core/src/main/scala/kafka/admin/ConfigCommand.scala +++ b/core/src/main/scala/kafka/admin/ConfigCommand.scala @@ -23,7 +23,6 @@ import java.util.{Collections, Properties} import joptsimple._ import kafka.common.Config import kafka.log.LogConfig -import kafka.server.DynamicConfig.QuotaConfigs import kafka.server.{ConfigEntityName, ConfigType, Defaults, DynamicBrokerConfig, DynamicConfig, KafkaConfig} import kafka.utils.{CommandDefaultOptions, CommandLineUtils, Exit, PasswordEncoder} import kafka.utils.Implicits._ @@ -365,14 +364,6 @@ object ConfigCommand extends Config { adminClient.incrementalAlterConfigs(Map(configResource -> alterLogLevelEntries).asJava, alterOptions).all().get(60, TimeUnit.SECONDS) case ConfigType.User | ConfigType.Client => - val nonQuotaConfigsToAdd = configsToBeAdded.keys.filterNot(QuotaConfigs.isQuotaConfig) - if (nonQuotaConfigsToAdd.nonEmpty) - throw new IllegalArgumentException(s"Only quota configs can be added for '$entityTypeHead' using --bootstrap-server. Unexpected config names: $nonQuotaConfigsToAdd") - val nonQuotaConfigsToDelete = configsToBeDeleted.filterNot(QuotaConfigs.isQuotaConfig) - if (nonQuotaConfigsToDelete.nonEmpty) - throw new IllegalArgumentException(s"Only quota configs can be deleted for '$entityTypeHead' using --bootstrap-server. Unexpected config names: $nonQuotaConfigsToDelete") - - val oldConfig = getClientQuotasConfig(adminClient, entityTypes, entityNames) val invalidConfigs = configsToBeDeleted.filterNot(oldConfig.contains) diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala index 9e8edaa5ea4e2..f9a9de2e31c08 100755 --- a/core/src/main/scala/kafka/cluster/Partition.scala +++ b/core/src/main/scala/kafka/cluster/Partition.scala @@ -332,7 +332,7 @@ class Partition(val topicPartition: TopicPartition, def getReplica(replicaId: Int): Option[Replica] = Option(remoteReplicasMap.get(replicaId)) private def getReplicaOrException(replicaId: Int): Replica = getReplica(replicaId).getOrElse{ - throw new NotLeaderOrFollowerException(s"Replica with id $replicaId is not available on broker $localBrokerId") + throw new ReplicaNotAvailableException(s"Replica with id $replicaId is not available on broker $localBrokerId") } private def checkCurrentLeaderEpoch(remoteLeaderEpochOpt: Optional[Integer]): Errors = { @@ -355,13 +355,16 @@ class Partition(val topicPartition: TopicPartition, checkCurrentLeaderEpoch(currentLeaderEpoch) match { case Errors.NONE => if (requireLeader && !isLeader) { - Right(Errors.NOT_LEADER_OR_FOLLOWER) + Right(Errors.NOT_LEADER_FOR_PARTITION) } else { log match { case Some(partitionLog) => Left(partitionLog) case _ => - Right(Errors.NOT_LEADER_OR_FOLLOWER) + if (requireLeader) + Right(Errors.NOT_LEADER_FOR_PARTITION) + else + Right(Errors.REPLICA_NOT_AVAILABLE) } } case error => @@ -370,12 +373,12 @@ class Partition(val topicPartition: TopicPartition, } def localLogOrException: Log = log.getOrElse { - throw new NotLeaderOrFollowerException(s"Log for partition $topicPartition is not available " + + throw new ReplicaNotAvailableException(s"Log for partition $topicPartition is not available " + s"on broker $localBrokerId") } def futureLocalLogOrException: Log = futureLog.getOrElse { - throw new NotLeaderOrFollowerException(s"Future log for partition $topicPartition is not available " + + throw new ReplicaNotAvailableException(s"Future log for partition $topicPartition is not available " + s"on broker $localBrokerId") } @@ -775,7 +778,7 @@ class Partition(val topicPartition: TopicPartition, } else (false, Errors.NONE) case None => - (false, Errors.NOT_LEADER_OR_FOLLOWER) + (false, Errors.NOT_LEADER_FOR_PARTITION) } } @@ -837,7 +840,7 @@ class Partition(val topicPartition: TopicPartition, */ def lowWatermarkIfLeader: Long = { if (!isLeader) - throw new NotLeaderOrFollowerException(s"Leader not local for partition $topicPartition on broker $localBrokerId") + throw new NotLeaderForPartitionException(s"Leader not local for partition $topicPartition on broker $localBrokerId") // lowWatermarkIfLeader may be called many times when a DeleteRecordsRequest is outstanding, // care has been taken to avoid generating unnecessary collections in this code @@ -1002,7 +1005,7 @@ class Partition(val topicPartition: TopicPartition, (info, maybeIncrementLeaderHW(leaderLog)) case None => - throw new NotLeaderOrFollowerException("Leader not local for partition %s on broker %d" + throw new NotLeaderForPartitionException("Leader not local for partition %s on broker %d" .format(topicPartition, localBrokerId)) } } @@ -1145,7 +1148,7 @@ class Partition(val topicPartition: TopicPartition, requestedOffset = convertedOffset, lowWatermark = lowWatermarkIfLeader) case None => - throw new NotLeaderOrFollowerException(s"Leader not local for partition $topicPartition on broker $localBrokerId") + throw new NotLeaderForPartitionException(s"Leader not local for partition $topicPartition on broker $localBrokerId") } } diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index 1fcdd91ad6af7..c77f7023e3886 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -285,7 +285,7 @@ class GroupMetadataManager(brokerId: Int, | Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND => Errors.COORDINATOR_NOT_AVAILABLE - case Errors.NOT_LEADER_OR_FOLLOWER + case Errors.NOT_LEADER_FOR_PARTITION | Errors.KAFKA_STORAGE_ERROR => Errors.NOT_COORDINATOR @@ -432,7 +432,7 @@ class GroupMetadataManager(brokerId: Int, | Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND => Errors.COORDINATOR_NOT_AVAILABLE - case Errors.NOT_LEADER_OR_FOLLOWER + case Errors.NOT_LEADER_FOR_PARTITION | Errors.KAFKA_STORAGE_ERROR => Errors.NOT_COORDINATOR diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala index c2efdbd934672..66edc47b59d01 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala @@ -143,7 +143,7 @@ class TransactionMarkerRequestCompletionHandler(brokerId: Int, throw new IllegalStateException(s"Received fatal error ${error.exceptionName} while sending txn marker for $transactionalId") case Errors.UNKNOWN_TOPIC_OR_PARTITION | - Errors.NOT_LEADER_OR_FOLLOWER | + Errors.NOT_LEADER_FOR_PARTITION | Errors.NOT_ENOUGH_REPLICAS | Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND | Errors.REQUEST_TIMED_OUT | diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala index 923aed9776635..8eb65df1d3393 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala @@ -513,7 +513,7 @@ class TransactionStateManager(brokerId: Int, | Errors.REQUEST_TIMED_OUT => // note that for timed out request we return NOT_AVAILABLE error code to let client retry Errors.COORDINATOR_NOT_AVAILABLE - case Errors.NOT_LEADER_OR_FOLLOWER + case Errors.NOT_LEADER_FOR_PARTITION | Errors.KAFKA_STORAGE_ERROR => Errors.NOT_COORDINATOR diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala index 1a6261bf65840..0aeb79c950641 100755 --- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala +++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala @@ -377,7 +377,7 @@ abstract class AbstractFetcherThread(name: String, case Errors.FENCED_LEADER_EPOCH => if (onPartitionFenced(topicPartition, requestEpoch)) partitionsWithError += topicPartition - case Errors.NOT_LEADER_OR_FOLLOWER => + case Errors.NOT_LEADER_FOR_PARTITION => debug(s"Remote broker is not the leader for partition $topicPartition, which could indicate " + "that the partition is being moved") partitionsWithError += topicPartition @@ -562,7 +562,7 @@ abstract class AbstractFetcherThread(name: String, case e @ (_ : UnknownTopicOrPartitionException | _ : UnknownLeaderEpochException | - _ : NotLeaderOrFollowerException) => + _ : NotLeaderForPartitionException) => info(s"Could not fetch offset for $topicPartition due to error: ${e.getMessage}") true diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala index e0a57b24d3574..d0522c4fd9f07 100644 --- a/core/src/main/scala/kafka/server/AdminManager.scala +++ b/core/src/main/scala/kafka/server/AdminManager.scala @@ -23,7 +23,6 @@ import kafka.common.TopicAlreadyMarkedForDeletionException import kafka.log.LogConfig import kafka.utils.Log4jController import kafka.metrics.KafkaMetricsGroup -import kafka.server.DynamicConfig.QuotaConfigs import kafka.utils._ import kafka.zk.{AdminZkClient, KafkaZkClient} import org.apache.kafka.clients.admin.AlterConfigOp @@ -885,20 +884,19 @@ class AdminManager(val config: KafkaConfig, !name.isDefined || !strict } - def fromProps(props: Map[String, String]): Map[String, Double] = { - props.map { case (key, value) => + def fromProps(props: Properties): Map[String, Double] = { + props.asScala.map { case (key, value) => val doubleValue = try value.toDouble catch { case _: NumberFormatException => - throw new IllegalStateException(s"Unexpected client quota configuration value: $key -> $value") + throw new IllegalStateException(s"Unexpected client quota configuration value: ${key} -> ${value}") } key -> doubleValue } } (userEntries ++ clientIdEntries ++ bothEntries).map { case ((u, c), p) => - val quotaProps = p.asScala.filter { case (key, _) => QuotaConfigs.isQuotaConfig(key) } - if (quotaProps.nonEmpty && matches(userComponent, u) && matches(clientIdComponent, c)) - Some(userClientIdToEntity(u, c) -> fromProps(quotaProps)) + if (!p.isEmpty && matches(userComponent, u) && matches(clientIdComponent, c)) + Some(userClientIdToEntity(u, c) -> fromProps(p)) else None }.flatten.toMap diff --git a/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala b/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala index e665688af056f..52752757f99bb 100644 --- a/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala +++ b/core/src/main/scala/kafka/server/DelayedDeleteRecords.scala @@ -80,7 +80,7 @@ class DelayedDeleteRecords(delayMs: Long, val leaderLW = partition.lowWatermarkIfLeader (leaderLW >= status.requiredOffset, Errors.NONE, leaderLW) case None => - (false, Errors.NOT_LEADER_OR_FOLLOWER, DeleteRecordsResponse.INVALID_LOW_WATERMARK) + (false, Errors.NOT_LEADER_FOR_PARTITION, DeleteRecordsResponse.INVALID_LOW_WATERMARK) } case HostedPartition.Offline => diff --git a/core/src/main/scala/kafka/server/DelayedFetch.scala b/core/src/main/scala/kafka/server/DelayedFetch.scala index 3aa2ee413b035..50cd4eeea7748 100644 --- a/core/src/main/scala/kafka/server/DelayedFetch.scala +++ b/core/src/main/scala/kafka/server/DelayedFetch.scala @@ -87,7 +87,8 @@ class DelayedFetch(delayMs: Long, val fetchLeaderEpoch = fetchStatus.fetchInfo.currentLeaderEpoch try { if (fetchOffset != LogOffsetMetadata.UnknownOffsetMetadata) { - val partition = replicaManager.getPartitionOrException(topicPartition) + val partition = replicaManager.getPartitionOrException(topicPartition, + expectLeader = fetchMetadata.fetchOnlyLeader) val offsetSnapshot = partition.fetchOffsetSnapshot(fetchLeaderEpoch, fetchMetadata.fetchOnlyLeader) val endOffset = fetchMetadata.fetchIsolation match { @@ -120,8 +121,11 @@ class DelayedFetch(delayMs: Long, } } } catch { - case _: NotLeaderOrFollowerException => // Case A or Case B - debug(s"Broker is no longer the leader or follower of $topicPartition, satisfy $fetchMetadata immediately") + case _: NotLeaderForPartitionException => // Case A + debug(s"Broker is no longer the leader of $topicPartition, satisfy $fetchMetadata immediately") + return forceComplete() + case _: ReplicaNotAvailableException => // Case B + debug(s"Broker no longer has a replica of $topicPartition, satisfy $fetchMetadata immediately") return forceComplete() case _: UnknownTopicOrPartitionException => // Case C debug(s"Broker no longer knows of partition $topicPartition, satisfy $fetchMetadata immediately") diff --git a/core/src/main/scala/kafka/server/DelayedProduce.scala b/core/src/main/scala/kafka/server/DelayedProduce.scala index 668851d947bf9..83e6142008205 100644 --- a/core/src/main/scala/kafka/server/DelayedProduce.scala +++ b/core/src/main/scala/kafka/server/DelayedProduce.scala @@ -85,7 +85,7 @@ class DelayedProduce(delayMs: Long, trace(s"Checking produce satisfaction for $topicPartition, current status $status") // skip those partitions that have already been satisfied if (status.acksPending) { - val (hasEnough, error) = replicaManager.getPartitionOrError(topicPartition) match { + val (hasEnough, error) = replicaManager.getPartitionOrError(topicPartition, expectLeader = true) match { case Left(err) => // Case A (false, err) diff --git a/core/src/main/scala/kafka/server/DynamicConfig.scala b/core/src/main/scala/kafka/server/DynamicConfig.scala index f3d40a11aea3d..13c64bfe3d61f 100644 --- a/core/src/main/scala/kafka/server/DynamicConfig.scala +++ b/core/src/main/scala/kafka/server/DynamicConfig.scala @@ -67,20 +67,11 @@ object DynamicConfig { def validate(props: Properties) = DynamicConfig.validate(brokerConfigDef, props, customPropsAllowed = true) } - object QuotaConfigs { + object Client { + //Properties val ProducerByteRateOverrideProp = "producer_byte_rate" val ConsumerByteRateOverrideProp = "consumer_byte_rate" val RequestPercentageOverrideProp = "request_percentage" - private val configNames = Set(ProducerByteRateOverrideProp, ConsumerByteRateOverrideProp, RequestPercentageOverrideProp) - - def isQuotaConfig(name: String): Boolean = configNames.contains(name) - } - - object Client { - //Properties - val ProducerByteRateOverrideProp = QuotaConfigs.ProducerByteRateOverrideProp - val ConsumerByteRateOverrideProp = QuotaConfigs.ConsumerByteRateOverrideProp - val RequestPercentageOverrideProp = QuotaConfigs.RequestPercentageOverrideProp //Defaults val DefaultProducerOverride = ClientQuotaManagerConfig.QuotaBytesPerSecondDefault diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index a7397664ca4a7..46bd890b459dc 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -923,10 +923,10 @@ class KafkaApis(val requestChannel: RequestChannel, fetchOnlyFromLeader = offsetRequest.replicaId != ListOffsetRequest.DEBUGGING_REPLICA_ID) (topicPartition, new ListOffsetResponse.PartitionData(Errors.NONE, offsets.map(JLong.valueOf).asJava)) } catch { - // NOTE: UnknownTopicOrPartitionException and NotLeaderOrFollowerException are special cased since these error messages + // NOTE: UnknownTopicOrPartitionException and NotLeaderForPartitionException are special cased since these error messages // are typically transient and there is no value in logging the entire stack trace for the same case e @ (_ : UnknownTopicOrPartitionException | - _ : NotLeaderOrFollowerException | + _ : NotLeaderForPartitionException | _ : KafkaStorageException) => debug("Offset request with correlation id %d from client %s on partition %s failed due to %s".format( correlationId, clientId, topicPartition, e.getMessage)) @@ -998,7 +998,7 @@ class KafkaApis(val requestChannel: RequestChannel, // NOTE: These exceptions are special cased since these error messages are typically transient or the client // would have received a clear exception and there is no value in logging the entire stack trace for the same case e @ (_ : UnknownTopicOrPartitionException | - _ : NotLeaderOrFollowerException | + _ : NotLeaderForPartitionException | _ : UnknownLeaderEpochException | _ : FencedLeaderEpochException | _ : KafkaStorageException | diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala index 91b873ff17c18..5db3464398d3c 100644 --- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala +++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala @@ -146,12 +146,12 @@ class ReplicaAlterLogDirsThread(name: String, } override protected def fetchEarliestOffsetFromLeader(topicPartition: TopicPartition, leaderEpoch: Int): Long = { - val partition = replicaMgr.getPartitionOrException(topicPartition) + val partition = replicaMgr.getPartitionOrException(topicPartition, expectLeader = false) partition.localLogOrException.logStartOffset } override protected def fetchLatestOffsetFromLeader(topicPartition: TopicPartition, leaderEpoch: Int): Long = { - val partition = replicaMgr.getPartitionOrException(topicPartition) + val partition = replicaMgr.getPartitionOrException(topicPartition, expectLeader = false) partition.localLogOrException.logEndOffset } @@ -166,7 +166,7 @@ class ReplicaAlterLogDirsThread(name: String, val endOffset = if (epochData.leaderEpoch == UNDEFINED_EPOCH) { new EpochEndOffset(UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET) } else { - val partition = replicaMgr.getPartitionOrException(tp) + val partition = replicaMgr.getPartitionOrException(tp, expectLeader = false) partition.lastOffsetForLeaderEpoch( currentLeaderEpoch = epochData.currentLeaderEpoch, leaderEpoch = epochData.leaderEpoch, @@ -198,12 +198,12 @@ class ReplicaAlterLogDirsThread(name: String, * exchange with the current replica to truncate to the largest common log prefix for the topic partition */ override def truncate(topicPartition: TopicPartition, truncationState: OffsetTruncationState): Unit = { - val partition = replicaMgr.getPartitionOrException(topicPartition) + val partition = replicaMgr.getPartitionOrException(topicPartition, expectLeader = false) partition.truncateTo(truncationState.offset, isFuture = true) } override protected def truncateFullyAndStartAt(topicPartition: TopicPartition, offset: Long): Unit = { - val partition = replicaMgr.getPartitionOrException(topicPartition) + val partition = replicaMgr.getPartitionOrException(topicPartition, expectLeader = false) partition.truncateFullyAndStartAt(offset, isFuture = true) } diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index be73bf2a2c159..7f503774d1172 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -498,8 +498,8 @@ class ReplicaManager(val config: KafkaConfig, allPartitions.values.iterator.count(_ == HostedPartition.Offline) } - def getPartitionOrException(topicPartition: TopicPartition): Partition = { - getPartitionOrError(topicPartition) match { + def getPartitionOrException(topicPartition: TopicPartition, expectLeader: Boolean): Partition = { + getPartitionOrError(topicPartition, expectLeader) match { case Left(Errors.KAFKA_STORAGE_ERROR) => throw new KafkaStorageException(s"Partition $topicPartition is in an offline log directory") @@ -510,7 +510,7 @@ class ReplicaManager(val config: KafkaConfig, } } - def getPartitionOrError(topicPartition: TopicPartition): Either[Errors, Partition] = { + def getPartitionOrError(topicPartition: TopicPartition, expectLeader: Boolean): Either[Errors, Partition] = { getPartition(topicPartition) match { case HostedPartition.Online(partition) => Right(partition) @@ -519,11 +519,15 @@ class ReplicaManager(val config: KafkaConfig, Left(Errors.KAFKA_STORAGE_ERROR) case HostedPartition.None if metadataCache.contains(topicPartition) => - // The topic exists, but this broker is no longer a replica of it, so we return NOT_LEADER_OR_FOLLOWER which - // forces clients to refresh metadata to find the new location. This can happen, for example, - // during a partition reassignment if a produce request from the client is sent to a broker after - // the local replica has been deleted. - Left(Errors.NOT_LEADER_OR_FOLLOWER) + if (expectLeader) { + // The topic exists, but this broker is no longer a replica of it, so we return NOT_LEADER which + // forces clients to refresh metadata to find the new location. This can happen, for example, + // during a partition reassignment if a produce request from the client is sent to a broker after + // the local replica has been deleted. + Left(Errors.NOT_LEADER_FOR_PARTITION) + } else { + Left(Errors.REPLICA_NOT_AVAILABLE) + } case HostedPartition.None => Left(Errors.UNKNOWN_TOPIC_OR_PARTITION) @@ -531,15 +535,15 @@ class ReplicaManager(val config: KafkaConfig, } def localLogOrException(topicPartition: TopicPartition): Log = { - getPartitionOrException(topicPartition).localLogOrException + getPartitionOrException(topicPartition, expectLeader = false).localLogOrException } def futureLocalLogOrException(topicPartition: TopicPartition): Log = { - getPartitionOrException(topicPartition).futureLocalLogOrException + getPartitionOrException(topicPartition, expectLeader = false).futureLocalLogOrException } def futureLogExists(topicPartition: TopicPartition): Boolean = { - getPartitionOrException(topicPartition).futureLog.isDefined + getPartitionOrException(topicPartition, expectLeader = false).futureLog.isDefined } def localLog(topicPartition: TopicPartition): Option[Log] = { @@ -620,12 +624,12 @@ class ReplicaManager(val config: KafkaConfig, (topicPartition, LogDeleteRecordsResult(-1L, -1L, Some(new InvalidTopicException(s"Cannot delete records of internal topic ${topicPartition.topic}")))) } else { try { - val partition = getPartitionOrException(topicPartition) + val partition = getPartitionOrException(topicPartition, expectLeader = true) val logDeleteResult = partition.deleteRecordsOnLeader(requestedOffset) (topicPartition, logDeleteResult) } catch { case e@ (_: UnknownTopicOrPartitionException | - _: NotLeaderOrFollowerException | + _: NotLeaderForPartitionException | _: OffsetOutOfRangeException | _: PolicyViolationException | _: KafkaStorageException) => @@ -682,11 +686,11 @@ class ReplicaManager(val config: KafkaConfig, // If the log for this partition has not been created yet: // 1) Record the destination log directory in the memory so that the partition will be created in this log directory // when broker receives LeaderAndIsrRequest for this partition later. - // 2) Respond with NotLeaderOrFollowerException for this partition in the AlterReplicaLogDirsResponse + // 2) Respond with ReplicaNotAvailableException for this partition in the AlterReplicaLogDirsResponse logManager.maybeUpdatePreferredLogDir(topicPartition, destinationDir) - // throw NotLeaderOrFollowerException if replica does not exist for the given partition - val partition = getPartitionOrException(topicPartition) + // throw ReplicaNotAvailableException if replica does not exist for the given partition + val partition = getPartitionOrException(topicPartition, expectLeader = false) partition.localLogOrException // If the destinationLDir is different from the current log directory of the replica: @@ -710,12 +714,8 @@ class ReplicaManager(val config: KafkaConfig, _: LogDirNotFoundException | _: ReplicaNotAvailableException | _: KafkaStorageException) => - warn(s"Unable to alter log dirs for $topicPartition", e) + warn("Unable to alter log dirs for %s".format(topicPartition), e) (topicPartition, Errors.forException(e)) - case e: NotLeaderOrFollowerException => - // Retaining REPLICA_NOT_AVAILABLE exception for ALTER_REPLICA_LOG_DIRS for compatibility - warn(s"Unable to alter log dirs for $topicPartition", e) - (topicPartition, Errors.REPLICA_NOT_AVAILABLE) case t: Throwable => error("Error while changing replica dir for partition %s".format(topicPartition), t) (topicPartition, Errors.forException(t)) @@ -877,7 +877,7 @@ class ReplicaManager(val config: KafkaConfig, Some(new InvalidTopicException(s"Cannot append to internal topic ${topicPartition.topic}")))) } else { try { - val partition = getPartitionOrException(topicPartition) + val partition = getPartitionOrException(topicPartition, expectLeader = true) val info = partition.appendRecordsToLeader(records, origin, requiredAcks) val numAppendedMessages = info.numMessages @@ -896,7 +896,7 @@ class ReplicaManager(val config: KafkaConfig, // NOTE: Failed produce requests metric is not incremented for known exceptions // it is supposed to indicate un-expected failures of a broker in handling a produce request case e@ (_: UnknownTopicOrPartitionException | - _: NotLeaderOrFollowerException | + _: NotLeaderForPartitionException | _: RecordTooLargeException | _: RecordBatchTooLargeException | _: CorruptRecordException | @@ -920,7 +920,7 @@ class ReplicaManager(val config: KafkaConfig, isolationLevel: Option[IsolationLevel], currentLeaderEpoch: Optional[Integer], fetchOnlyFromLeader: Boolean): Option[TimestampAndOffset] = { - val partition = getPartitionOrException(topicPartition) + val partition = getPartitionOrException(topicPartition, expectLeader = fetchOnlyFromLeader) partition.fetchOffsetForTimestamp(timestamp, isolationLevel, currentLeaderEpoch, fetchOnlyFromLeader) } @@ -929,7 +929,7 @@ class ReplicaManager(val config: KafkaConfig, maxNumOffsets: Int, isFromConsumer: Boolean, fetchOnlyFromLeader: Boolean): Seq[Long] = { - val partition = getPartitionOrException(topicPartition) + val partition = getPartitionOrException(topicPartition, expectLeader = fetchOnlyFromLeader) partition.legacyFetchOffsetsForTimestamp(timestamp, maxNumOffsets, isFromConsumer, fetchOnlyFromLeader) } @@ -1048,7 +1048,7 @@ class ReplicaManager(val config: KafkaConfig, s"remaining response limit $limitBytes" + (if (minOneMessage) s", ignoring response/partition size limits" else "")) - val partition = getPartitionOrException(tp) + val partition = getPartitionOrException(tp, expectLeader = fetchOnlyFromLeader) val fetchTimeMs = time.milliseconds // If we are the leader, determine the preferred read-replica @@ -1106,7 +1106,7 @@ class ReplicaManager(val config: KafkaConfig, // NOTE: Failed fetch requests metric is not incremented for known exceptions since it // is supposed to indicate un-expected failure of a broker in handling a fetch request case e@ (_: UnknownTopicOrPartitionException | - _: NotLeaderOrFollowerException | + _: NotLeaderForPartitionException | _: UnknownLeaderEpochException | _: FencedLeaderEpochException | _: ReplicaNotAvailableException | @@ -1781,7 +1781,7 @@ class ReplicaManager(val config: KafkaConfig, new EpochEndOffset(Errors.KAFKA_STORAGE_ERROR, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET) case HostedPartition.None if metadataCache.contains(tp) => - new EpochEndOffset(Errors.NOT_LEADER_OR_FOLLOWER, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET) + new EpochEndOffset(Errors.NOT_LEADER_FOR_PARTITION, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET) case HostedPartition.None => new EpochEndOffset(Errors.UNKNOWN_TOPIC_OR_PARTITION, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET) diff --git a/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala index 136e2a0383031..b181dc2efea37 100644 --- a/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala @@ -698,7 +698,7 @@ class PlaintextAdminIntegrationTest extends BaseAdminIntegrationTest { lowWatermark.contains(5L) } catch { case e: ExecutionException if e.getCause.isInstanceOf[LeaderNotAvailableException] || - e.getCause.isInstanceOf[NotLeaderOrFollowerException] => false + e.getCause.isInstanceOf[NotLeaderForPartitionException] => false } }, s"Expected low watermark of the partition to be 5 but got ${lowWatermark.getOrElse("no response within the timeout")}") } diff --git a/core/src/test/scala/integration/kafka/api/ProducerFailureHandlingTest.scala b/core/src/test/scala/integration/kafka/api/ProducerFailureHandlingTest.scala index fb4d839c0351f..beb52323e4689 100644 --- a/core/src/test/scala/integration/kafka/api/ProducerFailureHandlingTest.scala +++ b/core/src/test/scala/integration/kafka/api/ProducerFailureHandlingTest.scala @@ -161,7 +161,7 @@ class ProducerFailureHandlingTest extends KafkaServerTestHarness { * * TODO: other exceptions that can be thrown in ExecutionException: * UnknownTopicOrPartitionException - * NotLeaderOrFollowerException + * NotLeaderForPartitionException * LeaderNotAvailableException * CorruptRecordException * TimeoutException diff --git a/core/src/test/scala/integration/kafka/server/DelayedFetchTest.scala b/core/src/test/scala/integration/kafka/server/DelayedFetchTest.scala index 38831b558ac4e..c43d48125ab4c 100644 --- a/core/src/test/scala/integration/kafka/server/DelayedFetchTest.scala +++ b/core/src/test/scala/integration/kafka/server/DelayedFetchTest.scala @@ -21,7 +21,7 @@ import java.util.Optional import scala.collection.Seq import kafka.cluster.Partition import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.errors.{FencedLeaderEpochException, NotLeaderOrFollowerException} +import org.apache.kafka.common.errors.{FencedLeaderEpochException, ReplicaNotAvailableException} import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests.FetchRequest @@ -62,7 +62,7 @@ class DelayedFetchTest extends EasyMockSupport { val partition: Partition = mock(classOf[Partition]) - EasyMock.expect(replicaManager.getPartitionOrException(topicPartition)) + EasyMock.expect(replicaManager.getPartitionOrException(topicPartition, expectLeader = true)) .andReturn(partition) EasyMock.expect(partition.fetchOffsetSnapshot(currentLeaderEpoch, fetchOnlyFromLeader = true)) .andThrow(new FencedLeaderEpochException("Requested epoch has been fenced")) @@ -81,7 +81,7 @@ class DelayedFetchTest extends EasyMockSupport { } @Test - def testNotLeaderOrFollower(): Unit = { + def testReplicaNotAvailable(): Unit = { val topicPartition = new TopicPartition("topic", 0) val fetchOffset = 500L val logStartOffset = 0L @@ -106,9 +106,9 @@ class DelayedFetchTest extends EasyMockSupport { clientMetadata = None, responseCallback = callback) - EasyMock.expect(replicaManager.getPartitionOrException(topicPartition)) - .andThrow(new NotLeaderOrFollowerException(s"Replica for $topicPartition not available")) - expectReadFromReplicaWithError(replicaId, topicPartition, fetchStatus.fetchInfo, Errors.NOT_LEADER_OR_FOLLOWER) + EasyMock.expect(replicaManager.getPartitionOrException(topicPartition, expectLeader = true)) + .andThrow(new ReplicaNotAvailableException(s"Replica for $topicPartition not available")) + expectReadFromReplicaWithError(replicaId, topicPartition, fetchStatus.fetchInfo, Errors.REPLICA_NOT_AVAILABLE) EasyMock.expect(replicaManager.isAddingReplica(EasyMock.anyObject(), EasyMock.anyInt())).andReturn(false) replayAll() diff --git a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala index d4121237f63b4..2b357b1958001 100644 --- a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala @@ -485,27 +485,6 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { testShouldAddClientConfig(None, Some(null)) } - @Test - def shouldNotAlterNonQuotaClientConfigUsingBootstrapServer(): Unit = { - val node = new Node(1, "localhost", 9092) - val mockAdminClient = new MockAdminClient(util.Collections.singletonList(node), node) - - def verifyCommand(entityType: String, alterOpts: String*): Unit = { - val opts = new ConfigCommandOptions(Array("--bootstrap-server", "localhost:9092", - "--entity-type", entityType, "--entity-name", "admin", - "--alter") ++ alterOpts) - val e = intercept[IllegalArgumentException] { - ConfigCommand.alterConfig(mockAdminClient, opts) - } - assertTrue(s"Unexpected exception: $e", e.getMessage.contains("some_config")) - } - - verifyCommand("users", "--add-config", "consumer_byte_rate=20000,producer_byte_rate=10000,some_config=10") - verifyCommand("clients", "--add-config", "some_config=10") - verifyCommand("users", "--delete-config", "consumer_byte_rate=20000,some_config=10") - verifyCommand("clients", "--delete-config", "some_config=10") - } - @Test def shouldAddTopicConfigUsingZookeeper(): Unit = { val createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect, diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala index dc73b4db2f6ed..725117f5e5607 100644 --- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala +++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala @@ -30,7 +30,7 @@ import kafka.server._ import kafka.server.checkpoints.OffsetCheckpoints import kafka.utils._ import org.apache.kafka.common.{IsolationLevel, TopicPartition} -import org.apache.kafka.common.errors.{ApiException, NotLeaderOrFollowerException, OffsetNotAvailableException} +import org.apache.kafka.common.errors.{ApiException, OffsetNotAvailableException, ReplicaNotAvailableException} import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.record.FileRecords.TimestampAndOffset @@ -271,8 +271,8 @@ class PartitionTest extends AbstractPartitionTest { assertSnapshotError(Errors.FENCED_LEADER_EPOCH, Optional.of(leaderEpoch - 1), fetchOnlyLeader = false) assertSnapshotError(Errors.UNKNOWN_LEADER_EPOCH, Optional.of(leaderEpoch + 1), fetchOnlyLeader = false) - assertSnapshotError(Errors.NOT_LEADER_OR_FOLLOWER, Optional.of(leaderEpoch), fetchOnlyLeader = true) - assertSnapshotError(Errors.NOT_LEADER_OR_FOLLOWER, Optional.empty(), fetchOnlyLeader = true) + assertSnapshotError(Errors.NOT_LEADER_FOR_PARTITION, Optional.of(leaderEpoch), fetchOnlyLeader = true) + assertSnapshotError(Errors.NOT_LEADER_FOR_PARTITION, Optional.empty(), fetchOnlyLeader = true) assertSnapshotError(Errors.FENCED_LEADER_EPOCH, Optional.of(leaderEpoch - 1), fetchOnlyLeader = true) assertSnapshotError(Errors.UNKNOWN_LEADER_EPOCH, Optional.of(leaderEpoch + 1), fetchOnlyLeader = true) } @@ -312,8 +312,8 @@ class PartitionTest extends AbstractPartitionTest { assertLastOffsetForLeaderError(Errors.FENCED_LEADER_EPOCH, Optional.of(leaderEpoch - 1), fetchOnlyLeader = false) assertLastOffsetForLeaderError(Errors.UNKNOWN_LEADER_EPOCH, Optional.of(leaderEpoch + 1), fetchOnlyLeader = false) - assertLastOffsetForLeaderError(Errors.NOT_LEADER_OR_FOLLOWER, Optional.empty(), fetchOnlyLeader = true) - assertLastOffsetForLeaderError(Errors.NOT_LEADER_OR_FOLLOWER, Optional.of(leaderEpoch), fetchOnlyLeader = true) + assertLastOffsetForLeaderError(Errors.NOT_LEADER_FOR_PARTITION, Optional.empty(), fetchOnlyLeader = true) + assertLastOffsetForLeaderError(Errors.NOT_LEADER_FOR_PARTITION, Optional.of(leaderEpoch), fetchOnlyLeader = true) assertLastOffsetForLeaderError(Errors.FENCED_LEADER_EPOCH, Optional.of(leaderEpoch - 1), fetchOnlyLeader = true) assertLastOffsetForLeaderError(Errors.UNKNOWN_LEADER_EPOCH, Optional.of(leaderEpoch + 1), fetchOnlyLeader = true) } @@ -372,8 +372,8 @@ class PartitionTest extends AbstractPartitionTest { assertReadRecordsError(Errors.FENCED_LEADER_EPOCH, Optional.of(leaderEpoch - 1), fetchOnlyLeader = false) assertReadRecordsError(Errors.UNKNOWN_LEADER_EPOCH, Optional.of(leaderEpoch + 1), fetchOnlyLeader = false) - assertReadRecordsError(Errors.NOT_LEADER_OR_FOLLOWER, Optional.empty(), fetchOnlyLeader = true) - assertReadRecordsError(Errors.NOT_LEADER_OR_FOLLOWER, Optional.of(leaderEpoch), fetchOnlyLeader = true) + assertReadRecordsError(Errors.NOT_LEADER_FOR_PARTITION, Optional.empty(), fetchOnlyLeader = true) + assertReadRecordsError(Errors.NOT_LEADER_FOR_PARTITION, Optional.of(leaderEpoch), fetchOnlyLeader = true) assertReadRecordsError(Errors.FENCED_LEADER_EPOCH, Optional.of(leaderEpoch - 1), fetchOnlyLeader = true) assertReadRecordsError(Errors.UNKNOWN_LEADER_EPOCH, Optional.of(leaderEpoch + 1), fetchOnlyLeader = true) } @@ -430,8 +430,8 @@ class PartitionTest extends AbstractPartitionTest { assertFetchOffsetError(Errors.FENCED_LEADER_EPOCH, Optional.of(leaderEpoch - 1), fetchOnlyLeader = false) assertFetchOffsetError(Errors.UNKNOWN_LEADER_EPOCH, Optional.of(leaderEpoch + 1), fetchOnlyLeader = false) - assertFetchOffsetError(Errors.NOT_LEADER_OR_FOLLOWER, Optional.empty(), fetchOnlyLeader = true) - assertFetchOffsetError(Errors.NOT_LEADER_OR_FOLLOWER, Optional.of(leaderEpoch), fetchOnlyLeader = true) + assertFetchOffsetError(Errors.NOT_LEADER_FOR_PARTITION, Optional.empty(), fetchOnlyLeader = true) + assertFetchOffsetError(Errors.NOT_LEADER_FOR_PARTITION, Optional.of(leaderEpoch), fetchOnlyLeader = true) assertFetchOffsetError(Errors.FENCED_LEADER_EPOCH, Optional.of(leaderEpoch - 1), fetchOnlyLeader = true) assertFetchOffsetError(Errors.UNKNOWN_LEADER_EPOCH, Optional.of(leaderEpoch + 1), fetchOnlyLeader = true) } @@ -783,14 +783,14 @@ class PartitionTest extends AbstractPartitionTest { @Test def testGetReplica(): Unit = { assertEquals(None, partition.log) - assertThrows[NotLeaderOrFollowerException] { + assertThrows[ReplicaNotAvailableException] { partition.localLogOrException } } @Test def testAppendRecordsToFollowerWithNoReplicaThrowsException(): Unit = { - assertThrows[NotLeaderOrFollowerException] { + assertThrows[ReplicaNotAvailableException] { partition.appendRecordsToFollowerOrFutureReplica( createRecords(List(new SimpleRecord("k1".getBytes, "v1".getBytes)), baseOffset = 0L), isFuture = false) } diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala index 12fcf7146b79f..10e1f2cd58cc9 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala @@ -1018,7 +1018,7 @@ class GroupMetadataManagerTest { assertStoreGroupErrorMapping(Errors.UNKNOWN_TOPIC_OR_PARTITION, Errors.COORDINATOR_NOT_AVAILABLE) assertStoreGroupErrorMapping(Errors.NOT_ENOUGH_REPLICAS, Errors.COORDINATOR_NOT_AVAILABLE) assertStoreGroupErrorMapping(Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND, Errors.COORDINATOR_NOT_AVAILABLE) - assertStoreGroupErrorMapping(Errors.NOT_LEADER_OR_FOLLOWER, Errors.NOT_COORDINATOR) + assertStoreGroupErrorMapping(Errors.NOT_LEADER_FOR_PARTITION, Errors.NOT_COORDINATOR) assertStoreGroupErrorMapping(Errors.MESSAGE_TOO_LARGE, Errors.UNKNOWN_SERVER_ERROR) assertStoreGroupErrorMapping(Errors.RECORD_LIST_TOO_LARGE, Errors.UNKNOWN_SERVER_ERROR) assertStoreGroupErrorMapping(Errors.INVALID_FETCH_SIZE, Errors.UNKNOWN_SERVER_ERROR) @@ -1296,7 +1296,7 @@ class GroupMetadataManagerTest { assertCommitOffsetErrorMapping(Errors.UNKNOWN_TOPIC_OR_PARTITION, Errors.COORDINATOR_NOT_AVAILABLE) assertCommitOffsetErrorMapping(Errors.NOT_ENOUGH_REPLICAS, Errors.COORDINATOR_NOT_AVAILABLE) assertCommitOffsetErrorMapping(Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND, Errors.COORDINATOR_NOT_AVAILABLE) - assertCommitOffsetErrorMapping(Errors.NOT_LEADER_OR_FOLLOWER, Errors.NOT_COORDINATOR) + assertCommitOffsetErrorMapping(Errors.NOT_LEADER_FOR_PARTITION, Errors.NOT_COORDINATOR) assertCommitOffsetErrorMapping(Errors.MESSAGE_TOO_LARGE, Errors.INVALID_COMMIT_OFFSET_SIZE) assertCommitOffsetErrorMapping(Errors.RECORD_LIST_TOO_LARGE, Errors.INVALID_COMMIT_OFFSET_SIZE) assertCommitOffsetErrorMapping(Errors.INVALID_FETCH_SIZE, Errors.INVALID_COMMIT_OFFSET_SIZE) diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandlerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandlerTest.scala index 07b2fa8d6cd12..5ae961b648e00 100644 --- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandlerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandlerTest.scala @@ -177,8 +177,8 @@ class TransactionMarkerRequestCompletionHandlerTest { } @Test - def shouldRetryPartitionWhenNotLeaderOrFollowerError(): Unit = { - verifyRetriesPartitionOnError(Errors.NOT_LEADER_OR_FOLLOWER) + def shouldRetryPartitionWhenNotLeaderForPartitionError(): Unit = { + verifyRetriesPartitionOnError(Errors.NOT_LEADER_FOR_PARTITION) } @Test diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala index 2a8d046de7fe0..30557421b3c4d 100644 --- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala @@ -366,7 +366,7 @@ class TransactionStateManagerTest { expectedError = Errors.NOT_COORDINATOR var failedMetadata = txnMetadata1.prepareAddPartitions(Set[TopicPartition](new TopicPartition("topic2", 0)), time.milliseconds()) - prepareForTxnMessageAppend(Errors.NOT_LEADER_OR_FOLLOWER) + prepareForTxnMessageAppend(Errors.NOT_LEADER_FOR_PARTITION) transactionManager.appendTransactionToLog(transactionalId1, coordinatorEpoch = 10, failedMetadata, assertCallback) assertEquals(Right(Some(CoordinatorEpochAndTxnMetadata(coordinatorEpoch, txnMetadata1))), transactionManager.getTransactionState(transactionalId1)) assertTrue(txnMetadata1.pendingState.isEmpty) diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala index 6ddfd3b73cbfc..b5c694a6ce1c4 100644 --- a/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala @@ -23,9 +23,8 @@ import org.apache.kafka.common.quota.{ClientQuotaAlteration, ClientQuotaEntity, import org.apache.kafka.common.requests.{AlterClientQuotasRequest, AlterClientQuotasResponse, DescribeClientQuotasRequest, DescribeClientQuotasResponse} import org.junit.Assert._ import org.junit.Test -import java.util.concurrent.{ExecutionException, TimeUnit} -import org.apache.kafka.common.security.scram.internals.{ScramCredentialUtils, ScramFormatter, ScramMechanism} +import java.util.concurrent.{ExecutionException, TimeUnit} import scala.jdk.CollectionConverters._ @@ -38,7 +37,6 @@ class ClientQuotasRequestTest extends BaseRequestTest { @Test def testAlterClientQuotasRequest(): Unit = { - val entity = new ClientQuotaEntity(Map((ClientQuotaEntity.USER -> "user"), (ClientQuotaEntity.CLIENT_ID -> "client-id")).asJava) // Expect an empty configuration. @@ -164,32 +162,6 @@ class ClientQuotasRequestTest extends BaseRequestTest { )) } - @Test - def testClientQuotasForScramUsers(): Unit = { - val entityType = ConfigType.User - val userName = "user" - - val mechanism = ScramMechanism.SCRAM_SHA_256 - val credential = new ScramFormatter(mechanism).generateCredential("password", 4096) - val configs = adminZkClient.fetchEntityConfig(entityType, userName) - configs.setProperty(mechanism.mechanismName, ScramCredentialUtils.credentialToString(credential)) - adminZkClient.changeConfigs(entityType, userName, configs) - - val entity = new ClientQuotaEntity(Map(ClientQuotaEntity.USER -> userName).asJava) - - verifyDescribeEntityQuotas(entity, Map.empty) - - alterEntityQuotas(entity, Map( - (ProducerByteRateProp -> Some(10000.0)), - (ConsumerByteRateProp -> Some(20000.0)) - ), validateOnly = false) - - verifyDescribeEntityQuotas(entity, Map( - (ProducerByteRateProp -> 10000.0), - (ConsumerByteRateProp -> 20000.0) - )) - } - @Test(expected = classOf[InvalidRequestException]) def testAlterClientQuotasBadUser(): Unit = { val entity = new ClientQuotaEntity(Map((ClientQuotaEntity.USER -> "")).asJava) diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala index 6561a487ebc49..f0c4329ff8067 100644 --- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala @@ -209,7 +209,7 @@ class FetchRequestTest extends BaseRequestTest { Seq(topicPartition))).build() val fetchResponse = sendFetchRequest(nonReplicaId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, partitionData.error) + assertEquals(Errors.REPLICA_NOT_AVAILABLE, partitionData.error) } @Test diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 0c4970c85cac3..3832a698debdc 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -775,8 +775,8 @@ class KafkaApisTest { } @Test - def testLeaderReplicaIfLocalRaisesNotLeaderOrFollower(): Unit = { - testListOffsetFailedGetLeaderReplica(Errors.NOT_LEADER_OR_FOLLOWER) + def testLeaderReplicaIfLocalRaisesNotLeaderForPartition(): Unit = { + testListOffsetFailedGetLeaderReplica(Errors.NOT_LEADER_FOR_PARTITION) } @Test diff --git a/core/src/test/scala/unit/kafka/server/ListOffsetsRequestTest.scala b/core/src/test/scala/unit/kafka/server/ListOffsetsRequestTest.scala index 2600cceccfee0..920176ed647ca 100644 --- a/core/src/test/scala/unit/kafka/server/ListOffsetsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/ListOffsetsRequestTest.scala @@ -64,14 +64,14 @@ class ListOffsetsRequestTest extends BaseRequestTest { val nonReplica = servers.map(_.config.brokerId).find(!replicas.contains(_)).get // Follower - assertResponseError(Errors.NOT_LEADER_OR_FOLLOWER, follower, consumerRequest) - assertResponseError(Errors.NOT_LEADER_OR_FOLLOWER, follower, replicaRequest) + assertResponseError(Errors.NOT_LEADER_FOR_PARTITION, follower, consumerRequest) + assertResponseError(Errors.NOT_LEADER_FOR_PARTITION, follower, replicaRequest) assertResponseError(Errors.NONE, follower, debugReplicaRequest) // Non-replica - assertResponseError(Errors.NOT_LEADER_OR_FOLLOWER, nonReplica, consumerRequest) - assertResponseError(Errors.NOT_LEADER_OR_FOLLOWER, nonReplica, replicaRequest) - assertResponseError(Errors.NOT_LEADER_OR_FOLLOWER, nonReplica, debugReplicaRequest) + assertResponseError(Errors.NOT_LEADER_FOR_PARTITION, nonReplica, consumerRequest) + assertResponseError(Errors.NOT_LEADER_FOR_PARTITION, nonReplica, replicaRequest) + assertResponseError(Errors.REPLICA_NOT_AVAILABLE, nonReplica, debugReplicaRequest) } @Test @@ -105,8 +105,8 @@ class ListOffsetsRequestTest extends BaseRequestTest { // Check follower error codes val followerId = TestUtils.findFollowerId(topicPartition, servers) - assertResponseErrorForEpoch(Errors.NOT_LEADER_OR_FOLLOWER, followerId, Optional.empty()) - assertResponseErrorForEpoch(Errors.NOT_LEADER_OR_FOLLOWER, followerId, Optional.of(secondLeaderEpoch)) + assertResponseErrorForEpoch(Errors.NOT_LEADER_FOR_PARTITION, followerId, Optional.empty()) + assertResponseErrorForEpoch(Errors.NOT_LEADER_FOR_PARTITION, followerId, Optional.of(secondLeaderEpoch)) assertResponseErrorForEpoch(Errors.UNKNOWN_LEADER_EPOCH, followerId, Optional.of(secondLeaderEpoch + 1)) assertResponseErrorForEpoch(Errors.FENCED_LEADER_EPOCH, followerId, Optional.of(secondLeaderEpoch - 1)) } diff --git a/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala b/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala index 9a985af1279cf..8ac65b46b17bb 100644 --- a/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala +++ b/core/src/test/scala/unit/kafka/server/LogDirFailureTest.scala @@ -27,7 +27,7 @@ import kafka.utils.{CoreUtils, Exit, TestUtils} import org.apache.kafka.clients.consumer.KafkaConsumer import org.apache.kafka.clients.producer.{ProducerConfig, ProducerRecord} import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.errors.{KafkaStorageException, NotLeaderOrFollowerException} +import org.apache.kafka.common.errors.{KafkaStorageException, NotLeaderForPartitionException} import org.apache.kafka.common.utils.Utils import org.junit.Assert.{assertEquals, assertFalse, assertTrue} import org.junit.{Before, Test} @@ -150,16 +150,16 @@ class LogDirFailureTest extends IntegrationTestHarness { causeLogDirFailure(failureType, leaderServer, partition) - // send() should fail due to either KafkaStorageException or NotLeaderOrFollowerException + // send() should fail due to either KafkaStorageException or NotLeaderForPartitionException try { producer.send(record).get(6000, TimeUnit.MILLISECONDS) - fail("send() should fail with either KafkaStorageException or NotLeaderOrFollowerException") + fail("send() should fail with either KafkaStorageException or NotLeaderForPartitionException") } catch { case e: ExecutionException => e.getCause match { case t: KafkaStorageException => - case t: NotLeaderOrFollowerException => // This may happen if ProduceRequest version <= 3 - case t: Throwable => fail(s"send() should fail with either KafkaStorageException or NotLeaderOrFollowerException instead of ${t.toString}") + case t: NotLeaderForPartitionException => // This may happen if ProduceRequest version <= 3 + case t: Throwable => fail(s"send() should fail with either KafkaStorageException or NotLeaderForPartitionException instead of ${t.toString}") } } } diff --git a/core/src/test/scala/unit/kafka/server/OffsetsForLeaderEpochRequestTest.scala b/core/src/test/scala/unit/kafka/server/OffsetsForLeaderEpochRequestTest.scala index 7c008fc69e7dc..8fc9cdbab11a5 100644 --- a/core/src/test/scala/unit/kafka/server/OffsetsForLeaderEpochRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/OffsetsForLeaderEpochRequestTest.scala @@ -48,8 +48,8 @@ class OffsetsForLeaderEpochRequestTest extends BaseRequestTest { val follower = replicas.find(_ != leader).get val nonReplica = servers.map(_.config.brokerId).find(!replicas.contains(_)).get - assertResponseError(Errors.NOT_LEADER_OR_FOLLOWER, follower, request) - assertResponseError(Errors.NOT_LEADER_OR_FOLLOWER, nonReplica, request) + assertResponseError(Errors.NOT_LEADER_FOR_PARTITION, follower, request) + assertResponseError(Errors.NOT_LEADER_FOR_PARTITION, nonReplica, request) } @Test @@ -81,8 +81,8 @@ class OffsetsForLeaderEpochRequestTest extends BaseRequestTest { // Check follower error codes val followerId = TestUtils.findFollowerId(topicPartition, servers) - assertResponseErrorForEpoch(Errors.NOT_LEADER_OR_FOLLOWER, followerId, Optional.empty()) - assertResponseErrorForEpoch(Errors.NOT_LEADER_OR_FOLLOWER, followerId, Optional.of(secondLeaderEpoch)) + assertResponseErrorForEpoch(Errors.NOT_LEADER_FOR_PARTITION, followerId, Optional.empty()) + assertResponseErrorForEpoch(Errors.NOT_LEADER_FOR_PARTITION, followerId, Optional.of(secondLeaderEpoch)) assertResponseErrorForEpoch(Errors.UNKNOWN_LEADER_EPOCH, followerId, Optional.of(secondLeaderEpoch + 1)) assertResponseErrorForEpoch(Errors.FENCED_LEADER_EPOCH, followerId, Optional.of(secondLeaderEpoch - 1)) } diff --git a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala index 4368442a4ca74..d3a871a04394a 100644 --- a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala @@ -125,7 +125,7 @@ class ProduceRequestTest extends BaseRequestTest { val produceResponse = sendProduceRequest(nonReplicaId, produceRequest) assertEquals(1, produceResponse.responses.size) - assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, produceResponse.responses.asScala.head._2.error) + assertEquals(Errors.NOT_LEADER_FOR_PARTITION, produceResponse.responses.asScala.head._2.error) } /* returns a pair of partition id and leader id */ diff --git a/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala index b68723de59c03..f1b6a9aa56183 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaAlterLogDirsThreadTest.scala @@ -92,7 +92,7 @@ class ReplicaAlterLogDirsThreadTest { when(replicaManager.futureLocalLogOrException(t1p0)).thenReturn(futureLog) when(replicaManager.futureLogExists(t1p0)).thenReturn(true) when(replicaManager.nonOfflinePartition(t1p0)).thenReturn(Some(partition)) - when(replicaManager.getPartitionOrException(t1p0)).thenReturn(partition) + when(replicaManager.getPartitionOrException(t1p0, expectLeader = false)).thenReturn(partition) when(quotaManager.isQuotaExceeded).thenReturn(false) @@ -181,7 +181,7 @@ class ReplicaAlterLogDirsThreadTest { when(replicaManager.futureLocalLogOrException(t1p0)).thenReturn(futureLog) when(replicaManager.futureLogExists(t1p0)).thenReturn(true) when(replicaManager.nonOfflinePartition(t1p0)).thenReturn(Some(partition)) - when(replicaManager.getPartitionOrException(t1p0)).thenReturn(partition) + when(replicaManager.getPartitionOrException(t1p0, expectLeader = false)).thenReturn(partition) when(quotaManager.isQuotaExceeded).thenReturn(false) @@ -266,13 +266,13 @@ class ReplicaAlterLogDirsThreadTest { val leoT1p1 = 232 //Stubs - expect(replicaManager.getPartitionOrException(t1p0)) + expect(replicaManager.getPartitionOrException(t1p0, expectLeader = false)) .andStubReturn(partitionT1p0) expect(partitionT1p0.lastOffsetForLeaderEpoch(Optional.empty(), leaderEpochT1p0, fetchOnlyFromLeader = false)) .andReturn(new EpochEndOffset(leaderEpochT1p0, leoT1p0)) .anyTimes() - expect(replicaManager.getPartitionOrException(t1p1)) + expect(replicaManager.getPartitionOrException(t1p1, expectLeader = false)) .andStubReturn(partitionT1p1) expect(partitionT1p1.lastOffsetForLeaderEpoch(Optional.empty(), leaderEpochT1p1, fetchOnlyFromLeader = false)) .andReturn(new EpochEndOffset(leaderEpochT1p1, leoT1p1)) @@ -315,13 +315,13 @@ class ReplicaAlterLogDirsThreadTest { val leo = 13 //Stubs - expect(replicaManager.getPartitionOrException(t1p0)) + expect(replicaManager.getPartitionOrException(t1p0, expectLeader = false)) .andStubReturn(partitionT1p0) expect(partitionT1p0.lastOffsetForLeaderEpoch(Optional.empty(), leaderEpoch, fetchOnlyFromLeader = false)) .andReturn(new EpochEndOffset(leaderEpoch, leo)) .anyTimes() - expect(replicaManager.getPartitionOrException(t1p1)) + expect(replicaManager.getPartitionOrException(t1p1, expectLeader = false)) .andThrow(new KafkaStorageException).once() replay(partitionT1p0, replicaManager) @@ -375,9 +375,9 @@ class ReplicaAlterLogDirsThreadTest { val replicaT1p1LEO = 192 //Stubs - expect(replicaManager.getPartitionOrException(t1p0)) + expect(replicaManager.getPartitionOrException(t1p0, expectLeader = false)) .andStubReturn(partitionT1p0) - expect(replicaManager.getPartitionOrException(t1p1)) + expect(replicaManager.getPartitionOrException(t1p1, expectLeader = false)) .andStubReturn(partitionT1p1) expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLogT1p0) expect(replicaManager.futureLogExists(t1p0)).andStubReturn(true) @@ -452,7 +452,7 @@ class ReplicaAlterLogDirsThreadTest { val futureReplicaEpochEndOffset = 191 //Stubs - expect(replicaManager.getPartitionOrException(t1p0)) + expect(replicaManager.getPartitionOrException(t1p0, expectLeader = false)) .andStubReturn(partition) expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLog) expect(replicaManager.futureLogExists(t1p0)).andStubReturn(true) @@ -522,7 +522,7 @@ class ReplicaAlterLogDirsThreadTest { val initialFetchOffset = 100 //Stubs - expect(replicaManager.getPartitionOrException(t1p0)) + expect(replicaManager.getPartitionOrException(t1p0, expectLeader = false)) .andStubReturn(partition) expect(partition.truncateTo(capture(truncated), isFuture = EasyMock.eq(true))).anyTimes() expect(replicaManager.futureLocalLogOrException(t1p0)).andStubReturn(futureLog) @@ -577,7 +577,7 @@ class ReplicaAlterLogDirsThreadTest { val replicaLEO = 300 //Stubs - expect(replicaManager.getPartitionOrException(t1p0)) + expect(replicaManager.getPartitionOrException(t1p0, expectLeader = false)) .andStubReturn(partition) expect(partition.truncateTo(capture(truncated), isFuture = EasyMock.eq(true))).once() @@ -655,7 +655,7 @@ class ReplicaAlterLogDirsThreadTest { val futureReplicaLEO = 190 val replicaLEO = 213 - expect(replicaManager.getPartitionOrException(t1p0)) + expect(replicaManager.getPartitionOrException(t1p0, expectLeader = false)) .andStubReturn(partition) expect(partition.lastOffsetForLeaderEpoch(Optional.of(1), leaderEpoch, fetchOnlyFromLeader = false)) .andReturn(new EpochEndOffset(leaderEpoch, replicaLEO)) diff --git a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala index cb6b68f48579e..1ab2054137e94 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala @@ -600,7 +600,7 @@ class ReplicaFetcherThreadTest { //Define the offsets for the OffsetsForLeaderEpochResponse, these are used for truncation val offsetsReply = mutable.Map( - t1p0 -> new EpochEndOffset(NOT_LEADER_OR_FOLLOWER, EpochEndOffset.UNDEFINED_EPOCH, EpochEndOffset.UNDEFINED_EPOCH_OFFSET), + t1p0 -> new EpochEndOffset(NOT_LEADER_FOR_PARTITION, EpochEndOffset.UNDEFINED_EPOCH, EpochEndOffset.UNDEFINED_EPOCH_OFFSET), t1p1 -> new EpochEndOffset(UNKNOWN_SERVER_ERROR, EpochEndOffset.UNDEFINED_EPOCH, EpochEndOffset.UNDEFINED_EPOCH_OFFSET) ).asJava diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala index fe4b2ca408e91..771bc7d801c7c 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala @@ -165,7 +165,8 @@ class ReplicaManagerQuotasTest { .andReturn(offsetSnapshot) val replicaManager: ReplicaManager = EasyMock.createMock(classOf[ReplicaManager]) - EasyMock.expect(replicaManager.getPartitionOrException(EasyMock.anyObject[TopicPartition])) + EasyMock.expect(replicaManager.getPartitionOrException( + EasyMock.anyObject[TopicPartition], EasyMock.anyBoolean())) .andReturn(partition).anyTimes() EasyMock.expect(replicaManager.shouldLeaderThrottle(EasyMock.anyObject[ReplicaQuota], EasyMock.anyObject[Partition], EasyMock.anyObject[Int])) diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 17b3278c92f8e..4c41abdc732d0 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -23,9 +23,10 @@ import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference} import java.util.concurrent.{CountDownLatch, TimeUnit} import java.util.{Optional, Properties} -import kafka.api._ +import kafka.api.LeaderAndIsr +import kafka.api.Request import kafka.log.{AppendOrigin, Log, LogConfig, LogManager, ProducerStateManager} -import kafka.cluster.{BrokerEndPoint, Partition} +import kafka.cluster.BrokerEndPoint import kafka.log.LeaderOffsetIncremented import kafka.server.QuotaFactory.UnboundedQuota import kafka.server.checkpoints.LazyOffsetCheckpoints @@ -186,12 +187,12 @@ class ReplicaManagerTest { .setIsNew(false)).asJava, Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build() rm.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ()) - rm.getPartitionOrException(new TopicPartition(topic, 0)) + rm.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true) .localLogOrException val records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("first message".getBytes())) val appendResult = appendRecords(rm, new TopicPartition(topic, 0), records).onFire { response => - assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, response.error) + assertEquals(Errors.NOT_LEADER_FOR_PARTITION, response.error) } // Make this replica the follower @@ -245,7 +246,7 @@ class ReplicaManagerTest { Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build() replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0), (_, _) => ()) - val partition = replicaManager.getPartitionOrException(new TopicPartition(topic, 0)) + val partition = replicaManager.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true) assertEquals(1, replicaManager.logManager.liveLogDirs.filterNot(_ == partition.log.get.dir.getParentFile).size) val previousReplicaFolder = partition.log.get.dir.getParentFile @@ -303,7 +304,7 @@ class ReplicaManagerTest { .setIsNew(true)).asJava, Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build() replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ()) - replicaManager.getPartitionOrException(new TopicPartition(topic, 0)) + replicaManager.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true) .localLogOrException val producerId = 234L @@ -363,7 +364,7 @@ class ReplicaManagerTest { .setIsNew(true)).asJava, Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build() replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ()) - replicaManager.getPartitionOrException(new TopicPartition(topic, 0)) + replicaManager.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true) .localLogOrException val producerId = 234L @@ -469,7 +470,7 @@ class ReplicaManagerTest { .setIsNew(true)).asJava, Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build() replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ()) - replicaManager.getPartitionOrException(new TopicPartition(topic, 0)) + replicaManager.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true) .localLogOrException val producerId = 234L @@ -545,7 +546,7 @@ class ReplicaManagerTest { .setIsNew(false)).asJava, Set(new Node(0, "host1", 0), new Node(1, "host2", 1), new Node(2, "host2", 2)).asJava).build() rm.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ()) - rm.getPartitionOrException(new TopicPartition(topic, 0)) + rm.getPartitionOrException(new TopicPartition(topic, 0), expectLeader = true) .localLogOrException // Append a couple of messages. @@ -1084,7 +1085,7 @@ class ReplicaManagerTest { Optional.of(0)) fetchResult = sendConsumerFetch(replicaManager, tp0, partitionData, None) assertNotNull(fetchResult.get) - assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, fetchResult.get.error) + assertEquals(Errors.NOT_LEADER_FOR_PARTITION, fetchResult.get.error) } @Test @@ -1177,7 +1178,7 @@ class ReplicaManagerTest { replicaManager.becomeLeaderOrFollower(0, becomeFollowerRequest, (_, _) => ()) assertNotNull(fetchResult.get) - assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, fetchResult.get.error) + assertEquals(Errors.NOT_LEADER_FOR_PARTITION, fetchResult.get.error) } @Test @@ -1310,7 +1311,7 @@ class ReplicaManagerTest { .setLeaderEpoch(LeaderAndIsr.EpochDuringDelete))) assertNotNull(fetchResult.get) - assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, fetchResult.get.error) + assertEquals(Errors.NOT_LEADER_FOR_PARTITION, fetchResult.get.error) } @Test @@ -1349,7 +1350,7 @@ class ReplicaManagerTest { .setLeaderEpoch(LeaderAndIsr.EpochDuringDelete))) assertNotNull(produceResult.get) - assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, produceResult.get.error) + assertEquals(Errors.NOT_LEADER_FOR_PARTITION, produceResult.get.error) } private def sendProducerAppend(replicaManager: ReplicaManager, @@ -2123,31 +2124,4 @@ class ReplicaManagerTest { assertFalse(readLogStartOffsetCheckpoint().contains(tp0)) } } - - @Test - def testReplicaNotAvailable(): Unit = { - - def createReplicaManager(): ReplicaManager = { - val props = TestUtils.createBrokerConfig(1, TestUtils.MockZkConnect) - val config = KafkaConfig.fromProps(props) - val mockLogMgr = TestUtils.createLogManager(config.logDirs.map(new File(_))) - new ReplicaManager(config, metrics, time, kafkaZkClient, new MockScheduler(time), mockLogMgr, - new AtomicBoolean(false), QuotaFactory.instantiate(config, metrics, time, ""), new BrokerTopicStats, - new MetadataCache(config.brokerId), new LogDirFailureChannel(config.logDirs.size)) { - override def getPartitionOrException(topicPartition: TopicPartition): Partition = { - throw Errors.NOT_LEADER_OR_FOLLOWER.exception() - } - } - } - - val replicaManager = createReplicaManager() - try { - val tp = new TopicPartition(topic, 0) - val dir = replicaManager.logManager.liveLogDirs.head.getAbsolutePath - val errors = replicaManager.alterReplicaLogDirs(Map(tp -> dir)) - assertEquals(Errors.REPLICA_NOT_AVAILABLE, errors(tp)) - } finally { - replicaManager.shutdown(false) - } - } } diff --git a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala index 40432b9a6f489..29da8d5eb8495 100644 --- a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala +++ b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala @@ -128,7 +128,7 @@ class LeaderEpochIntegrationTest extends ZooKeeperTestHarness with Logging { //And should get no leader for partition error from t1p1 (as it's not on broker 0) assertTrue(offsetsForEpochs(t1p1).hasError) - assertEquals(NOT_LEADER_OR_FOLLOWER, offsetsForEpochs(t1p1).error) + assertEquals(NOT_LEADER_FOR_PARTITION, offsetsForEpochs(t1p1).error) assertEquals(UNDEFINED_EPOCH_OFFSET, offsetsForEpochs(t1p1).endOffset) //Repointing to broker 1 we should get the correct offset for t1p1 diff --git a/core/src/test/scala/unit/kafka/server/epoch/OffsetsForLeaderEpochTest.scala b/core/src/test/scala/unit/kafka/server/epoch/OffsetsForLeaderEpochTest.scala index fd4e8d96c9098..08be8a28f7967 100644 --- a/core/src/test/scala/unit/kafka/server/epoch/OffsetsForLeaderEpochTest.scala +++ b/core/src/test/scala/unit/kafka/server/epoch/OffsetsForLeaderEpochTest.scala @@ -87,7 +87,7 @@ class OffsetsForLeaderEpochTest { val response = replicaManager.lastOffsetForLeaderEpoch(request) //Then - assertEquals(new EpochEndOffset(Errors.NOT_LEADER_OR_FOLLOWER, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET), response(tp)) + assertEquals(new EpochEndOffset(Errors.NOT_LEADER_FOR_PARTITION, UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET), response(tp)) } @Test diff --git a/docs/upgrade.html b/docs/upgrade.html index 676bc69369b46..2983cf7d44161 100644 --- a/docs/upgrade.html +++ b/docs/upgrade.html @@ -35,10 +35,6 @@
Notable changes in 2 KIP-602 for more details. -
  • NotLeaderForPartitionException has been deprecated and replaced with NotLeaderOrFollowerException. - Fetch requests and other requests intended only for the leader or follower return NOT_LEADER_OR_FOLLOWER(6) instead of REPLICA_NOT_AVAILABLE(9) - if the broker is not a replica, ensuring that this transient error during reassignments is handled by all clients as a retriable exception. -
    • Notable changes in 2.5.0
    diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java index aed7d968954ae..304ec30c3b23e 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java @@ -121,8 +121,8 @@ public int hashCode() { private static class ClientMetadata { private final HostInfo hostInfo; - private final ClientState state; private final SortedSet consumers; + private final ClientState state; ClientMetadata(final String endPoint) { @@ -734,10 +734,8 @@ private boolean assignTasksToClients(final Cluster fullMetadata, statefulTasks, assignmentConfigs); - log.info("Assigned tasks {} including stateful {} to clients as: \n{}.", - allTasks, statefulTasks, clientStates.entrySet().stream() - .map(entry -> entry.getKey() + "=" + entry.getValue().currentAssignment()) - .collect(Collectors.joining(Utils.NL))); + log.info("Assigned tasks to clients as {}{}.", + Utils.NL, clientStates.entrySet().stream().map(Map.Entry::toString).collect(Collectors.joining(Utils.NL))); return probingRebalanceNeeded; } @@ -943,9 +941,7 @@ private Map computeNewAssignment(final Set statefulT state ); - // Arbitrarily choose the leader's client to be responsible for triggering the probing rebalance, - // note once we pick the first consumer within the process to trigger probing rebalance, other consumer - // would not set to trigger any more. + // Arbitrarily choose the leader's client to be responsible for triggering the probing rebalance final boolean encodeNextProbingRebalanceTime = shouldTriggerProbingRebalance && clientId.equals(taskManager.processId()); final boolean tasksRevoked = addClientAssignments( @@ -967,19 +963,6 @@ private Map computeNewAssignment(final Set statefulT rebalanceRequired = true; log.debug("Requested client {} to schedule a followup rebalance", clientId); } - - log.info("Client {} per-consumer assignment:\n" + - "\tprev owned active {}\n" + - "\tprev owned standby {}\n" + - "\tassigned active {}\n" + - "\trevoking active {}" + - "\tassigned standby {}\n", - clientId, - clientMetadata.state.prevOwnedActiveTasksByConsumer(), - clientMetadata.state.prevOwnedStandbyByConsumer(), - clientMetadata.state.assignedActiveTasksByConsumer(), - clientMetadata.state.revokingActiveTasksByConsumer(), - clientMetadata.state.assignedStandbyTasksByConsumer()); } if (rebalanceRequired) { @@ -1053,15 +1036,14 @@ private boolean addClientAssignments(final Set statefulTasks, if (!activeTasksRemovedPendingRevokation.isEmpty()) { // TODO: once KAFKA-10078 is resolved we can leave it to the client to trigger this rebalance - log.info("Requesting {} followup rebalance be scheduled immediately due to tasks changing ownership.", consumer); + log.info("Requesting followup rebalance be scheduled immediately due to tasks changing ownership."); info.setNextRebalanceTime(0L); followupRebalanceRequiredForRevokedTasks = true; // Don't bother to schedule a probing rebalance if an immediate one is already scheduled shouldEncodeProbingRebalance = false; } else if (shouldEncodeProbingRebalance) { final long nextRebalanceTimeMs = time.milliseconds() + probingRebalanceIntervalMs(); - log.info("Requesting {} followup rebalance be scheduled for {} ms to probe for caught-up replica tasks.", - consumer, nextRebalanceTimeMs); + log.info("Requesting followup rebalance be scheduled for {} ms to probe for caught-up replica tasks.", nextRebalanceTimeMs); info.setNextRebalanceTime(nextRebalanceTimeMs); shouldEncodeProbingRebalance = false; } @@ -1092,11 +1074,8 @@ private Set populateActiveTaskAndPartitionsLists(final List assignedPartitions = new ArrayList<>(); final Set removedActiveTasks = new TreeSet<>(); + // Build up list of all assigned partition-task pairs for (final TaskId taskId : activeTasksForConsumer) { - // Populate the consumer for assigned tasks without considering revocation, - // this is for debugging purposes only - clientState.assignActiveToConsumer(taskId, consumer); - final List assignedPartitionsForTask = new ArrayList<>(); for (final TopicPartition partition : partitionsForTask.get(taskId)) { final String oldOwner = clientState.previousOwnerForPartition(partition); @@ -1112,8 +1091,6 @@ private Set populateActiveTaskAndPartitionsLists(final List> buildStandbyTaskMap(final String consum final Map> partitionsForTask, final ClientState clientState) { final Map> standbyTaskMap = new HashMap<>(); - for (final TaskId task : standbyTasks) { - clientState.assignStandbyToConsumer(task, consumer); standbyTaskMap.put(task, partitionsForTask.get(task)); } - for (final TaskId task : revokedTasks) { if (allStatefulTasks.contains(task)) { - log.info("Adding removed stateful active task {} as a standby for {} before it is revoked in followup rebalance", - task, consumer); + log.info("Adding removed stateful active task {} as a standby for {} before it is safely revoked in followup rebalance", + task, consumer + ); + standbyTaskMap.put(task, partitionsForTask.get(task)); // This has no effect on the assignment, as we'll never consult the ClientState again, but // it does perform a useful assertion that the it's legal to assign this task as a standby to this instance - clientState.assignStandbyToConsumer(task, consumer); clientState.assignStandby(task); - - standbyTaskMap.put(task, partitionsForTask.get(task)); } } return standbyTaskMap; @@ -1289,7 +1262,7 @@ static Map> assignTasksToThreads(final Collection s private static SortedSet getPreviousTasksByLag(final ClientState state, final String consumer) { final SortedSet prevTasksByLag = new TreeSet<>(comparingLong(state::lagFor).thenComparing(TaskId::compareTo)); - prevTasksByLag.addAll(state.prevOwnedStatefulTasksByConsumer(consumer)); + prevTasksByLag.addAll(state.previousTasksForConsumer(consumer)); return prevTasksByLag; } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/ClientState.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/ClientState.java index 9be0b24c170c2..c98ed9b8af542 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/ClientState.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/ClientState.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.streams.processor.internals.assignment; +import java.util.stream.Collectors; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.streams.processor.TaskId; import org.apache.kafka.streams.processor.internals.Task; @@ -27,7 +28,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; +import java.util.SortedMap; import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; @@ -43,23 +44,15 @@ public class ClientState { private static final Logger LOG = LoggerFactory.getLogger(ClientState.class); public static final Comparator TOPIC_PARTITION_COMPARATOR = comparing(TopicPartition::topic).thenComparing(TopicPartition::partition); - private final Set activeTasks = new TreeSet<>(); - private final Set standbyTasks = new TreeSet<>(); + private final Set activeTasks; + private final Set standbyTasks; private final Set prevActiveTasks; private final Set prevStandbyTasks; + private final Map> consumerToPreviousStatefulTaskIds; + private final Map ownedPartitions; private final Map taskOffsetSums; // contains only stateful tasks we previously owned private final Map taskLagTotals; // contains lag for all stateful tasks in the app topology - private final Map ownedPartitions = new TreeMap<>(TOPIC_PARTITION_COMPARATOR); - private final Map> consumerToPreviousStatefulTaskIds = new TreeMap<>(); - - // the following four maps are used only for logging purposes; - // TODO KAFKA-10283: we could consider merging them with other book-keeping maps at client-levels - // so that they would not be inconsistent - private final Map> consumerToPreviousActiveTaskIds = new TreeMap<>(); - private final Map> consumerToAssignedActiveTaskIds = new TreeMap<>(); - private final Map> consumerToAssignedStandbyTaskIds = new TreeMap<>(); - private final Map> consumerToRevokingActiveTaskIds = new TreeMap<>(); private int capacity; @@ -68,25 +61,67 @@ public ClientState() { } ClientState(final int capacity) { + activeTasks = new TreeSet<>(); + standbyTasks = new TreeSet<>(); prevActiveTasks = new TreeSet<>(); prevStandbyTasks = new TreeSet<>(); + consumerToPreviousStatefulTaskIds = new TreeMap<>(); + ownedPartitions = new TreeMap<>(TOPIC_PARTITION_COMPARATOR); taskOffsetSums = new TreeMap<>(); taskLagTotals = new TreeMap<>(); this.capacity = capacity; } - // For testing only + private ClientState(final Set activeTasks, + final Set standbyTasks, + final Set prevActiveTasks, + final Set prevStandbyTasks, + final Map> consumerToPreviousStatefulTaskIds, + final SortedMap ownedPartitions, + final Map taskOffsetSums, + final Map taskLagTotals, + final int capacity) { + this.activeTasks = activeTasks; + this.standbyTasks = standbyTasks; + this.prevActiveTasks = prevActiveTasks; + this.prevStandbyTasks = prevStandbyTasks; + this.consumerToPreviousStatefulTaskIds = consumerToPreviousStatefulTaskIds; + this.ownedPartitions = ownedPartitions; + this.taskOffsetSums = taskOffsetSums; + this.taskLagTotals = taskLagTotals; + this.capacity = capacity; + } + public ClientState(final Set previousActiveTasks, final Set previousStandbyTasks, final Map taskLagTotals, final int capacity) { + activeTasks = new TreeSet<>(); + standbyTasks = new TreeSet<>(); prevActiveTasks = unmodifiableSet(new TreeSet<>(previousActiveTasks)); prevStandbyTasks = unmodifiableSet(new TreeSet<>(previousStandbyTasks)); + consumerToPreviousStatefulTaskIds = new TreeMap<>(); + ownedPartitions = new TreeMap<>(TOPIC_PARTITION_COMPARATOR); taskOffsetSums = emptyMap(); this.taskLagTotals = unmodifiableMap(taskLagTotals); this.capacity = capacity; } + public ClientState copy() { + final TreeMap newOwnedPartitions = new TreeMap<>(TOPIC_PARTITION_COMPARATOR); + newOwnedPartitions.putAll(ownedPartitions); + return new ClientState( + new TreeSet<>(activeTasks), + new TreeSet<>(standbyTasks), + new TreeSet<>(prevActiveTasks), + new TreeSet<>(prevStandbyTasks), + new TreeMap<>(consumerToPreviousStatefulTaskIds), + newOwnedPartitions, + new TreeMap<>(taskOffsetSums), + new TreeMap<>(taskLagTotals), + capacity); + } + int capacity() { return capacity; } @@ -115,53 +150,6 @@ public void assignActiveTasks(final Collection tasks) { activeTasks.addAll(tasks); } - public void assignActiveToConsumer(final TaskId task, final String consumer) { - consumerToAssignedActiveTaskIds.computeIfAbsent(consumer, k -> new HashSet<>()).add(task); - } - - public void assignStandbyToConsumer(final TaskId task, final String consumer) { - consumerToAssignedStandbyTaskIds.computeIfAbsent(consumer, k -> new HashSet<>()).add(task); - } - - public void revokeActiveFromConsumer(final TaskId task, final String consumer) { - consumerToRevokingActiveTaskIds.computeIfAbsent(consumer, k -> new HashSet<>()).add(task); - } - - public Map> prevOwnedActiveTasksByConsumer() { - return consumerToPreviousActiveTaskIds; - } - - public Map> prevOwnedStandbyByConsumer() { - // standbys are just those stateful tasks minus active tasks - final Map> consumerToPreviousStandbyTaskIds = new TreeMap<>(); - - for (final Map.Entry> entry: consumerToPreviousStatefulTaskIds.entrySet()) { - final Set standbyTaskIds = new HashSet<>(entry.getValue()); - if (consumerToPreviousActiveTaskIds.containsKey(entry.getKey())) - standbyTaskIds.removeAll(consumerToPreviousActiveTaskIds.get(entry.getKey())); - consumerToPreviousStandbyTaskIds.put(entry.getKey(), standbyTaskIds); - } - - return consumerToPreviousStandbyTaskIds; - } - - // including both active and standby tasks - public Set prevOwnedStatefulTasksByConsumer(final String memberId) { - return consumerToPreviousStatefulTaskIds.get(memberId); - } - - public Map> assignedActiveTasksByConsumer() { - return consumerToAssignedActiveTaskIds; - } - - public Map> revokingActiveTasksByConsumer() { - return consumerToRevokingActiveTaskIds; - } - - public Map> assignedStandbyTasksByConsumer() { - return consumerToAssignedStandbyTaskIds; - } - public void assignActive(final TaskId task) { assertNotAssigned(task); activeTasks.add(task); @@ -328,6 +316,10 @@ public Set statelessActiveTasks() { return activeTasks.stream().filter(task -> !isStateful(task)).collect(Collectors.toSet()); } + public Set previousTasksForConsumer(final String memberId) { + return consumerToPreviousStatefulTaskIds.get(memberId); + } + boolean hasUnfulfilledQuota(final int tasksPerThread) { return activeTasks.size() < capacity * tasksPerThread; } @@ -353,17 +345,13 @@ boolean hasMoreAvailableCapacityThan(final ClientState other) { } } - public String currentAssignment() { - return "[activeTasks: (" + activeTasks + - ") standbyTasks: (" + standbyTasks + ")]"; - } - @Override public String toString() { return "[activeTasks: (" + activeTasks + ") standbyTasks: (" + standbyTasks + ") prevActiveTasks: (" + prevActiveTasks + ") prevStandbyTasks: (" + prevStandbyTasks + + ") prevOwnedPartitionsByConsumerId: (" + ownedPartitions.keySet() + ") changelogOffsetTotalsByTask: (" + taskOffsetSums.entrySet() + ") taskLagTotals: (" + taskLagTotals.entrySet() + ") capacity: " + capacity + @@ -385,7 +373,6 @@ private void initializePrevActiveTasksFromOwnedPartitions(final Map new HashSet<>()).add(task); } else { LOG.error("No task found for topic partition {}", tp); } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentTestUtils.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentTestUtils.java index d1da94f5b6249..f0139ffdcfd82 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentTestUtils.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentTestUtils.java @@ -53,13 +53,6 @@ public final class AssignmentTestUtils { public static final UUID UUID_5 = uuidForInt(5); public static final UUID UUID_6 = uuidForInt(6); - public static final TopicPartition TP_0_0 = new TopicPartition("topic0", 0); - public static final TopicPartition TP_0_1 = new TopicPartition("topic0", 1); - public static final TopicPartition TP_0_2 = new TopicPartition("topic0", 2); - public static final TopicPartition TP_1_0 = new TopicPartition("topic1", 0); - public static final TopicPartition TP_1_1 = new TopicPartition("topic1", 1); - public static final TopicPartition TP_1_2 = new TopicPartition("topic1", 2); - public static final TaskId TASK_0_0 = new TaskId(0, 0); public static final TaskId TASK_0_1 = new TaskId(0, 1); public static final TaskId TASK_0_2 = new TaskId(0, 2); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/ClientStateTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/ClientStateTest.java index 952a9904c6aa1..d50c00eae2617 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/ClientStateTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/ClientStateTest.java @@ -32,12 +32,7 @@ import static org.apache.kafka.streams.processor.internals.assignment.AssignmentTestUtils.TASK_0_1; import static org.apache.kafka.streams.processor.internals.assignment.AssignmentTestUtils.TASK_0_2; import static org.apache.kafka.streams.processor.internals.assignment.AssignmentTestUtils.TASK_0_3; -import static org.apache.kafka.streams.processor.internals.assignment.AssignmentTestUtils.TP_0_0; -import static org.apache.kafka.streams.processor.internals.assignment.AssignmentTestUtils.TP_0_1; -import static org.apache.kafka.streams.processor.internals.assignment.AssignmentTestUtils.TP_0_2; -import static org.apache.kafka.streams.processor.internals.assignment.AssignmentTestUtils.TP_1_0; -import static org.apache.kafka.streams.processor.internals.assignment.AssignmentTestUtils.TP_1_1; -import static org.apache.kafka.streams.processor.internals.assignment.AssignmentTestUtils.TP_1_2; +import static org.apache.kafka.streams.processor.internals.assignment.AssignmentTestUtils.UUID_1; import static org.apache.kafka.streams.processor.internals.assignment.AssignmentTestUtils.hasActiveTasks; import static org.apache.kafka.streams.processor.internals.assignment.AssignmentTestUtils.hasStandbyTasks; import static org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo.UNKNOWN_OFFSET_SUM; @@ -314,80 +309,35 @@ public void shouldAddTasksWithLatestOffsetToPrevActiveTasks() { @Test public void shouldReturnPreviousStatefulTasksForConsumer() { - client.addPreviousTasksAndOffsetSums("c1", mkMap( - mkEntry(TASK_0_0, 100L), - mkEntry(TASK_0_1, Task.LATEST_OFFSET) - )); + client.addPreviousTasksAndOffsetSums("c1", Collections.singletonMap(TASK_0_1, Task.LATEST_OFFSET)); client.addPreviousTasksAndOffsetSums("c2", Collections.singletonMap(TASK_0_2, 0L)); client.addPreviousTasksAndOffsetSums("c3", Collections.emptyMap()); client.initializePrevTasks(Collections.emptyMap()); + client.computeTaskLags( + UUID_1, + mkMap( + mkEntry(TASK_0_1, 1_000L), + mkEntry(TASK_0_2, 1_000L) + ) + ); - assertThat(client.prevOwnedStatefulTasksByConsumer("c1"), equalTo(mkSet(TASK_0_0, TASK_0_1))); - assertThat(client.prevOwnedStatefulTasksByConsumer("c2"), equalTo(mkSet(TASK_0_2))); - assertTrue(client.prevOwnedStatefulTasksByConsumer("c3").isEmpty()); + assertThat(client.previousTasksForConsumer("c1"), equalTo(mkSet(TASK_0_1))); + assertThat(client.previousTasksForConsumer("c2"), equalTo(mkSet(TASK_0_2))); + assertTrue(client.previousTasksForConsumer("c3").isEmpty()); } @Test - public void shouldReturnPreviousActiveStandbyTasksForConsumer() { - client.addOwnedPartitions(mkSet(TP_0_1, TP_1_1), "c1"); - client.addOwnedPartitions(mkSet(TP_0_2, TP_1_2), "c2"); - client.initializePrevTasks(mkMap( - mkEntry(TP_0_0, TASK_0_0), - mkEntry(TP_0_1, TASK_0_1), - mkEntry(TP_0_2, TASK_0_2), - mkEntry(TP_1_0, TASK_0_0), - mkEntry(TP_1_1, TASK_0_1), - mkEntry(TP_1_2, TASK_0_2)) - ); - + public void shouldReturnPreviousTasksForConsumer() { client.addPreviousTasksAndOffsetSums("c1", mkMap( - mkEntry(TASK_0_1, Task.LATEST_OFFSET), - mkEntry(TASK_0_0, 10L))); - client.addPreviousTasksAndOffsetSums("c2", Collections.singletonMap(TASK_0_2, 0L)); + mkEntry(TASK_0_1, 100L), + mkEntry(TASK_0_2, 0L), + mkEntry(TASK_0_3, Task.LATEST_OFFSET) + )); - assertThat(client.prevOwnedStatefulTasksByConsumer("c1"), equalTo(mkSet(TASK_0_1, TASK_0_0))); - assertThat(client.prevOwnedStatefulTasksByConsumer("c2"), equalTo(mkSet(TASK_0_2))); - assertThat(client.prevOwnedActiveTasksByConsumer(), equalTo( - mkMap( - mkEntry("c1", Collections.singleton(TASK_0_1)), - mkEntry("c2", Collections.singleton(TASK_0_2)) - )) - ); - assertThat(client.prevOwnedStandbyByConsumer(), equalTo( - mkMap( - mkEntry("c1", Collections.singleton(TASK_0_0)), - mkEntry("c2", Collections.emptySet()) - )) - ); - } + client.initializePrevTasks(Collections.emptyMap()); - @Test - public void shouldReturnAssignedTasksForConsumer() { - client.assignActiveToConsumer(TASK_0_0, "c1"); - // calling it multiple tasks should be idempotent - client.assignActiveToConsumer(TASK_0_0, "c1"); - client.assignActiveToConsumer(TASK_0_1, "c1"); - client.assignActiveToConsumer(TASK_0_2, "c2"); - - client.assignStandbyToConsumer(TASK_0_2, "c1"); - client.assignStandbyToConsumer(TASK_0_0, "c2"); - // calling it multiple tasks should be idempotent - client.assignStandbyToConsumer(TASK_0_0, "c2"); - - client.revokeActiveFromConsumer(TASK_0_1, "c1"); - // calling it multiple tasks should be idempotent - client.revokeActiveFromConsumer(TASK_0_1, "c1"); - - assertThat(client.assignedActiveTasksByConsumer(), equalTo(mkMap( - mkEntry("c1", mkSet(TASK_0_0, TASK_0_1)), - mkEntry("c2", mkSet(TASK_0_2)) - ))); - assertThat(client.assignedStandbyTasksByConsumer(), equalTo(mkMap( - mkEntry("c1", mkSet(TASK_0_2)), - mkEntry("c2", mkSet(TASK_0_0)) - ))); - assertThat(client.revokingActiveTasksByConsumer(), equalTo(Collections.singletonMap("c1", mkSet(TASK_0_1)))); + assertThat(client.previousTasksForConsumer("c1"), equalTo(mkSet(TASK_0_3, TASK_0_2, TASK_0_1))); } @Test diff --git a/tests/kafkatest/services/connect.py b/tests/kafkatest/services/connect.py index 6ee6aded19f3a..b05635820cdf2 100644 --- a/tests/kafkatest/services/connect.py +++ b/tests/kafkatest/services/connect.py @@ -44,15 +44,13 @@ class ConnectServiceBase(KafkaPathResolverMixin, Service): CONNECT_REST_PORT = 8083 HEAP_DUMP_FILE = os.path.join(PERSISTENT_ROOT, "connect_heap_dump.bin") - # Currently the Connect worker supports waiting on four modes: + # Currently the Connect worker supports waiting on three modes: STARTUP_MODE_INSTANT = 'INSTANT' """STARTUP_MODE_INSTANT: Start Connect worker and return immediately""" STARTUP_MODE_LOAD = 'LOAD' """STARTUP_MODE_LOAD: Start Connect worker and return after discovering and loading plugins""" STARTUP_MODE_LISTEN = 'LISTEN' """STARTUP_MODE_LISTEN: Start Connect worker and return after opening the REST port.""" - STARTUP_MODE_JOIN = 'JOIN' - """STARTUP_MODE_JOIN: Start Connect worker and return after joining the group.""" logs = { "connect_log": { @@ -117,9 +115,8 @@ def listening(self, node): self.logger.debug("REST resources are not loaded yet") return False - def start(self, mode=None): - if mode: - self.startup_mode = mode + def start(self, mode=STARTUP_MODE_LISTEN): + self.startup_mode = mode super(ConnectServiceBase, self).start() def start_and_return_immediately(self, node, worker_type, remote_connector_configs): @@ -140,15 +137,6 @@ def start_and_wait_to_start_listening(self, node, worker_type, remote_connector_ err_msg="Kafka Connect failed to start on node: %s in condition mode: %s" % (str(node.account), self.startup_mode)) - def start_and_wait_to_join_group(self, node, worker_type, remote_connector_configs): - if worker_type != 'distributed': - raise RuntimeError("Cannot wait for joined group message for %s" % worker_type) - with node.account.monitor_log(self.LOG_FILE) as monitor: - self.start_and_return_immediately(node, worker_type, remote_connector_configs) - monitor.wait_until('Joined group', timeout_sec=self.startup_timeout_sec, - err_msg="Never saw message indicating Kafka Connect joined group on node: " + - "%s in condition mode: %s" % (str(node.account), self.startup_mode)) - def stop_node(self, node, clean_shutdown=True): self.logger.info((clean_shutdown and "Cleanly" or "Forcibly") + " stopping Kafka Connect on " + str(node.account)) pids = self.pids(node) @@ -322,8 +310,6 @@ def start_node(self, node): self.start_and_wait_to_load_plugins(node, 'standalone', remote_connector_configs) elif self.startup_mode == self.STARTUP_MODE_INSTANT: self.start_and_return_immediately(node, 'standalone', remote_connector_configs) - elif self.startup_mode == self.STARTUP_MODE_JOIN: - self.start_and_wait_to_join_group(node, 'standalone', remote_connector_configs) else: # The default mode is to wait until the complete startup of the worker self.start_and_wait_to_start_listening(node, 'standalone', remote_connector_configs) @@ -338,7 +324,6 @@ class ConnectDistributedService(ConnectServiceBase): def __init__(self, context, num_nodes, kafka, files, offsets_topic="connect-offsets", configs_topic="connect-configs", status_topic="connect-status", startup_timeout_sec = 60): super(ConnectDistributedService, self).__init__(context, num_nodes, kafka, files, startup_timeout_sec) - self.startup_mode = self.STARTUP_MODE_JOIN self.offsets_topic = offsets_topic self.configs_topic = configs_topic self.status_topic = status_topic @@ -372,11 +357,9 @@ def start_node(self, node): self.start_and_wait_to_load_plugins(node, 'distributed', '') elif self.startup_mode == self.STARTUP_MODE_INSTANT: self.start_and_return_immediately(node, 'distributed', '') - elif self.startup_mode == self.STARTUP_MODE_LISTEN: - self.start_and_wait_to_start_listening(node, 'distributed', '') else: # The default mode is to wait until the complete startup of the worker - self.start_and_wait_to_join_group(node, 'distributed', '') + self.start_and_wait_to_start_listening(node, 'distributed', '') if len(self.pids(node)) == 0: raise RuntimeError("No process ids recorded") diff --git a/tests/kafkatest/tests/client/client_compatibility_features_test.py b/tests/kafkatest/tests/client/client_compatibility_features_test.py index 7a0d471398b0f..9c4b3061fd6b5 100644 --- a/tests/kafkatest/tests/client/client_compatibility_features_test.py +++ b/tests/kafkatest/tests/client/client_compatibility_features_test.py @@ -23,7 +23,7 @@ from kafkatest.services.zookeeper import ZookeeperService from kafkatest.services.kafka import KafkaService from ducktape.tests.test import Test -from kafkatest.version import DEV_BRANCH, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, LATEST_2_4, LATEST_2_5, V_0_11_0_0, V_0_10_1_0, KafkaVersion +from kafkatest.version import DEV_BRANCH, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, LATEST_2_4, V_0_11_0_0, V_0_10_1_0, KafkaVersion def get_broker_features(broker_version): features = {} @@ -115,7 +115,6 @@ def invoke_compatibility_program(self, features): @parametrize(broker_version=str(LATEST_2_2)) @parametrize(broker_version=str(LATEST_2_3)) @parametrize(broker_version=str(LATEST_2_4)) - @parametrize(broker_version=str(LATEST_2_5)) def run_compatibility_test(self, broker_version): self.zk.start() self.kafka.set_version(KafkaVersion(broker_version)) diff --git a/tests/kafkatest/tests/client/client_compatibility_produce_consume_test.py b/tests/kafkatest/tests/client/client_compatibility_produce_consume_test.py index 09cdadce05d8d..5f663ee3f7665 100644 --- a/tests/kafkatest/tests/client/client_compatibility_produce_consume_test.py +++ b/tests/kafkatest/tests/client/client_compatibility_produce_consume_test.py @@ -22,7 +22,7 @@ from kafkatest.services.console_consumer import ConsoleConsumer from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest from kafkatest.utils import is_int_with_prefix -from kafkatest.version import DEV_BRANCH, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, LATEST_2_4, LATEST_2_5, KafkaVersion +from kafkatest.version import DEV_BRANCH, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, LATEST_2_4, KafkaVersion class ClientCompatibilityProduceConsumeTest(ProduceConsumeValidateTest): """ @@ -64,7 +64,6 @@ def min_cluster_size(self): @parametrize(broker_version=str(LATEST_2_2)) @parametrize(broker_version=str(LATEST_2_3)) @parametrize(broker_version=str(LATEST_2_4)) - @parametrize(broker_version=str(LATEST_2_5)) def test_produce_consume(self, broker_version): print("running producer_consumer_compat with broker_version = %s" % broker_version) self.kafka.set_version(KafkaVersion(broker_version)) diff --git a/tests/kafkatest/tests/client/consumer_test.py b/tests/kafkatest/tests/client/consumer_test.py index 731a24de915dd..131123f55ff3c 100644 --- a/tests/kafkatest/tests/client/consumer_test.py +++ b/tests/kafkatest/tests/client/consumer_test.py @@ -254,9 +254,7 @@ def test_fencing_static_consumer(self, num_conflict_consumers, fencing_stage): self.await_members(conflict_consumer, num_conflict_consumers) self.await_members(consumer, len(consumer.nodes) - num_conflict_consumers) - wait_until(lambda: len(consumer.dead_nodes()) == num_conflict_consumers, - timeout_sec=10, - err_msg="Timed out waiting for the fenced consumers to stop") + assert len(consumer.dead_nodes()) == num_conflict_consumers else: consumer.start() conflict_consumer.start() diff --git a/tests/kafkatest/tests/connect/connect_distributed_test.py b/tests/kafkatest/tests/connect/connect_distributed_test.py index 107c0d4290597..244bc64836ff6 100644 --- a/tests/kafkatest/tests/connect/connect_distributed_test.py +++ b/tests/kafkatest/tests/connect/connect_distributed_test.py @@ -387,22 +387,8 @@ def test_bounce(self, clean, connect_protocol): # through the test. time.sleep(15) - # Wait at least scheduled.rebalance.max.delay.ms to expire and rebalance - time.sleep(60) - # Allow the connectors to startup, recover, and exit cleanly before - # ending the test. It's possible for the source connector to make - # uncommitted progress, and for the sink connector to read messages that - # have not been committed yet, and fail a later assertion. - wait_until(lambda: self.is_running(self.source), timeout_sec=30, - err_msg="Failed to see connector transition to the RUNNING state") - time.sleep(15) self.source.stop() - # Ensure that the sink connector has an opportunity to read all - # committed messages from the source connector. - wait_until(lambda: self.is_running(self.sink), timeout_sec=30, - err_msg="Failed to see connector transition to the RUNNING state") - time.sleep(15) self.sink.stop() self.cc.stop() diff --git a/tests/kafkatest/tests/core/compatibility_test_new_broker_test.py b/tests/kafkatest/tests/core/compatibility_test_new_broker_test.py index cfc127250ea55..25a6e5d29b21b 100644 --- a/tests/kafkatest/tests/core/compatibility_test_new_broker_test.py +++ b/tests/kafkatest/tests/core/compatibility_test_new_broker_test.py @@ -23,7 +23,7 @@ from kafkatest.services.zookeeper import ZookeeperService from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest from kafkatest.utils import is_int -from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, LATEST_2_4, LATEST_2_5, DEV_BRANCH, KafkaVersion +from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, LATEST_2_4, DEV_BRANCH, KafkaVersion # Compatibility tests for moving to a new broker (e.g., 0.10.x) and using a mix of old and new clients (e.g., 0.9.x) class ClientCompatibilityTestNewBroker(ProduceConsumeValidateTest): @@ -51,7 +51,6 @@ def setUp(self): @parametrize(producer_version=str(LATEST_2_2), consumer_version=str(LATEST_2_2), compression_types=["none"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_2_3), consumer_version=str(LATEST_2_3), compression_types=["none"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_2_4), consumer_version=str(LATEST_2_4), compression_types=["none"], timestamp_type=str("CreateTime")) - @parametrize(producer_version=str(LATEST_2_5), consumer_version=str(LATEST_2_5), compression_types=["none"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_2_1), consumer_version=str(LATEST_2_1), compression_types=["zstd"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_2_0), consumer_version=str(LATEST_2_0), compression_types=["snappy"], timestamp_type=str("CreateTime")) @parametrize(producer_version=str(LATEST_1_1), consumer_version=str(LATEST_1_1), compression_types=["lz4"], timestamp_type=str("CreateTime")) diff --git a/tests/kafkatest/tests/core/upgrade_test.py b/tests/kafkatest/tests/core/upgrade_test.py index faab7b14990b4..6c0e447a8b37d 100644 --- a/tests/kafkatest/tests/core/upgrade_test.py +++ b/tests/kafkatest/tests/core/upgrade_test.py @@ -24,7 +24,7 @@ from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest from kafkatest.utils import is_int from kafkatest.utils.remote_account import java_version -from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, LATEST_2_4, LATEST_2_5, V_0_9_0_0, V_0_11_0_0, DEV_BRANCH, KafkaVersion +from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10, LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, LATEST_2_0, LATEST_2_1, LATEST_2_2, LATEST_2_3, LATEST_2_4, V_0_9_0_0, V_0_11_0_0, DEV_BRANCH, KafkaVersion from kafkatest.services.kafka.util import new_jdk_not_supported class TestUpgrade(ProduceConsumeValidateTest): @@ -78,8 +78,6 @@ def perform_upgrade(self, from_kafka_version, to_message_format_version=None): @cluster(num_nodes=6) @parametrize(from_kafka_version=str(LATEST_2_4), to_message_format_version=None, compression_types=["none"]) @parametrize(from_kafka_version=str(LATEST_2_4), to_message_format_version=None, compression_types=["zstd"]) - @parametrize(from_kafka_version=str(LATEST_2_5), to_message_format_version=None, compression_types=["none"]) - @parametrize(from_kafka_version=str(LATEST_2_5), to_message_format_version=None, compression_types=["zstd"]) @parametrize(from_kafka_version=str(LATEST_2_3), to_message_format_version=None, compression_types=["none"]) @parametrize(from_kafka_version=str(LATEST_2_3), to_message_format_version=None, compression_types=["zstd"]) @parametrize(from_kafka_version=str(LATEST_2_2), to_message_format_version=None, compression_types=["none"])