From 363a1db87ba69bd03c4a433cf2b27eeb4ce33016 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Wed, 14 Jul 2021 00:43:36 -0400 Subject: [PATCH] KAFKA-10000: Exactly-once support for source connectors --- checkstyle/suppressions.xml | 6 +- .../org/apache/kafka/clients/admin/Admin.java | 23 + .../clients/admin/FenceProducersOptions.java | 38 + .../clients/admin/FenceProducersResult.java | 84 ++ .../kafka/clients/admin/KafkaAdminClient.java | 11 + .../internals/AbortTransactionHandler.java | 4 +- .../admin/internals/AdminApiDriver.java | 19 +- .../admin/internals/AdminApiHandler.java | 64 +- .../AlterConsumerGroupOffsetsHandler.java | 4 +- .../DeleteConsumerGroupOffsetsHandler.java | 4 +- .../DeleteConsumerGroupsHandler.java | 4 +- .../DescribeConsumerGroupsHandler.java | 4 +- .../internals/DescribeProducersHandler.java | 4 +- .../DescribeTransactionsHandler.java | 4 +- .../internals/FenceProducersHandler.java | 146 ++ .../ListConsumerGroupOffsetsHandler.java | 4 +- .../internals/ListTransactionsHandler.java | 4 +- ...RemoveMembersFromConsumerGroupHandler.java | 4 +- .../requests/InitProducerIdRequest.java | 2 +- .../org/apache/kafka/common/utils/Utils.java | 17 + .../clients/admin/KafkaAdminClientTest.java | 27 + .../kafka/clients/admin/MockAdminClient.java | 5 + .../AbortTransactionHandlerTest.java | 2 +- .../admin/internals/AdminApiDriverTest.java | 4 +- .../AllBrokersStrategyIntegrationTest.java | 4 +- .../AlterConsumerGroupOffsetsHandlerTest.java | 2 +- ...DeleteConsumerGroupOffsetsHandlerTest.java | 2 +- .../DeleteConsumerGroupsHandlerTest.java | 2 +- .../DescribeConsumerGroupsHandlerTest.java | 4 +- .../DescribeProducersHandlerTest.java | 2 +- .../DescribeTransactionsHandlerTest.java | 2 +- .../internals/FenceProducersHandlerTest.java | 145 ++ .../ListConsumerGroupOffsetsHandlerTest.java | 2 +- .../ListTransactionsHandlerTest.java | 6 +- ...veMembersFromConsumerGroupHandlerTest.java | 2 +- .../ConnectorTransactionBoundaries.java | 31 + .../connect/source/ExactlyOnceSupport.java | 31 + .../kafka/connect/source/SourceConnector.java | 44 + .../kafka/connect/source/SourceTask.java | 61 +- .../connect/source/SourceTaskContext.java | 25 + .../connect/source/TransactionContext.java | 55 + .../auth/extension/JaasBasicAuthFilter.java | 33 +- .../extension/JaasBasicAuthFilterTest.java | 32 +- .../MirrorConnectorsIntegrationBaseTest.java | 11 +- .../MirrorConnectorsIntegrationSSLTest.java | 4 + .../kafka/connect/cli/ConnectDistributed.java | 2 +- .../kafka/connect/runtime/AbstractHerder.java | 120 +- .../runtime/AbstractWorkerSourceTask.java | 642 ++++++++ .../runtime/ConnectMetricsRegistry.java | 13 + .../runtime/ExactlyOnceWorkerSourceTask.java | 525 +++++++ .../apache/kafka/connect/runtime/Herder.java | 11 + .../runtime/SourceConnectorConfig.java | 168 ++- .../connect/runtime/SubmittedRecords.java | 66 +- .../apache/kafka/connect/runtime/Worker.java | 898 +++++++++-- .../kafka/connect/runtime/WorkerConfig.java | 60 +- .../connect/runtime/WorkerConnector.java | 16 +- .../kafka/connect/runtime/WorkerSinkTask.java | 2 +- .../runtime/WorkerSinkTaskContext.java | 2 +- .../connect/runtime/WorkerSourceTask.java | 575 ++----- .../runtime/WorkerSourceTaskContext.java | 21 +- .../kafka/connect/runtime/WorkerTask.java | 4 +- .../runtime/WorkerTransactionContext.java | 104 ++ .../distributed/DistributedConfig.java | 95 ++ .../distributed/DistributedHerder.java | 687 ++++++++- .../runtime/distributed/EagerAssignor.java | 1 + .../IncrementalCooperativeAssignor.java | 1 + .../distributed/WorkerCoordinator.java | 1 + .../distributed/WorkerRebalanceListener.java | 2 +- .../errors/RetryWithToleranceOperator.java | 1 - .../connect/runtime/rest/RestClient.java | 22 +- .../rest/resources/ConnectorsResource.java | 12 + .../runtime/standalone/StandaloneHerder.java | 40 +- .../storage/CloseableOffsetStorageReader.java | 4 +- .../ClusterConfigState.java | 60 +- .../connect/storage/ConfigBackingStore.java | 21 +- .../storage/ConnectorOffsetBackingStore.java | 337 +++++ .../storage/KafkaConfigBackingStore.java | 279 +++- .../storage/KafkaOffsetBackingStore.java | 105 +- .../storage/MemoryConfigBackingStore.java | 12 +- .../storage/OffsetStorageReaderImpl.java | 15 +- .../connect/storage/OffsetStorageWriter.java | 13 +- .../tools/VerifiableSinkConnector.java | 4 +- .../connect/tools/VerifiableSourceTask.java | 26 +- .../kafka/connect/util/ConnectUtils.java | 70 + .../kafka/connect/util/KafkaBasedLog.java | 113 +- .../apache/kafka/connect/util/TopicAdmin.java | 43 +- ...nnectorClientConfigOverridePolicyTest.java | 2 +- .../connect/integration/ConnectorHandle.java | 8 + .../ExactlyOnceSourceIntegrationTest.java | 925 ++++++++++++ .../MonitorableSourceConnector.java | 118 +- .../connect/runtime/AbstractHerderTest.java | 38 +- .../runtime/AbstractWorkerSourceTaskTest.java | 842 +++++++++++ .../runtime/ErrorHandlingTaskTest.java | 13 +- .../ExactlyOnceWorkerSourceTaskTest.java | 1330 +++++++++++++++++ .../connect/runtime/SubmittedRecordsTest.java | 41 +- .../connect/runtime/WorkerConnectorTest.java | 34 +- .../connect/runtime/WorkerSinkTaskTest.java | 2 +- .../runtime/WorkerSinkTaskThreadedTest.java | 2 +- .../connect/runtime/WorkerSourceTaskTest.java | 578 +------ .../kafka/connect/runtime/WorkerTest.java | 939 +++++++++++- .../connect/runtime/WorkerTestUtils.java | 5 +- .../runtime/WorkerTransactionContextTest.java | 110 ++ .../ConnectProtocolCompatibilityTest.java | 4 + .../distributed/DistributedConfigTest.java | 42 + .../distributed/DistributedHerderTest.java | 1167 +++++++++++++-- .../IncrementalCooperativeAssignorTest.java | 3 +- .../WorkerCoordinatorIncrementalTest.java | 1 + .../distributed/WorkerCoordinatorTest.java | 10 + .../resources/ConnectorsResourceTest.java | 66 +- .../standalone/StandaloneHerderTest.java | 32 +- .../storage/KafkaConfigBackingStoreTest.java | 407 ++++- .../storage/KafkaOffsetBackingStoreTest.java | 85 ++ .../storage/OffsetStorageReaderImplTest.java | 112 ++ .../storage/OffsetStorageWriterTest.java | 2 +- .../kafka/connect/util/ConnectUtilsTest.java | 71 + .../util/clusters/EmbeddedConnectCluster.java | 16 +- .../util/clusters/EmbeddedKafkaCluster.java | 72 +- .../src/test/resources/log4j.properties | 2 + gradle/spotbugs-exclude.xml | 2 +- tests/kafkatest/services/connect.py | 6 +- .../tests/connect/connect_distributed_test.py | 197 ++- .../templates/connect-distributed.properties | 2 + 122 files changed, 11583 insertions(+), 1884 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/FenceProducersOptions.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/FenceProducersResult.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/internals/FenceProducersHandler.java create mode 100644 clients/src/test/java/org/apache/kafka/clients/admin/internals/FenceProducersHandlerTest.java create mode 100644 connect/api/src/main/java/org/apache/kafka/connect/source/ConnectorTransactionBoundaries.java create mode 100644 connect/api/src/main/java/org/apache/kafka/connect/source/ExactlyOnceSupport.java create mode 100644 connect/api/src/main/java/org/apache/kafka/connect/source/TransactionContext.java create mode 100644 connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTask.java create mode 100644 connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTask.java create mode 100644 connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTransactionContext.java rename connect/runtime/src/main/java/org/apache/kafka/connect/{runtime/distributed => storage}/ClusterConfigState.java (81%) create mode 100644 connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConnectorOffsetBackingStore.java create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExactlyOnceSourceIntegrationTest.java create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTaskTest.java create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTaskTest.java create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTransactionContextTest.java create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/storage/OffsetStorageReaderImplTest.java diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index a53a50a75fcd3..b4d475ad9aa66 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -115,14 +115,14 @@ + files="(DistributedHerder|KafkaConfigBackingStore|Values|IncrementalCooperativeAssignor).java"/> + files="(RestServer|AbstractHerder|DistributedHerder|Worker).java"/> @@ -133,7 +133,7 @@ files="(JsonConverter|Values|ConnectHeaders).java"/> + files="(KafkaConfigBackingStore|Values|ConnectMetricsRegistry).java"/> diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java index 377f009a958df..0c795bc5206dc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java @@ -1575,6 +1575,29 @@ default ListTransactionsResult listTransactions() { */ ListTransactionsResult listTransactions(ListTransactionsOptions options); + /** + * Fence out all active producers that use any of the provided transactional IDs, with the default options. + *

+ * This is a convenience method for {@link #fenceProducers(Collection, FenceProducersOptions)} + * with default options. See the overload for more details. + * + * @param transactionalIds The IDs of the producers to fence. + * @return The FenceProducersResult. + */ + default FenceProducersResult fenceProducers(Collection transactionalIds) { + return fenceProducers(transactionalIds, new FenceProducersOptions()); + } + + /** + * Fence out all active producers that use any of the provided transactional IDs. + * + * @param transactionalIds The IDs of the producers to fence. + * @param options The options to use when fencing the producers. + * @return The FenceProducersResult. + */ + FenceProducersResult fenceProducers(Collection transactionalIds, + FenceProducersOptions options); + /** * Get the metrics kept by the adminClient */ diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/FenceProducersOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/FenceProducersOptions.java new file mode 100644 index 0000000000000..4e38281809b83 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/FenceProducersOptions.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Collection; + +/** + * Options for {@link Admin#fenceProducers(Collection, FenceProducersOptions)} + * + * The API of this class is evolving. See {@link Admin} for details. + */ +@InterfaceStability.Evolving +public class FenceProducersOptions extends AbstractOptions { + + @Override + public String toString() { + return "FenceProducersOptions{" + + "timeoutMs=" + timeoutMs + + '}'; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/FenceProducersResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/FenceProducersResult.java new file mode 100644 index 0000000000000..a1a8e66c115c5 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/FenceProducersResult.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.clients.admin.internals.CoordinatorKey; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.utils.ProducerIdAndEpoch; + +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * The result of the {@link Admin#fenceProducers(Collection)} call. + * + * The API of this class is evolving, see {@link Admin} for details. + */ +@InterfaceStability.Evolving +public class FenceProducersResult { + + private final Map> futures; + + FenceProducersResult(Map> futures) { + this.futures = futures; + } + + /** + * Return a map from transactional ID to futures which can be used to check the status of + * individual fencings. + */ + public Map> fencedProducers() { + return futures.entrySet().stream().collect(Collectors.toMap( + e -> e.getKey().idValue, + e -> e.getValue().thenApply(p -> null) + )); + } + + /** + * Returns a future that provides the producer ID generated while initializing the given transaction when the request completes. + */ + public KafkaFuture producerId(String transactionalId) { + return findAndApply(transactionalId, p -> p.producerId); + } + + /** + * Returns a future that provides the epoch ID generated while initializing the given transaction when the request completes. + */ + public KafkaFuture epochId(String transactionalId) { + return findAndApply(transactionalId, p -> p.epoch); + } + + /** + * Return a future which succeeds only if all the producer fencings succeed. + */ + public KafkaFuture all() { + return KafkaFuture.allOf(futures.values().toArray(new KafkaFuture[0])); + } + + private KafkaFuture findAndApply(String transactionalId, KafkaFuture.BaseFunction followup) { + CoordinatorKey key = CoordinatorKey.byTransactionalId(transactionalId); + KafkaFuture future = futures.get(key); + if (future == null) { + throw new IllegalArgumentException("TransactionalId " + + "`" + transactionalId + "` was not included in the request"); + } + return future.thenApply(followup); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index ea4e2d7e5ffb0..03322fdcf1dc8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -47,6 +47,7 @@ import org.apache.kafka.clients.admin.internals.DescribeConsumerGroupsHandler; import org.apache.kafka.clients.admin.internals.DescribeProducersHandler; import org.apache.kafka.clients.admin.internals.DescribeTransactionsHandler; +import org.apache.kafka.clients.admin.internals.FenceProducersHandler; import org.apache.kafka.clients.admin.internals.ListConsumerGroupOffsetsHandler; import org.apache.kafka.clients.admin.internals.ListTransactionsHandler; import org.apache.kafka.clients.admin.internals.MetadataOperationContext; @@ -234,6 +235,7 @@ import org.apache.kafka.common.utils.AppInfoParser; import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.ProducerIdAndEpoch; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; @@ -4390,6 +4392,15 @@ public ListTransactionsResult listTransactions(ListTransactionsOptions options) return new ListTransactionsResult(future.all()); } + @Override + public FenceProducersResult fenceProducers(Collection transactionalIds, FenceProducersOptions options) { + AdminApiFuture.SimpleAdminApiFuture future = + FenceProducersHandler.newFuture(transactionalIds); + FenceProducersHandler handler = new FenceProducersHandler(logContext); + invokeDriver(handler, future, options.timeoutMs); + return new FenceProducersResult(future.all()); + } + private void invokeDriver( AdminApiHandler handler, AdminApiFuture future, diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/AbortTransactionHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/AbortTransactionHandler.java index c25e4d8d3f479..4963a49c351b0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/AbortTransactionHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/AbortTransactionHandler.java @@ -38,7 +38,7 @@ import static java.util.Collections.singleton; import static java.util.Collections.singletonList; -public class AbortTransactionHandler implements AdminApiHandler { +public class AbortTransactionHandler extends AdminApiHandler.Batched { private final Logger log; private final AbortTransactionSpec abortSpec; private final PartitionLeaderStrategy lookupStrategy; @@ -69,7 +69,7 @@ public AdminApiLookupStrategy lookupStrategy() { } @Override - public WriteTxnMarkersRequest.Builder buildRequest( + public WriteTxnMarkersRequest.Builder buildBatchedRequest( int brokerId, Set topicPartitions ) { diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/AdminApiDriver.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/AdminApiDriver.java index b5c9ff32f264b..d00db4b18c694 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/AdminApiDriver.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/AdminApiDriver.java @@ -288,7 +288,7 @@ private void clearInflightRequest(long currentTimeMs, RequestSpec spec) { private void collectRequests( List> requests, BiMultimap multimap, - BiFunction, T, AbstractRequest.Builder> buildRequest + BiFunction, T, Collection>> buildRequest ) { for (Map.Entry> entry : multimap.entrySet()) { T scope = entry.getKey(); @@ -306,12 +306,19 @@ private void collectRequests( // Copy the keys to avoid exposing the underlying mutable set Set copyKeys = Collections.unmodifiableSet(new HashSet<>(keys)); - AbstractRequest.Builder request = buildRequest.apply(copyKeys, scope); + Collection> newRequests = buildRequest.apply(copyKeys, scope); + if (newRequests.isEmpty()) { + return; + } + + // Only process the first request; all the remaining requests will be targeted at the same broker + // and we don't want to issue more than one fulfillment request per broker at a time + AdminApiHandler.RequestAndKeys newRequest = newRequests.iterator().next(); RequestSpec spec = new RequestSpec<>( - handler.apiName() + "(api=" + request.apiKey() + ")", + handler.apiName() + "(api=" + newRequest.request.apiKey() + ")", scope, - copyKeys, - request, + newRequest.keys, + newRequest.request, requestState.nextAllowedRetryMs, deadlineMs, requestState.tries @@ -326,7 +333,7 @@ private void collectLookupRequests(List> requests) { collectRequests( requests, lookupMap, - (keys, scope) -> handler.lookupStrategy().buildRequest(keys) + (keys, scope) -> Collections.singletonList(new AdminApiHandler.RequestAndKeys<>(handler.lookupStrategy().buildRequest(keys), keys)) ); } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/AdminApiHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/AdminApiHandler.java index 9f8d0ac5f07f2..79df965ffb0f9 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/AdminApiHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/AdminApiHandler.java @@ -20,10 +20,12 @@ import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; public interface AdminApiHandler { @@ -33,16 +35,18 @@ public interface AdminApiHandler { String apiName(); /** - * Build the request. The set of keys are derived by {@link AdminApiDriver} - * during the lookup stage as the set of keys which all map to the same - * destination broker. + * Build the requests necessary for the given keys. The set of keys is derived by + * {@link AdminApiDriver} during the lookup stage as the set of keys which all map + * to the same destination broker. Handlers can choose to issue a single request for + * all of the provided keys (see {@link Batched}, issue one request per key (see + * {@link Unbatched}, or implement their own custom grouping logic if necessary. * * @param brokerId the target brokerId for the request * @param keys the set of keys that should be handled by this request * - * @return a builder for the request containing the given keys + * @return a collection of {@link RequestAndKeys} for the requests containing the given keys */ - AbstractRequest.Builder buildRequest(int brokerId, Set keys); + Collection> buildRequest(int brokerId, Set keys); /** * Callback that is invoked when a request returns successfully. @@ -122,4 +126,54 @@ public static ApiResult empty() { } } + class RequestAndKeys { + public final AbstractRequest.Builder request; + public final Set keys; + + public RequestAndKeys(AbstractRequest.Builder request, Set keys) { + this.request = request; + this.keys = keys; + } + } + + /** + * An {@link AdminApiHandler} that will group multiple keys into a single request when possible. + * Keys will be grouped together whenever they target the same broker. This type of handler + * should be used when when interacting with broker APIs that can act on multiple keys at once, + * such as describing or listing transactions. + */ + abstract class Batched implements AdminApiHandler { + abstract AbstractRequest.Builder buildBatchedRequest(int brokerId, Set keys); + + @Override + public final Collection> buildRequest(int brokerId, Set keys) { + return Collections.singleton(new RequestAndKeys<>(buildBatchedRequest(brokerId, keys), keys)); + } + } + + /** + * An {@link AdminApiHandler} that will create one request per key, not performing any grouping based + * on the targeted broker. This type of handler should only be used for broker APIs that do not accept + * multiple keys at once, such as initializing a transactional producer. + */ + abstract class Unbatched implements AdminApiHandler { + abstract AbstractRequest.Builder buildSingleRequest(int brokerId, K key); + abstract ApiResult handleSingleResponse(Node broker, K key, AbstractResponse response); + + @Override + public final Collection> buildRequest(int brokerId, Set keys) { + return keys.stream() + .map(key -> new RequestAndKeys<>(buildSingleRequest(brokerId, key), Collections.singleton(key))) + .collect(Collectors.toSet()); + } + + @Override + public final ApiResult handleResponse(Node broker, Set keys, AbstractResponse response) { + if (keys.size() != 1) { + throw new IllegalArgumentException("Unbatched admin handler should only be required to handle responses for a single key at a time"); + } + K key = keys.iterator().next(); + return handleSingleResponse(broker, key, response); + } + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/AlterConsumerGroupOffsetsHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/AlterConsumerGroupOffsetsHandler.java index cb7551e55ed21..eab2e2bb73a40 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/AlterConsumerGroupOffsetsHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/AlterConsumerGroupOffsetsHandler.java @@ -41,7 +41,7 @@ import org.apache.kafka.common.utils.LogContext; import org.slf4j.Logger; -public class AlterConsumerGroupOffsetsHandler implements AdminApiHandler> { +public class AlterConsumerGroupOffsetsHandler extends AdminApiHandler.Batched> { private final CoordinatorKey groupId; private final Map offsets; @@ -83,7 +83,7 @@ private void validateKeys(Set groupIds) { } @Override - public OffsetCommitRequest.Builder buildRequest( + public OffsetCommitRequest.Builder buildBatchedRequest( int coordinatorId, Set groupIds ) { diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupOffsetsHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupOffsetsHandler.java index a853eddcb31b1..b68334b55c8ae 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupOffsetsHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupOffsetsHandler.java @@ -38,7 +38,7 @@ import org.apache.kafka.common.utils.LogContext; import org.slf4j.Logger; -public class DeleteConsumerGroupOffsetsHandler implements AdminApiHandler> { +public class DeleteConsumerGroupOffsetsHandler extends AdminApiHandler.Batched> { private final CoordinatorKey groupId; private final Set partitions; @@ -80,7 +80,7 @@ private void validateKeys(Set groupIds) { } @Override - public OffsetDeleteRequest.Builder buildRequest(int coordinatorId, Set groupIds) { + public OffsetDeleteRequest.Builder buildBatchedRequest(int coordinatorId, Set groupIds) { validateKeys(groupIds); final OffsetDeleteRequestTopicCollection topics = new OffsetDeleteRequestTopicCollection(); diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupsHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupsHandler.java index 693d23625e8d3..c05a11b5cfb40 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupsHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupsHandler.java @@ -36,7 +36,7 @@ import org.apache.kafka.common.utils.LogContext; import org.slf4j.Logger; -public class DeleteConsumerGroupsHandler implements AdminApiHandler { +public class DeleteConsumerGroupsHandler extends AdminApiHandler.Batched { private final Logger log; private final AdminApiLookupStrategy lookupStrategy; @@ -71,7 +71,7 @@ private static Set buildKeySet(Collection groupIds) { } @Override - public DeleteGroupsRequest.Builder buildRequest( + public DeleteGroupsRequest.Builder buildBatchedRequest( int coordinatorId, Set keys ) { diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/DescribeConsumerGroupsHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/DescribeConsumerGroupsHandler.java index 5c5022a37eccc..09a2a7b4cdcdf 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/DescribeConsumerGroupsHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/DescribeConsumerGroupsHandler.java @@ -51,7 +51,7 @@ import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; -public class DescribeConsumerGroupsHandler implements AdminApiHandler { +public class DescribeConsumerGroupsHandler extends AdminApiHandler.Batched { private final boolean includeAuthorizedOperations; private final Logger log; @@ -89,7 +89,7 @@ public AdminApiLookupStrategy lookupStrategy() { } @Override - public DescribeGroupsRequest.Builder buildRequest(int coordinatorId, Set keys) { + public DescribeGroupsRequest.Builder buildBatchedRequest(int coordinatorId, Set keys) { List groupIds = keys.stream().map(key -> { if (key.type != FindCoordinatorRequest.CoordinatorType.GROUP) { throw new IllegalArgumentException("Invalid transaction coordinator key " + key + diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/DescribeProducersHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/DescribeProducersHandler.java index 4b279d5c90c78..8555bf59ab4ed 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/DescribeProducersHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/DescribeProducersHandler.java @@ -46,7 +46,7 @@ import java.util.Set; import java.util.stream.Collectors; -public class DescribeProducersHandler implements AdminApiHandler { +public class DescribeProducersHandler extends AdminApiHandler.Batched { private final Logger log; private final DescribeProducersOptions options; private final AdminApiLookupStrategy lookupStrategy; @@ -82,7 +82,7 @@ public AdminApiLookupStrategy lookupStrategy() { } @Override - public DescribeProducersRequest.Builder buildRequest( + public DescribeProducersRequest.Builder buildBatchedRequest( int brokerId, Set topicPartitions ) { diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/DescribeTransactionsHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/DescribeTransactionsHandler.java index d270145a423da..4a3d7c0a05272 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/DescribeTransactionsHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/DescribeTransactionsHandler.java @@ -43,7 +43,7 @@ import java.util.Set; import java.util.stream.Collectors; -public class DescribeTransactionsHandler implements AdminApiHandler { +public class DescribeTransactionsHandler extends AdminApiHandler.Batched { private final Logger log; private final AdminApiLookupStrategy lookupStrategy; @@ -77,7 +77,7 @@ public AdminApiLookupStrategy lookupStrategy() { } @Override - public DescribeTransactionsRequest.Builder buildRequest( + public DescribeTransactionsRequest.Builder buildBatchedRequest( int brokerId, Set keys ) { diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/FenceProducersHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/FenceProducersHandler.java new file mode 100644 index 0000000000000..7b6ff817efac2 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/FenceProducersHandler.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.admin.internals; + +import org.apache.kafka.common.Node; +import org.apache.kafka.common.errors.InvalidProducerEpochException; +import org.apache.kafka.common.errors.TransactionalIdAuthorizationException; +import org.apache.kafka.common.errors.TransactionalIdNotFoundException; +import org.apache.kafka.common.message.InitProducerIdRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.FindCoordinatorRequest; +import org.apache.kafka.common.requests.InitProducerIdRequest; +import org.apache.kafka.common.requests.InitProducerIdResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.ProducerIdAndEpoch; +import org.slf4j.Logger; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class FenceProducersHandler extends AdminApiHandler.Unbatched { + private final Logger log; + private final AdminApiLookupStrategy lookupStrategy; + + public FenceProducersHandler( + LogContext logContext + ) { + this.log = logContext.logger(DescribeTransactionsHandler.class); + this.lookupStrategy = new CoordinatorStrategy(FindCoordinatorRequest.CoordinatorType.TRANSACTION, logContext); + } + + public static AdminApiFuture.SimpleAdminApiFuture newFuture( + Collection transactionalIds + ) { + return AdminApiFuture.forKeys(buildKeySet(transactionalIds)); + } + + private static Set buildKeySet(Collection transactionalIds) { + return transactionalIds.stream() + .map(CoordinatorKey::byTransactionalId) + .collect(Collectors.toSet()); + } + + @Override + public String apiName() { + return "fenceProducer"; + } + + @Override + public AdminApiLookupStrategy lookupStrategy() { + return lookupStrategy; + } + + @Override + InitProducerIdRequest.Builder buildSingleRequest(int brokerId, CoordinatorKey key) { + if (key.type != FindCoordinatorRequest.CoordinatorType.TRANSACTION) { + throw new IllegalArgumentException("Invalid group coordinator key " + key + + " when building `InitProducerId` request"); + } + InitProducerIdRequestData data = new InitProducerIdRequestData() + .setProducerEpoch(ProducerIdAndEpoch.NONE.epoch) + .setProducerId(ProducerIdAndEpoch.NONE.producerId) + .setTransactionalId(key.idValue) + // Set transaction timeout to 1 since it's only being initialized to fence out older producers with the same transactional ID, + // and shouldn't be used for any actual record writes + .setTransactionTimeoutMs(1); + return new InitProducerIdRequest.Builder(data); + } + + @Override + public ApiResult handleSingleResponse( + Node broker, + CoordinatorKey key, + AbstractResponse abstractResponse + ) { + InitProducerIdResponse response = (InitProducerIdResponse) abstractResponse; + + Errors error = Errors.forCode(response.data().errorCode()); + if (error != Errors.NONE) { + return handleError(key, error); + } + + Map completed = Collections.singletonMap(key, new ProducerIdAndEpoch( + response.data().producerId(), + response.data().producerEpoch() + )); + + return new ApiResult<>(completed, Collections.emptyMap(), Collections.emptyList()); + } + + private ApiResult handleError(CoordinatorKey transactionalIdKey, Errors error) { + switch (error) { + case TRANSACTIONAL_ID_AUTHORIZATION_FAILED: + case CLUSTER_AUTHORIZATION_FAILED: + return ApiResult.failed(transactionalIdKey, new TransactionalIdAuthorizationException( + "InitProducerId request for transactionalId `" + transactionalIdKey.idValue + "` " + + "failed due to authorization failure")); + + case TRANSACTIONAL_ID_NOT_FOUND: + return ApiResult.failed(transactionalIdKey, new TransactionalIdNotFoundException( + "InitProducerId request for transactionalId `" + transactionalIdKey.idValue + "` " + + "failed because the ID could not be found")); + + case INVALID_PRODUCER_EPOCH: + return ApiResult.failed(transactionalIdKey, new InvalidProducerEpochException( + "InitProducerId request with " + transactionalIdKey.idValue + " failed due an invalid producer epoch")); + + case COORDINATOR_LOAD_IN_PROGRESS: + // If the coordinator is in the middle of loading, then we just need to retry + log.debug("InitProducerId request for transactionalId `{}` failed because the " + + "coordinator is still in the process of loading state. Will retry", + transactionalIdKey.idValue); + return ApiResult.empty(); + + case NOT_COORDINATOR: + case COORDINATOR_NOT_AVAILABLE: + // If the coordinator is unavailable or there was a coordinator change, then we unmap + // the key so that we retry the `FindCoordinator` request + log.debug("InitProducerId request for transactionalId `{}` returned error {}. Will attempt " + + "to find the coordinator again and retry", transactionalIdKey.idValue, error); + return ApiResult.unmapped(Collections.singletonList(transactionalIdKey)); + + default: + return ApiResult.failed(transactionalIdKey, error.exception("InitProducerId request for " + + "transactionalId `" + transactionalIdKey.idValue + "` failed due to unexpected error")); + } + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/ListConsumerGroupOffsetsHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/ListConsumerGroupOffsetsHandler.java index b1d2e9dc69665..b591548954b96 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/ListConsumerGroupOffsetsHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/ListConsumerGroupOffsetsHandler.java @@ -36,7 +36,7 @@ import org.apache.kafka.common.utils.LogContext; import org.slf4j.Logger; -public class ListConsumerGroupOffsetsHandler implements AdminApiHandler> { +public class ListConsumerGroupOffsetsHandler extends AdminApiHandler.Batched> { private final CoordinatorKey groupId; private final List partitions; @@ -78,7 +78,7 @@ private void validateKeys(Set groupIds) { } @Override - public OffsetFetchRequest.Builder buildRequest(int coordinatorId, Set groupIds) { + public OffsetFetchRequest.Builder buildBatchedRequest(int coordinatorId, Set groupIds) { validateKeys(groupIds); // Set the flag to false as for admin client request, // we don't need to wait for any pending offset state to clear. diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/ListTransactionsHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/ListTransactionsHandler.java index d60580c85bf22..ca249bca7f4a4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/ListTransactionsHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/ListTransactionsHandler.java @@ -35,7 +35,7 @@ import java.util.Set; import java.util.stream.Collectors; -public class ListTransactionsHandler implements AdminApiHandler> { +public class ListTransactionsHandler extends AdminApiHandler.Batched> { private final Logger log; private final ListTransactionsOptions options; private final AllBrokersStrategy lookupStrategy; @@ -64,7 +64,7 @@ public AdminApiLookupStrategy lookupStrategy() { } @Override - public ListTransactionsRequest.Builder buildRequest( + public ListTransactionsRequest.Builder buildBatchedRequest( int brokerId, Set keys ) { diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/internals/RemoveMembersFromConsumerGroupHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/internals/RemoveMembersFromConsumerGroupHandler.java index 90b3865d0bd65..83aee36587d3a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/internals/RemoveMembersFromConsumerGroupHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/internals/RemoveMembersFromConsumerGroupHandler.java @@ -35,7 +35,7 @@ import org.apache.kafka.common.utils.LogContext; import org.slf4j.Logger; -public class RemoveMembersFromConsumerGroupHandler implements AdminApiHandler> { +public class RemoveMembersFromConsumerGroupHandler extends AdminApiHandler.Batched> { private final CoordinatorKey groupId; private final List members; @@ -79,7 +79,7 @@ private void validateKeys( } @Override - public LeaveGroupRequest.Builder buildRequest(int coordinatorId, Set groupIds) { + public LeaveGroupRequest.Builder buildBatchedRequest(int coordinatorId, Set groupIds) { validateKeys(groupIds); return new LeaveGroupRequest.Builder(groupId.idValue, members); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java index 5c24b41b351df..9d92f0e5351dd 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java @@ -27,7 +27,7 @@ public class InitProducerIdRequest extends AbstractRequest { public static class Builder extends AbstractRequest.Builder { - private final InitProducerIdRequestData data; + public final InitProducerIdRequestData data; public Builder(InitProducerIdRequestData data) { super(ApiKeys.INIT_PRODUCER_ID); diff --git a/clients/src/main/java/org/apache/kafka/common/utils/Utils.java b/clients/src/main/java/org/apache/kafka/common/utils/Utils.java index 0685d1d49b76a..741fd332fa31c 100755 --- a/clients/src/main/java/org/apache/kafka/common/utils/Utils.java +++ b/clients/src/main/java/org/apache/kafka/common/utils/Utils.java @@ -1415,4 +1415,21 @@ public static Map initializeMap(Collection keys, Supplier val return res; } + /** + * Get an array containing all of the {@link Object#toString names} of a given enumerable type. + * @param enumClass the enum class; may not be null + * @return an array with the names of every value for the enum class; never null, but may be empty + * if there are no values defined for the enum + */ + public static String[] enumOptions(Class> enumClass) { + Objects.requireNonNull(enumClass); + if (!enumClass.isEnum()) { + throw new IllegalArgumentException("Class " + enumClass + " is not an enumerable type"); + } + + return Stream.of(enumClass.getEnumConstants()) + .map(Object::toString) + .toArray(String[]::new); + } + } diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index fcd93a96b3989..c936bab6d6aae 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -109,6 +109,7 @@ import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData; import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse; import org.apache.kafka.common.message.LeaveGroupRequestData; +import org.apache.kafka.common.message.InitProducerIdResponseData; import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; @@ -167,6 +168,8 @@ import org.apache.kafka.common.requests.FindCoordinatorRequest; import org.apache.kafka.common.requests.FindCoordinatorResponse; import org.apache.kafka.common.requests.IncrementalAlterConfigsResponse; +import org.apache.kafka.common.requests.InitProducerIdRequest; +import org.apache.kafka.common.requests.InitProducerIdResponse; import org.apache.kafka.common.requests.JoinGroupRequest; import org.apache.kafka.common.requests.LeaveGroupRequest; import org.apache.kafka.common.requests.LeaveGroupResponse; @@ -6281,6 +6284,30 @@ public void testClientSideTimeoutAfterFailureToReceiveResponse() throws Exceptio } } + @Test + public void testFenceProducers() throws Exception { + try (AdminClientUnitTestEnv env = mockClientEnv()) { + String transactionalId = "copyCat"; + Node transactionCoordinator = env.cluster().nodes().iterator().next(); + + env.kafkaClient().prepareResponse(prepareFindCoordinatorResponse(Errors.NONE, transactionalId, transactionCoordinator)); + + InitProducerIdResponseData initProducerIdResponseData = new InitProducerIdResponseData() + .setProducerId(4761) + .setProducerEpoch((short) 489); + env.kafkaClient().prepareResponseFrom( + request -> request instanceof InitProducerIdRequest, + new InitProducerIdResponse(initProducerIdResponseData), + transactionCoordinator + ); + + FenceProducersResult result = env.adminClient().fenceProducers(Collections.singleton(transactionalId)); + assertNull(result.all().get()); + assertEquals(4761, result.producerId(transactionalId).get()); + assertEquals((short) 489, result.epochId(transactionalId).get()); + } + } + private UnregisterBrokerResponse prepareUnregisterBrokerResponse(Errors error, int throttleTimeMs) { return new UnregisterBrokerResponse(new UnregisterBrokerResponseData() .setErrorCode(error.code()) diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java index 1593b6328df74..15cdc5ccc4116 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java @@ -1003,6 +1003,11 @@ public ListTransactionsResult listTransactions(ListTransactionsOptions options) throw new UnsupportedOperationException("Not implemented yet"); } + @Override + public FenceProducersResult fenceProducers(Collection transactionalIds, FenceProducersOptions options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + @Override synchronized public void close(Duration timeout) {} diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/AbortTransactionHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/AbortTransactionHandlerTest.java index 78b33a7f59dff..68c850583d6ae 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/internals/AbortTransactionHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/AbortTransactionHandlerTest.java @@ -63,7 +63,7 @@ public void testInvalidBuildRequestCall() { @Test public void testValidBuildRequestCall() { AbortTransactionHandler handler = new AbortTransactionHandler(abortSpec, logContext); - WriteTxnMarkersRequest.Builder request = handler.buildRequest(1, singleton(topicPartition)); + WriteTxnMarkersRequest.Builder request = handler.buildBatchedRequest(1, singleton(topicPartition)); assertEquals(1, request.data.markers().size()); WriteTxnMarkersRequestData.WritableTxnMarker markerRequest = request.data.markers().get(0); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/AdminApiDriverTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/AdminApiDriverTest.java index 6ff393fddd65e..2c664e857fced 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/internals/AdminApiDriverTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/AdminApiDriverTest.java @@ -734,7 +734,7 @@ public void reset() { } } - private static class MockAdminApiHandler implements AdminApiHandler { + private static class MockAdminApiHandler extends AdminApiHandler.Batched { private final Map, ApiResult> expectedRequests = new HashMap<>(); private final MockLookupStrategy lookupStrategy; @@ -757,7 +757,7 @@ public void expectRequest(Set keys, ApiResult result) { } @Override - public AbstractRequest.Builder buildRequest(int brokerId, Set keys) { + public AbstractRequest.Builder buildBatchedRequest(int brokerId, Set keys) { // The request is just a placeholder in these tests assertTrue(expectedRequests.containsKey(keys), "Unexpected fulfillment request for keys " + keys); return new MetadataRequest.Builder(Collections.emptyList(), false); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/AllBrokersStrategyIntegrationTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/AllBrokersStrategyIntegrationTest.java index 2b989058082fb..b2e0a0d285b06 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/internals/AllBrokersStrategyIntegrationTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/AllBrokersStrategyIntegrationTest.java @@ -215,7 +215,7 @@ private MetadataResponse responseWithBrokers(Set brokerIds) { return new MetadataResponse(response, ApiKeys.METADATA.latestVersion()); } - private class MockApiHandler implements AdminApiHandler { + private class MockApiHandler extends AdminApiHandler.Batched { private final AllBrokersStrategy allBrokersStrategy = new AllBrokersStrategy(logContext); @Override @@ -224,7 +224,7 @@ public String apiName() { } @Override - public AbstractRequest.Builder buildRequest( + public AbstractRequest.Builder buildBatchedRequest( int brokerId, Set keys ) { diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/AlterConsumerGroupOffsetsHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/AlterConsumerGroupOffsetsHandlerTest.java index c0ea2ba9f0e7e..80453cc9b85d4 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/internals/AlterConsumerGroupOffsetsHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/AlterConsumerGroupOffsetsHandlerTest.java @@ -62,7 +62,7 @@ public void setUp() { @Test public void testBuildRequest() { AlterConsumerGroupOffsetsHandler handler = new AlterConsumerGroupOffsetsHandler(groupId, partitions, logContext); - OffsetCommitRequest request = handler.buildRequest(-1, singleton(CoordinatorKey.byGroupId(groupId))).build(); + OffsetCommitRequest request = handler.buildBatchedRequest(-1, singleton(CoordinatorKey.byGroupId(groupId))).build(); assertEquals(groupId, request.data().groupId()); assertEquals(2, request.data().topics().size()); assertEquals(2, request.data().topics().get(0).partitions().size()); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupOffsetsHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupOffsetsHandlerTest.java index b4aea93c3f3f6..629ca89801559 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupOffsetsHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupOffsetsHandlerTest.java @@ -59,7 +59,7 @@ public class DeleteConsumerGroupOffsetsHandlerTest { @Test public void testBuildRequest() { DeleteConsumerGroupOffsetsHandler handler = new DeleteConsumerGroupOffsetsHandler(groupId, tps, logContext); - OffsetDeleteRequest request = handler.buildRequest(1, singleton(CoordinatorKey.byGroupId(groupId))).build(); + OffsetDeleteRequest request = handler.buildBatchedRequest(1, singleton(CoordinatorKey.byGroupId(groupId))).build(); assertEquals(groupId, request.data().groupId()); assertEquals(2, request.data().topics().size()); assertEquals(2, request.data().topics().find("t0").partitions().size()); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupsHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupsHandlerTest.java index 8d3a2376fde9a..3e7cead6f5d40 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupsHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/DeleteConsumerGroupsHandlerTest.java @@ -45,7 +45,7 @@ public class DeleteConsumerGroupsHandlerTest { @Test public void testBuildRequest() { DeleteConsumerGroupsHandler handler = new DeleteConsumerGroupsHandler(logContext); - DeleteGroupsRequest request = handler.buildRequest(1, singleton(CoordinatorKey.byGroupId(groupId1))).build(); + DeleteGroupsRequest request = handler.buildBatchedRequest(1, singleton(CoordinatorKey.byGroupId(groupId1))).build(); assertEquals(1, request.data().groupsNames().size()); assertEquals(groupId1, request.data().groupsNames().get(0)); } diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/DescribeConsumerGroupsHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/DescribeConsumerGroupsHandlerTest.java index aef207aca6a75..892a81e8c0e11 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/internals/DescribeConsumerGroupsHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/DescribeConsumerGroupsHandlerTest.java @@ -69,12 +69,12 @@ public class DescribeConsumerGroupsHandlerTest { @Test public void testBuildRequest() { DescribeConsumerGroupsHandler handler = new DescribeConsumerGroupsHandler(false, logContext); - DescribeGroupsRequest request = handler.buildRequest(1, keys).build(); + DescribeGroupsRequest request = handler.buildBatchedRequest(1, keys).build(); assertEquals(2, request.data().groups().size()); assertFalse(request.data().includeAuthorizedOperations()); handler = new DescribeConsumerGroupsHandler(true, logContext); - request = handler.buildRequest(1, keys).build(); + request = handler.buildBatchedRequest(1, keys).build(); assertEquals(2, request.data().groups().size()); assertTrue(request.data().includeAuthorizedOperations()); } diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/DescribeProducersHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/DescribeProducersHandlerTest.java index 8daed06ddf133..0f39b4dd01633 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/internals/DescribeProducersHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/DescribeProducersHandlerTest.java @@ -118,7 +118,7 @@ public void testBuildRequest() { ); int brokerId = 3; - DescribeProducersRequest.Builder request = handler.buildRequest(brokerId, topicPartitions); + DescribeProducersRequest.Builder request = handler.buildBatchedRequest(brokerId, topicPartitions); List topics = request.data.topics(); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/DescribeTransactionsHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/DescribeTransactionsHandlerTest.java index 04eac89a373ac..7ffda2b00e014 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/internals/DescribeTransactionsHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/DescribeTransactionsHandlerTest.java @@ -156,7 +156,7 @@ private void assertLookup( Set transactionalIds ) { Set keys = coordinatorKeys(transactionalIds); - DescribeTransactionsRequest.Builder request = handler.buildRequest(1, keys); + DescribeTransactionsRequest.Builder request = handler.buildBatchedRequest(1, keys); assertEquals(transactionalIds, new HashSet<>(request.data.transactionalIds())); } diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/FenceProducersHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/FenceProducersHandlerTest.java new file mode 100644 index 0000000000000..08e856ec3fa16 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/FenceProducersHandlerTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.admin.internals; + +import org.apache.kafka.clients.admin.internals.AdminApiHandler.ApiResult; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.message.InitProducerIdResponseData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.InitProducerIdRequest; +import org.apache.kafka.common.requests.InitProducerIdResponse; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.ProducerIdAndEpoch; +import org.junit.jupiter.api.Test; + +import java.util.Set; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singleton; +import static java.util.Collections.singletonList; +import static org.apache.kafka.common.utils.Utils.mkSet; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class FenceProducersHandlerTest { + private final LogContext logContext = new LogContext(); + private final Node node = new Node(1, "host", 1234); + + @Test + public void testBuildRequest() { + FenceProducersHandler handler = new FenceProducersHandler(logContext); + mkSet("foo", "bar", "baz").forEach(transactionalId -> assertLookup(handler, transactionalId)); + } + + @Test + public void testHandleSuccessfulResponse() { + String transactionalId = "foo"; + CoordinatorKey key = CoordinatorKey.byTransactionalId(transactionalId); + + FenceProducersHandler handler = new FenceProducersHandler(logContext); + + short epoch = 57; + long producerId = 7; + InitProducerIdResponse response = new InitProducerIdResponse(new InitProducerIdResponseData() + .setProducerEpoch(epoch) + .setProducerId(producerId)); + + ApiResult result = handler.handleSingleResponse( + node, key, response); + + assertEquals(emptyList(), result.unmappedKeys); + assertEquals(emptyMap(), result.failedKeys); + assertEquals(singleton(key), result.completedKeys.keySet()); + + ProducerIdAndEpoch expected = new ProducerIdAndEpoch(producerId, epoch); + assertEquals(expected, result.completedKeys.get(key)); + } + + @Test + public void testHandleErrorResponse() { + String transactionalId = "foo"; + FenceProducersHandler handler = new FenceProducersHandler(logContext); + assertFatalError(handler, transactionalId, Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED); + assertFatalError(handler, transactionalId, Errors.TRANSACTIONAL_ID_NOT_FOUND); + assertFatalError(handler, transactionalId, Errors.UNKNOWN_SERVER_ERROR); + assertFatalError(handler, transactionalId, Errors.INVALID_PRODUCER_EPOCH); + assertRetriableError(handler, transactionalId, Errors.COORDINATOR_LOAD_IN_PROGRESS); + assertUnmappedKey(handler, transactionalId, Errors.NOT_COORDINATOR); + assertUnmappedKey(handler, transactionalId, Errors.COORDINATOR_NOT_AVAILABLE); + } + + private void assertFatalError( + FenceProducersHandler handler, + String transactionalId, + Errors error + ) { + CoordinatorKey key = CoordinatorKey.byTransactionalId(transactionalId); + ApiResult result = handleResponseError(handler, transactionalId, error); + assertEquals(emptyList(), result.unmappedKeys); + assertEquals(mkSet(key), result.failedKeys.keySet()); + + Throwable throwable = result.failedKeys.get(key); + assertTrue(error.exception().getClass().isInstance(throwable)); + } + + private void assertRetriableError( + FenceProducersHandler handler, + String transactionalId, + Errors error + ) { + ApiResult result = handleResponseError(handler, transactionalId, error); + assertEquals(emptyList(), result.unmappedKeys); + assertEquals(emptyMap(), result.failedKeys); + } + + private void assertUnmappedKey( + FenceProducersHandler handler, + String transactionalId, + Errors error + ) { + CoordinatorKey key = CoordinatorKey.byTransactionalId(transactionalId); + ApiResult result = handleResponseError(handler, transactionalId, error); + assertEquals(emptyMap(), result.failedKeys); + assertEquals(singletonList(key), result.unmappedKeys); + } + + private ApiResult handleResponseError( + FenceProducersHandler handler, + String transactionalId, + Errors error + ) { + int brokerId = 1; + + CoordinatorKey key = CoordinatorKey.byTransactionalId(transactionalId); + Set keys = mkSet(key); + + InitProducerIdResponse response = new InitProducerIdResponse(new InitProducerIdResponseData() + .setErrorCode(error.code())); + + ApiResult result = handler.handleResponse(node, keys, response); + assertEquals(emptyMap(), result.completedKeys); + return result; + } + + private void assertLookup(FenceProducersHandler handler, String transactionalId) { + CoordinatorKey key = CoordinatorKey.byTransactionalId(transactionalId); + InitProducerIdRequest.Builder request = handler.buildSingleRequest(1, key); + assertEquals(transactionalId, request.data.transactionalId()); + assertEquals(1, request.data.transactionTimeoutMs()); + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/ListConsumerGroupOffsetsHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/ListConsumerGroupOffsetsHandlerTest.java index 9c9bb1e58adb5..27597ce035b00 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/internals/ListConsumerGroupOffsetsHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/ListConsumerGroupOffsetsHandlerTest.java @@ -56,7 +56,7 @@ public class ListConsumerGroupOffsetsHandlerTest { @Test public void testBuildRequest() { ListConsumerGroupOffsetsHandler handler = new ListConsumerGroupOffsetsHandler(groupId, tps, logContext); - OffsetFetchRequest request = handler.buildRequest(1, singleton(CoordinatorKey.byGroupId(groupId))).build(); + OffsetFetchRequest request = handler.buildBatchedRequest(1, singleton(CoordinatorKey.byGroupId(groupId))).build(); assertEquals(groupId, request.data().groups().get(0).groupId()); assertEquals(2, request.data().groups().get(0).topics().size()); assertEquals(2, request.data().groups().get(0).topics().get(0).partitionIndexes().size()); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/ListTransactionsHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/ListTransactionsHandlerTest.java index a8923d1aa2c9c..3c54fad65e8f9 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/internals/ListTransactionsHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/ListTransactionsHandlerTest.java @@ -52,7 +52,7 @@ public void testBuildRequestWithoutFilters() { BrokerKey brokerKey = new BrokerKey(OptionalInt.of(brokerId)); ListTransactionsOptions options = new ListTransactionsOptions(); ListTransactionsHandler handler = new ListTransactionsHandler(options, logContext); - ListTransactionsRequest request = handler.buildRequest(brokerId, singleton(brokerKey)).build(); + ListTransactionsRequest request = handler.buildBatchedRequest(brokerId, singleton(brokerKey)).build(); assertEquals(Collections.emptyList(), request.data().producerIdFilters()); assertEquals(Collections.emptyList(), request.data().stateFilters()); } @@ -65,7 +65,7 @@ public void testBuildRequestWithFilteredProducerId() { ListTransactionsOptions options = new ListTransactionsOptions() .filterProducerIds(singleton(filteredProducerId)); ListTransactionsHandler handler = new ListTransactionsHandler(options, logContext); - ListTransactionsRequest request = handler.buildRequest(brokerId, singleton(brokerKey)).build(); + ListTransactionsRequest request = handler.buildBatchedRequest(brokerId, singleton(brokerKey)).build(); assertEquals(Collections.singletonList(filteredProducerId), request.data().producerIdFilters()); assertEquals(Collections.emptyList(), request.data().stateFilters()); } @@ -78,7 +78,7 @@ public void testBuildRequestWithFilteredState() { ListTransactionsOptions options = new ListTransactionsOptions() .filterStates(singleton(filteredState)); ListTransactionsHandler handler = new ListTransactionsHandler(options, logContext); - ListTransactionsRequest request = handler.buildRequest(brokerId, singleton(brokerKey)).build(); + ListTransactionsRequest request = handler.buildBatchedRequest(brokerId, singleton(brokerKey)).build(); assertEquals(Collections.singletonList(filteredState.toString()), request.data().stateFilters()); assertEquals(Collections.emptyList(), request.data().producerIdFilters()); } diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/internals/RemoveMembersFromConsumerGroupHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/internals/RemoveMembersFromConsumerGroupHandlerTest.java index 6f5dfda5bc307..3ecd1f10ee2f6 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/internals/RemoveMembersFromConsumerGroupHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/internals/RemoveMembersFromConsumerGroupHandlerTest.java @@ -55,7 +55,7 @@ public class RemoveMembersFromConsumerGroupHandlerTest { @Test public void testBuildRequest() { RemoveMembersFromConsumerGroupHandler handler = new RemoveMembersFromConsumerGroupHandler(groupId, members, logContext); - LeaveGroupRequest request = handler.buildRequest(1, singleton(CoordinatorKey.byGroupId(groupId))).build(); + LeaveGroupRequest request = handler.buildBatchedRequest(1, singleton(CoordinatorKey.byGroupId(groupId))).build(); assertEquals(groupId, request.data().groupId()); assertEquals(2, request.data().members().size()); } diff --git a/connect/api/src/main/java/org/apache/kafka/connect/source/ConnectorTransactionBoundaries.java b/connect/api/src/main/java/org/apache/kafka/connect/source/ConnectorTransactionBoundaries.java new file mode 100644 index 0000000000000..73746ba0993f3 --- /dev/null +++ b/connect/api/src/main/java/org/apache/kafka/connect/source/ConnectorTransactionBoundaries.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.source; + +/** + * An enum to represent the level of support for connector-defined transaction boundaries. + */ +public enum ConnectorTransactionBoundaries { + /** + * Signals that a connector can define its own transaction boundaries. + */ + SUPPORTED, + /** + * Signals that a connector cannot define its own transaction boundaries. + */ + UNSUPPORTED +} diff --git a/connect/api/src/main/java/org/apache/kafka/connect/source/ExactlyOnceSupport.java b/connect/api/src/main/java/org/apache/kafka/connect/source/ExactlyOnceSupport.java new file mode 100644 index 0000000000000..3980410e4b538 --- /dev/null +++ b/connect/api/src/main/java/org/apache/kafka/connect/source/ExactlyOnceSupport.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.source; + +/** + * An enum to represent the level of support for exactly-once delivery from a source connector. + */ +public enum ExactlyOnceSupport { + /** + * Signals that a connector supports exactly-once delivery. + */ + SUPPORTED, + /** + * Signals that a connector does not support exactly-once delivery. + */ + UNSUPPORTED; +} diff --git a/connect/api/src/main/java/org/apache/kafka/connect/source/SourceConnector.java b/connect/api/src/main/java/org/apache/kafka/connect/source/SourceConnector.java index 6e9694024d334..206abfad32034 100644 --- a/connect/api/src/main/java/org/apache/kafka/connect/source/SourceConnector.java +++ b/connect/api/src/main/java/org/apache/kafka/connect/source/SourceConnector.java @@ -18,6 +18,8 @@ import org.apache.kafka.connect.connector.Connector; +import java.util.Map; + /** * SourceConnectors implement the connector interface to pull data from another system and send * it to Kafka. @@ -28,4 +30,46 @@ public abstract class SourceConnector extends Connector { protected SourceConnectorContext context() { return (SourceConnectorContext) context; } + + /** + * Signals whether the connector supports exactly-once delivery guarantees with a proposed configuration. + * Developers can assume that worker-level exactly-once support is enabled when this method is invoked. + * + *

For backwards compatibility, the default implementation will return {@code null}, but connector developers are + * strongly encouraged to override this method to return a non-null value such as + * {@link ExactlyOnceSupport#SUPPORTED SUPPORTED} or {@link ExactlyOnceSupport#UNSUPPORTED UNSUPPORTED}. + * + *

Similar to {@link #validate(Map) validate}, this method may be called by the runtime before the + * {@link #start(Map) start} method is invoked when the connector will be run with exactly-once support. + * + * @param connectorConfig the configuration that will be used for the connector. + * @return {@link ExactlyOnceSupport#SUPPORTED} if the connector can provide exactly-once support with the given + * configuration, and {@link ExactlyOnceSupport#UNSUPPORTED} if it cannot. If this method is overridden by a + * connector, should not be {@code null}, but if {@code null}, it will be assumed that the connector cannot provide + * exactly-once guarantees. + * @since 3.2 + */ + public ExactlyOnceSupport exactlyOnceSupport(Map connectorConfig) { + return null; + } + + /** + * Signals whether the connector implementation is capable of defining the transaction boundaries for a + * connector with the given configuration. This method is called before {@link #start(Map)}, only when the + * runtime supports exactly-once and the connector configuration includes {@code transaction.boundary=connector}. + * + *

This method need not be implemented if the connector implementation does not support defining + * transaction boundaries. + * + * @param connectorConfig the configuration that will be used for the connector + * @return {@link ConnectorTransactionBoundaries#SUPPORTED} if the connector will define its own transaction boundaries, + * or {@link ConnectorTransactionBoundaries#UNSUPPORTED} otherwise. If this method is overridden by a + * connector, should not be {@code null}, but if {@code null}, it will be assumed that the connector cannot define its own + * transaction boundaries. + * @since 3.2 + * @see TransactionContext + */ + public ConnectorTransactionBoundaries canDefineTransactionBoundaries(Map connectorConfig) { + return ConnectorTransactionBoundaries.UNSUPPORTED; + } } diff --git a/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTask.java b/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTask.java index f5209e1ccab64..2159e68e8a88a 100644 --- a/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTask.java +++ b/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTask.java @@ -16,10 +16,11 @@ */ package org.apache.kafka.connect.source; -import org.apache.kafka.connect.connector.Task; import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.connect.connector.Task; import java.util.List; +import java.util.Locale; import java.util.Map; /** @@ -27,6 +28,51 @@ */ public abstract class SourceTask implements Task { + /** + * The configuration key that determines how source tasks will define transaction boundaries + * when exactly-once support is enabled. + */ + public static final String TRANSACTION_BOUNDARY_CONFIG = "transaction.boundary"; + + /** + * Represents the permitted values for the {@link #TRANSACTION_BOUNDARY_CONFIG} property. + */ + public enum TransactionBoundary { + /** + * A new transaction will be started and committed for every batch of records returned by {@link #poll()}. + */ + POLL, + /** + * Transactions will be started and committed on a user-defined time interval. + */ + INTERVAL, + /** + * Transactions will be defined by the connector itself, via a {@link TransactionContext}. + */ + CONNECTOR; + + /** + * The default transaction boundary style that will be used for source connectors when no style is explicitly + * configured. + */ + public static final TransactionBoundary DEFAULT = POLL; + + /** + * Parse a {@link TransactionBoundary} from the given string. + * @param property the string to parse; should not be null + * @return the {@link TransactionBoundary} whose name matches the given string + * @throws IllegalArgumentException if there is no transaction boundary type with the given name + */ + public static TransactionBoundary fromProperty(String property) { + return TransactionBoundary.valueOf(property.toUpperCase(Locale.ROOT).trim()); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + protected SourceTaskContext context; /** @@ -44,16 +90,13 @@ public void initialize(SourceTaskContext context) { public abstract void start(Map props); /** - *

* Poll this source task for new records. If no data is currently available, this method * should block but return control to the caller regularly (by returning {@code null}) in * order for the task to transition to the {@code PAUSED} state if requested to do so. - *

*

* The task will be {@link #stop() stopped} on a separate thread, and when that happens * this method is expected to unblock, quickly finish up any remaining processing, and * return. - *

* * @return a list of source records */ @@ -63,12 +106,10 @@ public void initialize(SourceTaskContext context) { *

* Commit the offsets, up to the offsets that have been returned by {@link #poll()}. This * method should block until the commit is complete. - *

*

* SourceTasks are not required to implement this functionality; Kafka Connect will record offsets * automatically. This hook is provided for systems that also need to store offsets internally * in their own system. - *

*/ public void commit() throws InterruptedException { // This space intentionally left blank. @@ -91,17 +132,14 @@ public void commit() throws InterruptedException { *

* Commit an individual {@link SourceRecord} when the callback from the producer client is received. This method is * also called when a record is filtered by a transformation, and thus will never be ACK'd by a broker. - *

*

* This is an alias for {@link #commitRecord(SourceRecord, RecordMetadata)} for backwards compatibility. The default * implementation of {@link #commitRecord(SourceRecord, RecordMetadata)} just calls this method. It is not necessary * to override both methods. - *

*

* SourceTasks are not required to implement this functionality; Kafka Connect will record offsets * automatically. This hook is provided for systems that also need to store offsets internally * in their own system. - *

* * @param record {@link SourceRecord} that was successfully sent via the producer or filtered by a transformation * @throws InterruptedException @@ -115,19 +153,16 @@ public void commitRecord(SourceRecord record) throws InterruptedException { /** *

* Commit an individual {@link SourceRecord} when the callback from the producer client is received. This method is - * also called when a record is filtered by a transformation or when {@link ConnectorConfig} "errors.tolerance" is set to "all" + * also called when a record is filtered by a transformation or when "errors.tolerance" is set to "all" * and thus will never be ACK'd by a broker. * In both cases {@code metadata} will be null. - *

*

* SourceTasks are not required to implement this functionality; Kafka Connect will record offsets * automatically. This hook is provided for systems that also need to store offsets internally * in their own system. - *

*

* The default implementation just calls {@link #commitRecord(SourceRecord)}, which is a nop by default. It is * not necessary to implement both methods. - *

* * @param record {@link SourceRecord} that was successfully sent via the producer, filtered by a transformation, or dropped on producer exception * @param metadata {@link RecordMetadata} record metadata returned from the broker, or null if the record was filtered or if producer exceptions are ignored diff --git a/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTaskContext.java b/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTaskContext.java index ddb0a78718351..5bd7889cb2aa5 100644 --- a/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTaskContext.java +++ b/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTaskContext.java @@ -38,4 +38,29 @@ public interface SourceTaskContext { * Get the OffsetStorageReader for this SourceTask. */ OffsetStorageReader offsetStorageReader(); + + /** + * Get a {@link TransactionContext} that can be used to define producer transaction boundaries + * when exactly-once support is enabled for the connector. + * + *

This method was added in Apache Kafka 3.2. Source tasks that use this method but want to + * maintain backward compatibility so they can also be deployed to older Connect runtimes + * should guard the call to this method with a try-catch block, since calling this method will result in a + * {@link NoSuchMethodException} or {@link NoClassDefFoundError} when the source connector is deployed to + * Connect runtimes older than Kafka 3.2. For example: + * 

+     *     TransactionContext transactionContext;
+     *     try {
+     *         transactionContext = context.transactionContext();
+     *     } catch (NoSuchMethodError | NoClassDefFoundError e) {
+     *         transactionContext = null;
+     *     }
+     *
+ * + * @return the transaction context, or null if the connector was not configured to specify transaction boundaries + * @since 3.2 + */ + default TransactionContext transactionContext() { + return null; + } } diff --git a/connect/api/src/main/java/org/apache/kafka/connect/source/TransactionContext.java b/connect/api/src/main/java/org/apache/kafka/connect/source/TransactionContext.java new file mode 100644 index 0000000000000..d8aac108be180 --- /dev/null +++ b/connect/api/src/main/java/org/apache/kafka/connect/source/TransactionContext.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.source; + +/** + * Provided to source tasks to allow them to define their own producer transaction boundaries when + * exactly-once support is enabled. + */ +public interface TransactionContext { + + /** + * Request a transaction commit after the next batch of records from {@link SourceTask#poll()} + * is processed. + */ + void commitTransaction(); + + /** + * Request a transaction commit after a source record is processed. The source record will be the + * last record in the committed transaction. + * @param record the record to commit the transaction after; may not be null. + */ + void commitTransaction(SourceRecord record); + + /** + * Requests a transaction abort the next batch of records from {@link SourceTask#poll()}. All of + * the records in that transaction will be discarded and will not appear in a committed transaction. + * However, offsets for that transaction will still be committed. If the data should be reprocessed, + * the task should not invoke this method and should instead throw an exception. + */ + void abortTransaction(); + + /** + * Requests a transaction abort after a source record is processed. The source record will be the + * last record in the aborted transaction. All of the records in that transaction will be discarded + * and will not appear in a committed transaction. However, offsets for that transaction will still + * be committed. If the data should be reprocessed, the task should not invoke this method and + * should instead throw an exception. + * @param record the record to abort the transaction after; may not be null. + */ + void abortTransaction(SourceRecord record); +} diff --git a/connect/basic-auth-extension/src/main/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilter.java b/connect/basic-auth-extension/src/main/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilter.java index 0299cbba0b546..d582a0ce9b125 100644 --- a/connect/basic-auth-extension/src/main/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilter.java +++ b/connect/basic-auth-extension/src/main/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilter.java @@ -18,7 +18,11 @@ package org.apache.kafka.connect.rest.basic.auth.extension; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.function.Predicate; import java.util.regex.Pattern; import javax.security.auth.login.Configuration; import javax.ws.rs.HttpMethod; @@ -45,7 +49,10 @@ public class JaasBasicAuthFilter implements ContainerRequestFilter { private static final Logger log = LoggerFactory.getLogger(JaasBasicAuthFilter.class); - private static final Pattern TASK_REQUEST_PATTERN = Pattern.compile("/?connectors/([^/]+)/tasks/?"); + private static final Set INTERNAL_REQUEST_MATCHERS = new HashSet<>(Arrays.asList( + new RequestMatcher("/?connectors/([^/]+)/tasks/?", HttpMethod.POST), + new RequestMatcher("/?connectors/[^/]+/fence/?", HttpMethod.PUT) + )); private static final String CONNECT_LOGIN_MODULE = "KafkaConnect"; static final String AUTHORIZATION = "Authorization"; @@ -53,13 +60,29 @@ public class JaasBasicAuthFilter implements ContainerRequestFilter { // Package-private for testing final Configuration configuration; + private static class RequestMatcher implements Predicate { + private final Pattern path; + private final String method; + + public RequestMatcher(String path, String method) { + this.path = Pattern.compile(path); + this.method = method; + } + + @Override + public boolean test(ContainerRequestContext requestContext) { + return requestContext.getMethod().equals(method) + && path.matcher(requestContext.getUriInfo().getPath()).matches(); + } + } + public JaasBasicAuthFilter(Configuration configuration) { this.configuration = configuration; } @Override public void filter(ContainerRequestContext requestContext) throws IOException { - if (isInternalTaskConfigRequest(requestContext)) { + if (isInternalRequest(requestContext)) { log.trace("Skipping authentication for internal request"); return; } @@ -82,12 +105,10 @@ public void filter(ContainerRequestContext requestContext) throws IOException { } } - private static boolean isInternalTaskConfigRequest(ContainerRequestContext requestContext) { - return requestContext.getMethod().equals(HttpMethod.POST) - && TASK_REQUEST_PATTERN.matcher(requestContext.getUriInfo().getPath()).matches(); + private boolean isInternalRequest(ContainerRequestContext requestContext) { + return INTERNAL_REQUEST_MATCHERS.stream().anyMatch(m -> m.test(requestContext)); } - public static class BasicAuthCallBackHandler implements CallbackHandler { private static final String BASIC = "basic"; diff --git a/connect/basic-auth-extension/src/test/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilterTest.java b/connect/basic-auth-extension/src/test/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilterTest.java index 561095f68218a..599ca0baa265c 100644 --- a/connect/basic-auth-extension/src/test/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilterTest.java +++ b/connect/basic-auth-extension/src/test/java/org/apache/kafka/connect/rest/basic/auth/extension/JaasBasicAuthFilterTest.java @@ -42,6 +42,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -58,7 +59,7 @@ public void testSuccess() throws IOException { ContainerRequestContext requestContext = setMock("Basic", "user", "password"); jaasBasicAuthFilter.filter(requestContext); - verify(requestContext).getMethod(); + verify(requestContext, atLeastOnce()).getMethod(); verify(requestContext).getHeaderString(JaasBasicAuthFilter.AUTHORIZATION); } @@ -69,7 +70,7 @@ public void testEmptyCredentialsFile() throws IOException { ContainerRequestContext requestContext = setMock("Basic", "user", "password"); jaasBasicAuthFilter.filter(requestContext); - verify(requestContext).getMethod(); + verify(requestContext, atLeastOnce()).getMethod(); verify(requestContext).getHeaderString(JaasBasicAuthFilter.AUTHORIZATION); } @@ -81,7 +82,7 @@ public void testBadCredential() throws IOException { jaasBasicAuthFilter.filter(requestContext); verify(requestContext).abortWith(any(Response.class)); - verify(requestContext).getMethod(); + verify(requestContext, atLeastOnce()).getMethod(); verify(requestContext).getHeaderString(JaasBasicAuthFilter.AUTHORIZATION); } @@ -93,7 +94,7 @@ public void testBadPassword() throws IOException { jaasBasicAuthFilter.filter(requestContext); verify(requestContext).abortWith(any(Response.class)); - verify(requestContext).getMethod(); + verify(requestContext, atLeastOnce()).getMethod(); verify(requestContext).getHeaderString(JaasBasicAuthFilter.AUTHORIZATION); } @@ -105,7 +106,7 @@ public void testUnknownBearer() throws IOException { jaasBasicAuthFilter.filter(requestContext); verify(requestContext).abortWith(any(Response.class)); - verify(requestContext).getMethod(); + verify(requestContext, atLeastOnce()).getMethod(); verify(requestContext).getHeaderString(JaasBasicAuthFilter.AUTHORIZATION); } @@ -117,7 +118,7 @@ public void testUnknownLoginModule() throws IOException { jaasBasicAuthFilter.filter(requestContext); verify(requestContext).abortWith(any(Response.class)); - verify(requestContext).getMethod(); + verify(requestContext, atLeastOnce()).getMethod(); verify(requestContext).getHeaderString(JaasBasicAuthFilter.AUTHORIZATION); } @@ -128,7 +129,7 @@ public void testUnknownCredentialsFile() throws IOException { jaasBasicAuthFilter.filter(requestContext); verify(requestContext).abortWith(any(Response.class)); - verify(requestContext).getMethod(); + verify(requestContext, atLeastOnce()).getMethod(); verify(requestContext).getHeaderString(JaasBasicAuthFilter.AUTHORIZATION); } @@ -139,14 +140,23 @@ public void testNoFileOption() throws IOException { jaasBasicAuthFilter.filter(requestContext); verify(requestContext).abortWith(any(Response.class)); - verify(requestContext).getMethod(); + verify(requestContext, atLeastOnce()).getMethod(); verify(requestContext).getHeaderString(JaasBasicAuthFilter.AUTHORIZATION); } @Test - public void testPostWithoutAppropriateCredential() throws IOException { + public void testInternalTaskConfigEndpointSkipped() throws IOException { + testInternalEndpointSkipped("connectors/connName/tasks"); + } + + @Test + public void testInternalZombieFencingEndpointSkipped() throws IOException { + testInternalEndpointSkipped("connectors/connName/fence"); + } + + private void testInternalEndpointSkipped(String endpoint) throws IOException { UriInfo uriInfo = mock(UriInfo.class); - when(uriInfo.getPath()).thenReturn("connectors/connName/tasks"); + when(uriInfo.getPath()).thenReturn(endpoint); ContainerRequestContext requestContext = mock(ContainerRequestContext.class); when(requestContext.getMethod()).thenReturn(HttpMethod.POST); @@ -158,7 +168,7 @@ public void testPostWithoutAppropriateCredential() throws IOException { jaasBasicAuthFilter.filter(requestContext); verify(uriInfo).getPath(); - verify(requestContext).getMethod(); + verify(requestContext, atLeastOnce()).getMethod(); verify(requestContext).getUriInfo(); } diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java index 8f692ca911612..2b6b045ae8f6e 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java @@ -26,6 +26,8 @@ import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.config.TopicConfig; +import org.apache.kafka.common.network.ListenerName; +import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.utils.Exit; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.connect.connector.Connector; @@ -112,7 +114,9 @@ public abstract class MirrorConnectorsIntegrationBaseTest { protected Properties backupBrokerProps = new Properties(); protected Map primaryWorkerProps = new HashMap<>(); protected Map backupWorkerProps = new HashMap<>(); - + protected ListenerName primaryBrokerListener = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT); + protected ListenerName backupBrokerListener = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT); + @BeforeEach public void startClusters() throws Exception { startClusters(new HashMap() {{ @@ -171,6 +175,7 @@ public void startClusters(Map additionalMM2Config) throws Except .brokerProps(primaryBrokerProps) .workerProps(primaryWorkerProps) .maskExitProcedures(false) + .brokerListener(primaryBrokerListener) .build(); backup = new EmbeddedConnectCluster.Builder() @@ -180,6 +185,7 @@ public void startClusters(Map additionalMM2Config) throws Except .brokerProps(backupBrokerProps) .workerProps(backupWorkerProps) .maskExitProcedures(false) + .brokerListener(backupBrokerListener) .build(); primary.start(); @@ -718,9 +724,12 @@ private void createTopics() { adminClientConfig.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, REQUEST_TIMEOUT_DURATION_MS); // create these topics before starting the connectors so we don't need to wait for discovery + adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, primary.kafka().bootstrapServers(primaryBrokerListener)); primary.kafka().createTopic("test-topic-1", NUM_PARTITIONS, 1, topicConfig, adminClientConfig); primary.kafka().createTopic("backup.test-topic-1", 1, 1, emptyMap, adminClientConfig); primary.kafka().createTopic("heartbeats", 1, 1, emptyMap, adminClientConfig); + + adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, backup.kafka().bootstrapServers(backupBrokerListener)); backup.kafka().createTopic("test-topic-1", NUM_PARTITIONS, 1, emptyMap, adminClientConfig); backup.kafka().createTopic("primary.test-topic-1", 1, 1, emptyMap, adminClientConfig); backup.kafka().createTopic("heartbeats", 1, 1, emptyMap, adminClientConfig); diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationSSLTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationSSLTest.java index eb2af4842b60a..b1b81f9b923cb 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationSSLTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationSSLTest.java @@ -24,7 +24,9 @@ import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.config.SslConfigs; import org.apache.kafka.common.config.types.Password; +import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.network.Mode; +import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.test.TestSslUtils; import org.apache.kafka.test.TestUtils; @@ -59,6 +61,8 @@ public void startClusters() throws Exception { // set SSL config for producer used by source task in MM2 mm2Props.putAll(sslProps.entrySet().stream().collect(Collectors.toMap( e -> BACKUP_CLUSTER_ALIAS + ".producer." + e.getKey(), e -> String.valueOf(e.getValue())))); + + backupBrokerListener = ListenerName.forSecurityProtocol(SecurityProtocol.SSL); super.startClusters(); } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectDistributed.java b/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectDistributed.java index 8d93e795911b9..ffd17e9195fba 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectDistributed.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectDistributed.java @@ -103,7 +103,7 @@ public Connect startConnect(Map workerProps) { URI advertisedUrl = rest.advertisedUrl(); String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort(); - // Create the admin client to be shared by all backing stores. + // Create the admin client to be shared by all worker-global backing stores. Map adminProps = new HashMap<>(config.originals()); ConnectUtils.addMetricsContextProperties(adminProps, config, kafkaClusterId); SharedTopicAdmin sharedAdmin = new SharedTopicAdmin(adminProps); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java index 89020a4d1741a..baf25ea23bd6f 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractHerder.java @@ -16,6 +16,8 @@ */ package org.apache.kafka.connect.runtime; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.config.AbstractConfig; import org.apache.kafka.common.config.Config; @@ -28,7 +30,8 @@ import org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy; import org.apache.kafka.connect.connector.policy.ConnectorClientConfigRequest; import org.apache.kafka.connect.errors.NotFoundException; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; +import org.apache.kafka.connect.sink.SinkConnector; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.runtime.isolation.Plugins; import org.apache.kafka.connect.runtime.rest.entities.ActiveTopicsInfo; import org.apache.kafka.connect.runtime.rest.entities.ConfigInfo; @@ -344,9 +347,11 @@ public ConnectorStateInfo.TaskState taskStatus(ConnectorTaskId id) { status.workerId(), status.trace()); } - protected Map validateBasicConnectorConfig(Connector connector, - ConfigDef configDef, - Map config) { + protected Map validateSinkConnectorConfig(SinkConnector connector, ConfigDef configDef, Map config) { + return configDef.validateAll(config); + } + + protected Map validateSourceConnectorConfig(SourceConnector connector, ConfigDef configDef, Map config) { return configDef.validateAll(config); } @@ -412,7 +417,23 @@ public Optional buildRestartPlan(RestartRequest request) { conf == null ? ConnectorType.UNKNOWN : connectorTypeForClass(conf.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG)) ); return Optional.of(new RestartPlan(request, stateInfo)); + } + + protected boolean connectorUsesConsumer(org.apache.kafka.connect.health.ConnectorType connectorType, Map connProps) { + return connectorType == org.apache.kafka.connect.health.ConnectorType.SINK; + } + protected boolean connectorUsesAdmin(org.apache.kafka.connect.health.ConnectorType connectorType, Map connProps) { + if (connectorType == org.apache.kafka.connect.health.ConnectorType.SOURCE) { + return SourceConnectorConfig.usesTopicCreation(connProps); + } else { + return SinkConnectorConfig.hasDlqTopicConfig(connProps); + } + } + + protected boolean connectorUsesProducer(org.apache.kafka.connect.health.ConnectorType connectorType, Map connProps) { + return connectorType == org.apache.kafka.connect.health.ConnectorType.SOURCE + || SinkConnectorConfig.hasDlqTopicConfig(connProps); } ConfigInfos validateConnectorConfig(Map connectorProps, boolean doLog) { @@ -426,22 +447,20 @@ ConfigInfos validateConnectorConfig(Map connectorProps, boolean Connector connector = getConnector(connType); org.apache.kafka.connect.health.ConnectorType connectorType; ClassLoader savedLoader = plugins().compareAndSwapLoaders(connector); + ConfigDef enrichedConfigDef; + Map validatedConnectorConfig; try { - ConfigDef baseConfigDef; if (connector instanceof SourceConnector) { - baseConfigDef = SourceConnectorConfig.configDef(); connectorType = org.apache.kafka.connect.health.ConnectorType.SOURCE; + enrichedConfigDef = ConnectorConfig.enrich(plugins(), SourceConnectorConfig.configDef(), connectorProps, false); + validatedConnectorConfig = validateSourceConnectorConfig((SourceConnector) connector, enrichedConfigDef, connectorProps); } else { - baseConfigDef = SinkConnectorConfig.configDef(); SinkConnectorConfig.validate(connectorProps); connectorType = org.apache.kafka.connect.health.ConnectorType.SINK; + enrichedConfigDef = ConnectorConfig.enrich(plugins(), SinkConnectorConfig.configDef(), connectorProps, false); + validatedConnectorConfig = validateSinkConnectorConfig((SinkConnector) connector, enrichedConfigDef, connectorProps); } - ConfigDef enrichedConfigDef = ConnectorConfig.enrich(plugins(), baseConfigDef, connectorProps, false); - Map validatedConnectorConfig = validateBasicConnectorConfig( - connector, - enrichedConfigDef, - connectorProps - ); + connectorProps.entrySet().stream() .filter(e -> e.getValue() == null) .map(Map.Entry::getKey) @@ -449,6 +468,7 @@ ConfigInfos validateConnectorConfig(Map connectorProps, boolean validatedConnectorConfig.computeIfAbsent(prop, ConfigValue::new) .addErrorMessage("Null value can not be supplied as the configuration value.") ); + List configValues = new ArrayList<>(validatedConnectorConfig.values()); Map configKeys = new LinkedHashMap<>(enrichedConfigDef.configKeys()); Set allGroups = new LinkedHashSet<>(enrichedConfigDef.groups()); @@ -482,38 +502,39 @@ ConfigInfos validateConnectorConfig(Map connectorProps, boolean ConfigInfos producerConfigInfos = null; ConfigInfos consumerConfigInfos = null; ConfigInfos adminConfigInfos = null; - if (connectorType.equals(org.apache.kafka.connect.health.ConnectorType.SOURCE)) { - producerConfigInfos = validateClientOverrides(connName, - ConnectorConfig.CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX, - connectorConfig, - ProducerConfig.configDef(), - connector.getClass(), - connectorType, - ConnectorClientConfigRequest.ClientType.PRODUCER, - connectorClientConfigOverridePolicy); - return mergeConfigInfos(connType, configInfos, producerConfigInfos); - } else { - consumerConfigInfos = validateClientOverrides(connName, - ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX, - connectorConfig, - ProducerConfig.configDef(), - connector.getClass(), - connectorType, - ConnectorClientConfigRequest.ClientType.CONSUMER, - connectorClientConfigOverridePolicy); - // check if topic for dead letter queue exists - String topic = connectorProps.get(SinkConnectorConfig.DLQ_TOPIC_NAME_CONFIG); - if (topic != null && !topic.isEmpty()) { - adminConfigInfos = validateClientOverrides(connName, - ConnectorConfig.CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX, - connectorConfig, - ProducerConfig.configDef(), - connector.getClass(), - connectorType, - ConnectorClientConfigRequest.ClientType.ADMIN, - connectorClientConfigOverridePolicy); - } + if (connectorUsesProducer(connectorType, connectorProps)) { + producerConfigInfos = validateClientOverrides( + connName, + ConnectorConfig.CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX, + connectorConfig, + ProducerConfig.configDef(), + connector.getClass(), + connectorType, + ConnectorClientConfigRequest.ClientType.PRODUCER, + connectorClientConfigOverridePolicy); + } + if (connectorUsesAdmin(connectorType, connectorProps)) { + adminConfigInfos = validateClientOverrides( + connName, + ConnectorConfig.CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX, + connectorConfig, + AdminClientConfig.configDef(), + connector.getClass(), + connectorType, + ConnectorClientConfigRequest.ClientType.ADMIN, + connectorClientConfigOverridePolicy); + } + if (connectorUsesConsumer(connectorType, connectorProps)) { + consumerConfigInfos = validateClientOverrides( + connName, + ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX, + connectorConfig, + ConsumerConfig.configDef(), + connector.getClass(), + connectorType, + ConnectorClientConfigRequest.ClientType.CONSUMER, + connectorClientConfigOverridePolicy); } return mergeConfigInfos(connType, configInfos, producerConfigInfos, consumerConfigInfos, adminConfigInfos); } finally { @@ -660,7 +681,7 @@ protected Connector getConnector(String connType) { return tempConnectors.computeIfAbsent(connType, k -> plugins().newConnector(k)); } - /* + /** * Retrieves ConnectorType for the corresponding connector class * @param connClass class of the connector */ @@ -668,6 +689,15 @@ public ConnectorType connectorTypeForClass(String connClass) { return ConnectorType.from(getConnector(connClass).getClass()); } + /** + * Retrieves ConnectorType for the class specified in the connector config + * @param connConfig the connector config; may not be null + * @return the {@link ConnectorType} of the connector + */ + public ConnectorType connectorTypeForConfig(Map connConfig) { + return connectorTypeForClass(connConfig.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG)); + } + /** * Checks a given {@link ConfigInfos} for validation error messages and adds an exception * to the given {@link Callback} if any were found. diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTask.java new file mode 100644 index 0000000000000..a3327464a4d84 --- /dev/null +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTask.java @@ -0,0 +1,642 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.runtime; + +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.CumulativeSum; +import org.apache.kafka.common.metrics.stats.Max; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.metrics.stats.Value; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.connect.errors.ConnectException; +import org.apache.kafka.connect.errors.RetriableException; +import org.apache.kafka.connect.header.Header; +import org.apache.kafka.connect.header.Headers; +import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator; +import org.apache.kafka.connect.runtime.errors.Stage; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.source.SourceTaskContext; +import org.apache.kafka.connect.storage.CloseableOffsetStorageReader; +import org.apache.kafka.connect.storage.ConnectorOffsetBackingStore; +import org.apache.kafka.connect.storage.Converter; +import org.apache.kafka.connect.storage.HeaderConverter; +import org.apache.kafka.connect.storage.OffsetStorageWriter; +import org.apache.kafka.connect.storage.StatusBackingStore; +import org.apache.kafka.connect.util.ConnectUtils; +import org.apache.kafka.connect.util.ConnectorTaskId; +import org.apache.kafka.connect.util.TopicAdmin; +import org.apache.kafka.connect.util.TopicCreation; +import org.apache.kafka.connect.util.TopicCreationGroup; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +import static org.apache.kafka.connect.runtime.WorkerConfig.TOPIC_TRACKING_ENABLE_CONFIG; + +/** + * WorkerTask that contains shared logic for running source tasks with either standard or exactly-once delivery guarantees. + */ +public abstract class AbstractWorkerSourceTask extends WorkerTask { + private static final Logger log = LoggerFactory.getLogger(AbstractWorkerSourceTask.class); + + private static final long SEND_FAILED_BACKOFF_MS = 100; + + /** + * Hook to define custom startup behavior before the calling {@link SourceTask#initialize(SourceTaskContext)} + * and {@link SourceTask#start(Map)}. + */ + protected abstract void prepareToInitializeTask(); + + /** + * Hook to define custom initialization behavior when preparing to begin the poll-convert-send loop for the first time, + * or when re-entering the loop after being paused. + */ + protected abstract void prepareToEnterSendLoop(); + + /** + * Hook to define custom periodic behavior to be performed at the top of every iteration of the poll-convert-send loop. + */ + protected abstract void beginSendIteration(); + + /** + * Hook to define custom periodic checks for health, metrics, etc. Called whenever {@link SourceTask#poll()} is about to be invoked. + */ + protected abstract void prepareToPollTask(); + + /** + * Invoked when a record provided by the task has been filtered out by a transform or the converter, + * or will be discarded due to failures during transformation or conversion. + * @param record the pre-transform record that has been dropped; never null. + */ + protected abstract void recordDropped(SourceRecord record); + + /** + * Invoked when a record is about to be dispatched to the producer. May be invoked multiple times for the same + * record if retriable errors are encountered. + * @param sourceRecord the pre-transform {@link SourceRecord} provided by the source task; never null. + * @param producerRecord the {@link ProducerRecord} produced by transforming and converting the + * {@code sourceRecord}; never null; + * @return a {@link SubmittedRecords.SubmittedRecord} to be {@link SubmittedRecords.SubmittedRecord#ack() acknowledged} + * if the corresponding producer record is ack'd by Kafka or {@link SubmittedRecords.SubmittedRecord#drop() dropped} + * if synchronously rejected by the producer. Can also be {@link Optional#empty()} if it is not necessary to track the acknowledgment + * of individual producer records + */ + protected abstract Optional prepareToSendRecord( + SourceRecord sourceRecord, + ProducerRecord producerRecord + ); + + /** + * Invoked when a record has been transformed, converted, and dispatched to the producer successfully via + * {@link Producer#send}. Does not guarantee that the record has been sent to Kafka or ack'd by the required number + * of brokers, but does guarantee that it will never be re-processed. + * @param record the pre-transform {@link SourceRecord} that was successfully dispatched to the producer; never null. + */ + protected abstract void recordDispatched(SourceRecord record); + + /** + * Invoked when an entire batch of records returned from {@link SourceTask#poll} has been transformed, converted, + * and either discarded due to transform/conversion errors, filtered by a transform, or dispatched to the producer + * successfully via {@link Producer#send}. Does not guarantee that the records have been sent to Kafka or ack'd by the + * required number of brokers, but does guarantee that none of the records in the batch will ever be re-processed during + * the lifetime of this task. At most one record batch is polled from the task in between calls to this method. + */ + protected abstract void batchDispatched(); + + /** + * Invoked when a record has been sent and ack'd by the Kafka cluster. Note that this method may be invoked + * concurrently and should therefore be made thread-safe. + * @param sourceRecord the pre-transform {@link SourceRecord} that was successfully sent to Kafka; never null. + * @param producerRecord the {@link ProducerRecord} produced by transforming and converting the + * {@code sourceRecord}; never null; + * @param recordMetadata the {@link RecordMetadata} for the corresponding producer record; never null. + */ + protected abstract void recordSent( + SourceRecord sourceRecord, + ProducerRecord producerRecord, + RecordMetadata recordMetadata + ); + + /** + * Invoked when a record given to {@link Producer#send(ProducerRecord, Callback)} has failed with a non-retriable error. + * @param synchronous whether the error occurred during the invocation of {@link Producer#send(ProducerRecord, Callback)}. + * If {@code false}, indicates that the error was reported asynchronously by the producer by a {@link Callback} + * @param producerRecord the {@link ProducerRecord} that the producer failed to send; never null + * @param preTransformRecord the pre-transform {@link SourceRecord} that the producer record was derived from; never null + * @param e the exception that was either thrown from {@link Producer#send(ProducerRecord, Callback)}, or reported by the producer + * via {@link Callback} after the call to {@link Producer#send(ProducerRecord, Callback)} completed + */ + protected abstract void producerSendFailed( + boolean synchronous, + ProducerRecord producerRecord, + SourceRecord preTransformRecord, + Exception e + ); + + /** + * Invoked when no more records will be polled from the task or dispatched to the producer. Should attempt to + * commit the offsets for any outstanding records when possible. + * @param failed whether the task is undergoing a healthy or an unhealthy shutdown + */ + protected abstract void finalOffsetCommit(boolean failed); + + + protected final WorkerConfig workerConfig; + protected final WorkerSourceTaskContext sourceTaskContext; + protected final OffsetStorageWriter offsetWriter; + protected final Producer producer; + + private final SourceTask task; + private final Converter keyConverter; + private final Converter valueConverter; + private final HeaderConverter headerConverter; + private final TransformationChain transformationChain; + private final TopicAdmin admin; + private final CloseableOffsetStorageReader offsetReader; + private final ConnectorOffsetBackingStore offsetStore; + private final SourceTaskMetricsGroup sourceTaskMetricsGroup; + private final CountDownLatch stopRequestedLatch; + private final boolean topicTrackingEnabled; + private final TopicCreation topicCreation; + private final Executor closeExecutor; + + // Visible for testing + List toSend; + protected Map taskConfig; + protected boolean started = false; + + protected AbstractWorkerSourceTask(ConnectorTaskId id, + SourceTask task, + TaskStatus.Listener statusListener, + TargetState initialState, + Converter keyConverter, + Converter valueConverter, + HeaderConverter headerConverter, + TransformationChain transformationChain, + WorkerSourceTaskContext sourceTaskContext, + Producer producer, + TopicAdmin admin, + Map topicGroups, + CloseableOffsetStorageReader offsetReader, + OffsetStorageWriter offsetWriter, + ConnectorOffsetBackingStore offsetStore, + WorkerConfig workerConfig, + ConnectMetrics connectMetrics, + ClassLoader loader, + Time time, + RetryWithToleranceOperator retryWithToleranceOperator, + StatusBackingStore statusBackingStore, + Executor closeExecutor) { + + super(id, statusListener, initialState, loader, connectMetrics, + retryWithToleranceOperator, time, statusBackingStore); + + this.workerConfig = workerConfig; + this.task = task; + this.keyConverter = keyConverter; + this.valueConverter = valueConverter; + this.headerConverter = headerConverter; + this.transformationChain = transformationChain; + this.producer = producer; + this.admin = admin; + this.offsetReader = offsetReader; + this.offsetWriter = offsetWriter; + this.offsetStore = offsetStore; + this.closeExecutor = closeExecutor; + this.sourceTaskContext = sourceTaskContext; + + this.stopRequestedLatch = new CountDownLatch(1); + this.sourceTaskMetricsGroup = new SourceTaskMetricsGroup(id, connectMetrics); + this.topicTrackingEnabled = workerConfig.getBoolean(TOPIC_TRACKING_ENABLE_CONFIG); + this.topicCreation = TopicCreation.newTopicCreation(workerConfig, topicGroups); + } + + @Override + public void initialize(TaskConfig taskConfig) { + try { + this.taskConfig = taskConfig.originalsStrings(); + } catch (Throwable t) { + log.error("{} Task failed initialization and will not be started.", this, t); + onFailure(t); + } + } + + @Override + protected void initializeAndStart() { + prepareToInitializeTask(); + // If we try to start the task at all by invoking initialize, then count this as + // "started" and expect a subsequent call to the task's stop() method + // to properly clean up any resources allocated by its initialize() or + // start() methods. If the task throws an exception during stop(), + // the worst thing that happens is another exception gets logged for an already- + // failed task + started = true; + task.initialize(sourceTaskContext); + task.start(taskConfig); + log.info("{} Source task finished initialization and start", this); + } + + @Override + public void cancel() { + super.cancel(); + // Preemptively close the offset reader in case the task is blocked on an offset read. + offsetReader.close(); + // We proactively close the producer here as the main work thread for the task may + // be blocked indefinitely in a call to Producer::send if automatic topic creation is + // not enabled on either the connector or the Kafka cluster. Closing the producer should + // unblock it in that case and allow shutdown to proceed normally. + // With a duration of 0, the producer's own shutdown logic should be fairly quick, + // but closing user-pluggable classes like interceptors may lag indefinitely. So, we + // call close on a separate thread in order to avoid blocking the herder's tick thread. + closeExecutor.execute(() -> closeProducer(Duration.ZERO)); + } + + @Override + public void stop() { + super.stop(); + stopRequestedLatch.countDown(); + } + + @Override + public void removeMetrics() { + Utils.closeQuietly(sourceTaskMetricsGroup::close, "source task metrics tracker"); + super.removeMetrics(); + } + + @Override + protected void close() { + if (started) { + Utils.closeQuietly(task::stop, "source task"); + } + + closeProducer(Duration.ofSeconds(30)); + + if (admin != null) { + Utils.closeQuietly(() -> admin.close(Duration.ofSeconds(30)), "source task admin"); + } + Utils.closeQuietly(transformationChain, "transformation chain"); + Utils.closeQuietly(retryWithToleranceOperator, "retry operator"); + Utils.closeQuietly(offsetReader, "offset reader"); + Utils.closeQuietly(offsetStore::stop, "offset backing store"); + } + + private void closeProducer(Duration duration) { + if (producer != null) { + Utils.closeQuietly(() -> producer.close(duration), "source task producer"); + } + } + + @Override + public void execute() { + try { + prepareToEnterSendLoop(); + while (!isStopping()) { + beginSendIteration(); + + if (shouldPause()) { + onPause(); + if (awaitUnpause()) { + onResume(); + prepareToEnterSendLoop(); + } + continue; + } + + if (toSend == null) { + prepareToPollTask(); + + log.trace("{} Nothing to send to Kafka. Polling source for additional records", this); + long start = time.milliseconds(); + toSend = poll(); + if (toSend != null) { + recordPollReturned(toSend.size(), time.milliseconds() - start); + } + } + if (toSend == null) + continue; + log.trace("{} About to send {} records to Kafka", this, toSend.size()); + if (sendRecords()) { + batchDispatched(); + } else { + stopRequestedLatch.await(SEND_FAILED_BACKOFF_MS, TimeUnit.MILLISECONDS); + } + } + } catch (InterruptedException e) { + // Ignore and allow to exit. + } catch (RuntimeException e) { + try { + finalOffsetCommit(true); + } catch (Exception offsetException) { + log.error("Failed to commit offsets for already-failing task", offsetException); + } + throw e; + } + finalOffsetCommit(false); + } + + /** + * Try to send a batch of records. If a send fails and is retriable, this saves the remainder of the batch so it can + * be retried after backing off. If a send fails and is not retriable, this will throw a ConnectException. + * @return true if all messages were sent, false if some need to be retried + */ + // Visible for testing + boolean sendRecords() { + int processed = 0; + recordBatch(toSend.size()); + final SourceRecordWriteCounter counter = + toSend.size() > 0 ? new SourceRecordWriteCounter(toSend.size(), sourceTaskMetricsGroup) : null; + for (final SourceRecord preTransformRecord : toSend) { + retryWithToleranceOperator.sourceRecord(preTransformRecord); + final SourceRecord record = transformationChain.apply(preTransformRecord); + final ProducerRecord producerRecord = convertTransformedRecord(record); + if (producerRecord == null || retryWithToleranceOperator.failed()) { + counter.skipRecord(); + recordDropped(preTransformRecord); + continue; + } + + log.trace("{} Appending record to the topic {} with key {}, value {}", this, record.topic(), record.key(), record.value()); + Optional submittedRecord = prepareToSendRecord(preTransformRecord, producerRecord); + try { + final String topic = producerRecord.topic(); + maybeCreateTopic(topic); + producer.send( + producerRecord, + (recordMetadata, e) -> { + if (e != null) { + log.debug("{} failed to send record to {}: ", AbstractWorkerSourceTask.this, topic, e); + log.trace("{} Failed record: {}", AbstractWorkerSourceTask.this, preTransformRecord); + producerSendFailed(false, producerRecord, preTransformRecord, e); + } else { + counter.completeRecord(); + log.trace("{} Wrote record successfully: topic {} partition {} offset {}", + AbstractWorkerSourceTask.this, + recordMetadata.topic(), recordMetadata.partition(), + recordMetadata.offset()); + recordSent(preTransformRecord, producerRecord, recordMetadata); + submittedRecord.ifPresent(SubmittedRecords.SubmittedRecord::ack); + if (topicTrackingEnabled) { + recordActiveTopic(producerRecord.topic()); + } + } + }); + // Note that this will cause retries to take place within a transaction + } catch (RetriableException | org.apache.kafka.common.errors.RetriableException e) { + log.warn("{} Failed to send record to topic '{}' and partition '{}'. Backing off before retrying: ", + this, producerRecord.topic(), producerRecord.partition(), e); + toSend = toSend.subList(processed, toSend.size()); + submittedRecord.ifPresent(SubmittedRecords.SubmittedRecord::drop); + counter.retryRemaining(); + return false; + } catch (ConnectException e) { + log.warn("{} Failed to send record to topic '{}' and partition '{}' due to an unrecoverable exception: ", + this, producerRecord.topic(), producerRecord.partition(), e); + log.trace("{} Failed to send {} with unrecoverable exception: ", this, producerRecord, e); + throw e; + } catch (KafkaException e) { + producerSendFailed(true, producerRecord, preTransformRecord, e); + } + processed++; + recordDispatched(preTransformRecord); + } + toSend = null; + return true; + } + + protected List poll() throws InterruptedException { + try { + return task.poll(); + } catch (RetriableException | org.apache.kafka.common.errors.RetriableException e) { + log.warn("{} failed to poll records from SourceTask. Will retry operation.", this, e); + // Do nothing. Let the framework poll whenever it's ready. + return null; + } + } + + /** + * Convert the source record into a producer record. + * + * @param record the transformed record + * @return the producer record which can sent over to Kafka. A null is returned if the input is null or + * if an error was encountered during any of the converter stages. + */ + protected ProducerRecord convertTransformedRecord(SourceRecord record) { + if (record == null) { + return null; + } + + RecordHeaders headers = retryWithToleranceOperator.execute(() -> convertHeaderFor(record), Stage.HEADER_CONVERTER, headerConverter.getClass()); + + byte[] key = retryWithToleranceOperator.execute(() -> keyConverter.fromConnectData(record.topic(), headers, record.keySchema(), record.key()), + Stage.KEY_CONVERTER, keyConverter.getClass()); + + byte[] value = retryWithToleranceOperator.execute(() -> valueConverter.fromConnectData(record.topic(), headers, record.valueSchema(), record.value()), + Stage.VALUE_CONVERTER, valueConverter.getClass()); + + if (retryWithToleranceOperator.failed()) { + return null; + } + + return new ProducerRecord<>(record.topic(), record.kafkaPartition(), + ConnectUtils.checkAndConvertTimestamp(record.timestamp()), key, value, headers); + } + + // Due to transformations that may change the destination topic of a record (such as + // RegexRouter) topic creation can not be batched for multiple topics + private void maybeCreateTopic(String topic) { + if (!topicCreation.isTopicCreationRequired(topic)) { + log.trace("Topic creation by the connector is disabled or the topic {} was previously created." + + "If auto.create.topics.enable is enabled on the broker, " + + "the topic will be created with default settings", topic); + return; + } + log.info("The task will send records to topic '{}' for the first time. Checking " + + "whether topic exists", topic); + Map existing = admin.describeTopics(topic); + if (!existing.isEmpty()) { + log.info("Topic '{}' already exists.", topic); + topicCreation.addTopic(topic); + return; + } + + log.info("Creating topic '{}'", topic); + TopicCreationGroup topicGroup = topicCreation.findFirstGroup(topic); + log.debug("Topic '{}' matched topic creation group: {}", topic, topicGroup); + NewTopic newTopic = topicGroup.newTopic(topic); + + TopicAdmin.TopicCreationResponse response = admin.createOrFindTopics(newTopic); + if (response.isCreated(newTopic.name())) { + topicCreation.addTopic(topic); + log.info("Created topic '{}' using creation group {}", newTopic, topicGroup); + } else if (response.isExisting(newTopic.name())) { + topicCreation.addTopic(topic); + log.info("Found existing topic '{}'", newTopic); + } else { + // The topic still does not exist and could not be created, so treat it as a task failure + log.warn("Request to create new topic '{}' failed", topic); + throw new ConnectException("Task failed to create new topic " + newTopic + ". Ensure " + + "that the task is authorized to create topics or that the topic exists and " + + "restart the task"); + } + } + + protected RecordHeaders convertHeaderFor(SourceRecord record) { + Headers headers = record.headers(); + RecordHeaders result = new RecordHeaders(); + if (headers != null) { + String topic = record.topic(); + for (Header header : headers) { + String key = header.key(); + byte[] rawHeader = headerConverter.fromConnectHeader(topic, key, header.schema(), header.value()); + result.add(key, rawHeader); + } + } + return result; + } + + protected void commitTaskRecord(SourceRecord record, RecordMetadata metadata) { + try { + task.commitRecord(record, metadata); + } catch (Throwable t) { + log.error("{} Exception thrown while calling task.commitRecord()", this, t); + } + } + + protected void commitSourceTask() { + try { + this.task.commit(); + } catch (Throwable t) { + log.error("{} Exception thrown while calling task.commit()", this, t); + } + } + + protected void recordPollReturned(int numRecordsInBatch, long duration) { + sourceTaskMetricsGroup.recordPoll(numRecordsInBatch, duration); + } + + SourceTaskMetricsGroup sourceTaskMetricsGroup() { + return sourceTaskMetricsGroup; + } + + static class SourceRecordWriteCounter { + private final SourceTaskMetricsGroup metricsGroup; + private final int batchSize; + private boolean completed = false; + private int counter; + public SourceRecordWriteCounter(int batchSize, SourceTaskMetricsGroup metricsGroup) { + assert batchSize > 0; + assert metricsGroup != null; + this.batchSize = batchSize; + counter = batchSize; + this.metricsGroup = metricsGroup; + } + public void skipRecord() { + if (counter > 0 && --counter == 0) { + finishedAllWrites(); + } + } + public void completeRecord() { + if (counter > 0 && --counter == 0) { + finishedAllWrites(); + } + } + public void retryRemaining() { + finishedAllWrites(); + } + private void finishedAllWrites() { + if (!completed) { + metricsGroup.recordWrite(batchSize - counter); + completed = true; + } + } + } + + static class SourceTaskMetricsGroup { + private final ConnectMetrics.MetricGroup metricGroup; + private final Sensor sourceRecordPoll; + private final Sensor sourceRecordWrite; + private final Sensor sourceRecordActiveCount; + private final Sensor pollTime; + private int activeRecordCount; + + public SourceTaskMetricsGroup(ConnectorTaskId id, ConnectMetrics connectMetrics) { + ConnectMetricsRegistry registry = connectMetrics.registry(); + metricGroup = connectMetrics.group(registry.sourceTaskGroupName(), + registry.connectorTagName(), id.connector(), + registry.taskTagName(), Integer.toString(id.task())); + // remove any previously created metrics in this group to prevent collisions. + metricGroup.close(); + + sourceRecordPoll = metricGroup.sensor("source-record-poll"); + sourceRecordPoll.add(metricGroup.metricName(registry.sourceRecordPollRate), new Rate()); + sourceRecordPoll.add(metricGroup.metricName(registry.sourceRecordPollTotal), new CumulativeSum()); + + sourceRecordWrite = metricGroup.sensor("source-record-write"); + sourceRecordWrite.add(metricGroup.metricName(registry.sourceRecordWriteRate), new Rate()); + sourceRecordWrite.add(metricGroup.metricName(registry.sourceRecordWriteTotal), new CumulativeSum()); + + pollTime = metricGroup.sensor("poll-batch-time"); + pollTime.add(metricGroup.metricName(registry.sourceRecordPollBatchTimeMax), new Max()); + pollTime.add(metricGroup.metricName(registry.sourceRecordPollBatchTimeAvg), new Avg()); + + sourceRecordActiveCount = metricGroup.sensor("source-record-active-count"); + sourceRecordActiveCount.add(metricGroup.metricName(registry.sourceRecordActiveCount), new Value()); + sourceRecordActiveCount.add(metricGroup.metricName(registry.sourceRecordActiveCountMax), new Max()); + sourceRecordActiveCount.add(metricGroup.metricName(registry.sourceRecordActiveCountAvg), new Avg()); + } + + void close() { + metricGroup.close(); + } + + void recordPoll(int batchSize, long duration) { + sourceRecordPoll.record(batchSize); + pollTime.record(duration); + activeRecordCount += batchSize; + sourceRecordActiveCount.record(activeRecordCount); + } + + void recordWrite(int recordCount) { + sourceRecordWrite.record(recordCount); + activeRecordCount -= recordCount; + activeRecordCount = Math.max(0, activeRecordCount); + sourceRecordActiveCount.record(activeRecordCount); + } + + protected ConnectMetrics.MetricGroup metricGroup() { + return metricGroup; + } + } +} diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java index cf567450fe2a0..f301439da8356 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java @@ -112,6 +112,9 @@ public class ConnectMetricsRegistry { public final MetricNameTemplate dlqProduceRequests; public final MetricNameTemplate dlqProduceFailures; public final MetricNameTemplate lastErrorTimestamp; + public final MetricNameTemplate transactionSizeMin; + public final MetricNameTemplate transactionSizeMax; + public final MetricNameTemplate transactionSizeAvg; public Map connectorStatusMetrics; @@ -207,6 +210,16 @@ public ConnectMetricsRegistry(Set tags) { "completely written to Kafka.", sourceTaskTags); + transactionSizeMin = createTemplate("transaction-size-min", SOURCE_TASK_GROUP_NAME, + "The number of records in the smallest transaction the task has committed so far. ", + sourceTaskTags); + transactionSizeMax = createTemplate("transaction-size-max", SOURCE_TASK_GROUP_NAME, + "The number of records in the largest transaction the task has committed so far.", + sourceTaskTags); + transactionSizeAvg = createTemplate("transaction-size-avg", SOURCE_TASK_GROUP_NAME, + "The average number of records in the transactions the task has committed so far.", + sourceTaskTags); + /***** Sink worker task level *****/ Set sinkTaskTags = new LinkedHashSet<>(tags); sinkTaskTags.add(CONNECTOR_TAG_NAME); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTask.java new file mode 100644 index 0000000000000..fe08819f65676 --- /dev/null +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTask.java @@ -0,0 +1,525 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.runtime; + +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.errors.InvalidProducerEpochException; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.Max; +import org.apache.kafka.common.metrics.stats.Min; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.connect.errors.ConnectException; +import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.source.SourceTask.TransactionBoundary; +import org.apache.kafka.connect.storage.CloseableOffsetStorageReader; +import org.apache.kafka.connect.storage.ClusterConfigState; +import org.apache.kafka.connect.storage.ConnectorOffsetBackingStore; +import org.apache.kafka.connect.storage.Converter; +import org.apache.kafka.connect.storage.HeaderConverter; +import org.apache.kafka.connect.storage.OffsetStorageWriter; +import org.apache.kafka.connect.storage.StatusBackingStore; +import org.apache.kafka.connect.util.ConnectorTaskId; +import org.apache.kafka.connect.util.LoggingContext; +import org.apache.kafka.connect.util.TopicAdmin; +import org.apache.kafka.connect.util.TopicCreationGroup; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; + + +/** + * WorkerTask that uses a SourceTask to ingest data into Kafka, with support for exactly-once delivery guarantees. + */ +class ExactlyOnceWorkerSourceTask extends AbstractWorkerSourceTask { + private static final Logger log = LoggerFactory.getLogger(ExactlyOnceWorkerSourceTask.class); + + private boolean transactionOpen; + private final LinkedHashMap commitableRecords; + + private final TransactionManager transactionManager; + private final TransactionMetricsGroup transactionMetrics; + + private final ConnectorOffsetBackingStore offsetBackingStore; + private final Runnable preProducerCheck; + private final Runnable postProducerCheck; + + public ExactlyOnceWorkerSourceTask(ConnectorTaskId id, + SourceTask task, + TaskStatus.Listener statusListener, + TargetState initialState, + Converter keyConverter, + Converter valueConverter, + HeaderConverter headerConverter, + TransformationChain transformationChain, + Producer producer, + TopicAdmin admin, + Map topicGroups, + CloseableOffsetStorageReader offsetReader, + OffsetStorageWriter offsetWriter, + ConnectorOffsetBackingStore offsetBackingStore, + WorkerConfig workerConfig, + ClusterConfigState configState, + ConnectMetrics connectMetrics, + ClassLoader loader, + Time time, + RetryWithToleranceOperator retryWithToleranceOperator, + StatusBackingStore statusBackingStore, + SourceConnectorConfig sourceConfig, + Executor closeExecutor, + Runnable preProducerCheck, + Runnable postProducerCheck) { + super(id, task, statusListener, initialState, keyConverter, valueConverter, headerConverter, transformationChain, + new WorkerSourceTaskContext(offsetReader, id, configState, buildTransactionContext(sourceConfig)), + producer, admin, topicGroups, offsetReader, offsetWriter, offsetBackingStore, workerConfig, connectMetrics, + loader, time, retryWithToleranceOperator, statusBackingStore, closeExecutor); + + this.transactionOpen = false; + this.commitableRecords = new LinkedHashMap<>(); + this.offsetBackingStore = offsetBackingStore; + + this.preProducerCheck = preProducerCheck; + this.postProducerCheck = postProducerCheck; + + this.transactionManager = buildTransactionManager(workerConfig, sourceConfig, sourceTaskContext.transactionContext()); + this.transactionMetrics = new TransactionMetricsGroup(id, connectMetrics); + } + + private static WorkerTransactionContext buildTransactionContext(SourceConnectorConfig sourceConfig) { + return TransactionBoundary.CONNECTOR.equals(sourceConfig.transactionBoundary()) + ? new WorkerTransactionContext() + : null; + } + + @Override + protected void prepareToInitializeTask() { + preProducerCheck.run(); + + // Try not to start up the offset store (which has its own producer and consumer) if we've already been shut down at this point + if (isStopping()) + return; + offsetBackingStore.start(); + + // Try not to initialize the transactional producer (which may accidentally fence out other, later task generations) if we've already + // been shut down at this point + if (isStopping()) + return; + producer.initTransactions(); + + postProducerCheck.run(); + } + + @Override + protected void prepareToEnterSendLoop() { + transactionManager.initialize(); + } + + @Override + protected void beginSendIteration() { + // No-op + } + + @Override + protected void prepareToPollTask() { + // No-op + } + + @Override + protected void recordDropped(SourceRecord record) { + synchronized (this) { + commitableRecords.put(record, null); + } + transactionManager.maybeCommitTransactionForRecord(record); + } + + @Override + protected Optional prepareToSendRecord( + SourceRecord sourceRecord, + ProducerRecord producerRecord + ) { + if (offsetBackingStore.primaryOffsetsTopic().equals(producerRecord.topic())) { + // This is to prevent deadlock that occurs when: + // 1. A task provides a record whose topic is the task's offsets topic + // 2. That record is dispatched to the task's producer in a transaction that remains open + // at least until the worker polls the task again + // 3. In the subsequent call to SourceTask::poll, the task requests offsets from the worker + // (which requires a read to the end of the offsets topic, and will block until any open + // transactions on the topic are either committed or aborted) + throw new ConnectException("Source tasks may not produce to their own offsets topics when exactly-once support is enabled"); + } + maybeBeginTransaction(); + return Optional.empty(); + } + + @Override + protected void recordDispatched(SourceRecord record) { + // Offsets are converted & serialized in the OffsetWriter + // Important: we only save offsets for the record after it has been accepted by the producer; this way, + // we commit those offsets if and only if the record is sent successfully. + offsetWriter.offset(record.sourcePartition(), record.sourceOffset()); + transactionMetrics.addRecord(); + transactionManager.maybeCommitTransactionForRecord(record); + } + + @Override + protected void batchDispatched() { + transactionManager.maybeCommitTransactionForBatch(); + } + + @Override + protected void recordSent( + SourceRecord sourceRecord, + ProducerRecord producerRecord, + RecordMetadata recordMetadata + ) { + synchronized (this) { + commitableRecords.put(sourceRecord, recordMetadata); + } + } + + @Override + protected void producerSendFailed( + boolean synchronous, + ProducerRecord producerRecord, + SourceRecord preTransformRecord, + Exception e + ) { + if (synchronous) { + throw maybeWrapProducerSendException( + "Unrecoverable exception trying to send", + e + ); + } else { + // No-op; all asynchronously-reported producer exceptions should be bubbled up again by Producer::commitTransaction + } + } + + @Override + protected void finalOffsetCommit(boolean failed) { + if (failed) { + log.debug("Skipping final offset commit as task has failed"); + return; + } + + // It should be safe to commit here even if we were in the middle of retrying on RetriableExceptions in the + // send loop since we only track source offsets for records that have been successfully dispatched to the + // producer. + // Any records that we were retrying on (and any records after them in the batch) won't be included in the + // transaction and their offsets won't be committed, but (unless the user has requested connector-defined + // transaction boundaries), it's better to commit some data than none. + transactionManager.maybeCommitFinalTransaction(); + } + + @Override + protected void onPause() { + super.onPause(); + // Commit the transaction now so that we don't end up with a hanging transaction, or worse, get fenced out + // and fail the task once unpaused + transactionManager.maybeCommitFinalTransaction(); + } + + private void maybeBeginTransaction() { + if (!transactionOpen) { + producer.beginTransaction(); + transactionOpen = true; + } + } + + private void commitTransaction() { + log.debug("{} Committing offsets", this); + + long started = time.milliseconds(); + + // We might have just aborted a transaction, in which case we'll have to begin a new one + // in order to commit offsets + maybeBeginTransaction(); + + AtomicReference flushError = new AtomicReference<>(); + Future offsetFlush = null; + if (offsetWriter.beginFlush()) { + // Now we can actually write the offsets to the internal topic. + offsetFlush = offsetWriter.doFlush((error, result) -> { + if (error != null) { + log.error("{} Failed to flush offsets to storage: ", ExactlyOnceWorkerSourceTask.this, error); + flushError.compareAndSet(null, error); + } else { + log.trace("{} Finished flushing offsets to storage", ExactlyOnceWorkerSourceTask.this); + } + }); + } + + // Commit the transaction + // Blocks until all outstanding records have been sent and ack'd + try { + producer.commitTransaction(); + if (offsetFlush != null) { + // Although it's guaranteed by the above call to Producer::commitTransaction that all outstanding + // records for the task's producer (including those sent to the offsets topic) have been delivered and + // ack'd, there is no guarantee that the producer callbacks for those records have been completed. So, + // we add this call to Future::get to ensure that these callbacks are invoked successfully before + // proceeding. + offsetFlush.get(); + } + } catch (Throwable t) { + flushError.compareAndSet(null, t); + } + + transactionOpen = false; + + Throwable error = flushError.get(); + if (error != null) { + recordCommitFailure(time.milliseconds() - started, null); + offsetWriter.cancelFlush(); + throw maybeWrapProducerSendException( + "Failed to flush offsets and/or records for task " + id, + error + ); + } + + transactionMetrics.commitTransaction(); + + long durationMillis = time.milliseconds() - started; + recordCommitSuccess(durationMillis); + log.debug("{} Finished commitOffsets successfully in {} ms", this, durationMillis); + + // No need for any synchronization here; all other accesses to this field take place in producer callbacks, + // which should all be completed by this point + commitableRecords.forEach(this::commitTaskRecord); + commitableRecords.clear(); + commitSourceTask(); + } + + private RuntimeException maybeWrapProducerSendException(String message, Throwable error) { + if (isPossibleTransactionTimeoutError(error)) { + return wrapTransactionTimeoutError(error); + } else { + return new ConnectException(message, error); + } + } + + private static boolean isPossibleTransactionTimeoutError(Throwable error) { + return error instanceof InvalidProducerEpochException + || error.getCause() instanceof InvalidProducerEpochException; + } + + private ConnectException wrapTransactionTimeoutError(Throwable error) { + return new ConnectException( + "The task " + id + " was unable to finish writing records to Kafka before its producer transaction expired. " + + "It may be necessary to reconfigure this connector in order for it to run healthily with exactly-once support. " + + "Options for this include: tune the connector's producer configuration for higher throughput, " + + "increase the transaction timeout for the connector's producers, " + + "decrease the offset commit interval (if using interval-based transaction boundaries), " + + "or use the 'poll' transaction boundary (if the connector is not already configured to use it).", + error + ); + } + + @Override + public String toString() { + return "ExactlyOnceWorkerSourceTask{" + + "id=" + id + + '}'; + } + + private abstract class TransactionManager { + protected boolean shouldCommitTransactionForRecord(SourceRecord record) { + return false; + } + + protected boolean shouldCommitTransactionForBatch(long currentTimeMs) { + return false; + } + + protected boolean shouldCommitFinalTransaction() { + return false; + } + + /** + * Hook to signal that a new transaction cycle has been started. May be invoked + * multiple times if the task is paused and then resumed. It can be assumed that + * a new transaction is created at least every time an existing transaction is + * committed; this is just a hook to notify that a new transaction may have been + * created outside of that flow as well. + */ + protected void initialize() { + } + + public void maybeCommitTransactionForRecord(SourceRecord record) { + maybeCommitTransaction(shouldCommitTransactionForRecord(record)); + } + + public void maybeCommitTransactionForBatch() { + maybeCommitTransaction(shouldCommitTransactionForBatch(time.milliseconds())); + } + + public void maybeCommitFinalTransaction() { + maybeCommitTransaction(shouldCommitFinalTransaction()); + } + + private void maybeCommitTransaction(boolean shouldCommit) { + if (shouldCommit && (transactionOpen || offsetWriter.willFlush())) { + try (LoggingContext loggingContext = LoggingContext.forOffsets(id)) { + commitTransaction(); + } + } + } + } + + private TransactionManager buildTransactionManager( + WorkerConfig workerConfig, + SourceConnectorConfig sourceConfig, + WorkerTransactionContext transactionContext) { + TransactionBoundary boundary = sourceConfig.transactionBoundary(); + switch (boundary) { + case POLL: + return new TransactionManager() { + @Override + protected boolean shouldCommitTransactionForBatch(long currentTimeMs) { + return true; + } + + @Override + protected boolean shouldCommitFinalTransaction() { + return true; + } + }; + + case INTERVAL: + long transactionBoundaryInterval = Optional.ofNullable(sourceConfig.transactionBoundaryInterval()) + .orElse(workerConfig.offsetCommitInterval()); + return new TransactionManager() { + private final long commitInterval = transactionBoundaryInterval; + private long lastCommit; + + @Override + public void initialize() { + this.lastCommit = time.milliseconds(); + } + + @Override + protected boolean shouldCommitTransactionForBatch(long currentTimeMs) { + if (time.milliseconds() >= lastCommit + commitInterval) { + lastCommit = time.milliseconds(); + return true; + } else { + return false; + } + } + + @Override + protected boolean shouldCommitFinalTransaction() { + return true; + } + }; + + case CONNECTOR: + Objects.requireNonNull(transactionContext, "Transaction context must be provided when using connector-defined transaction boundaries"); + return new TransactionManager() { + @Override + protected boolean shouldCommitFinalTransaction() { + return shouldCommitTransactionForBatch(time.milliseconds()); + } + + @Override + protected boolean shouldCommitTransactionForBatch(long currentTimeMs) { + if (transactionContext.shouldAbortBatch()) { + log.info("Aborting transaction for batch as requested by connector"); + abortTransaction(); + // We abort the transaction, which causes all the records up to this point to be dropped, but we still want to + // commit offsets so that the task doesn't see the same records all over again + return true; + } + return transactionContext.shouldCommitBatch(); + } + + @Override + protected boolean shouldCommitTransactionForRecord(SourceRecord record) { + if (transactionContext.shouldAbortOn(record)) { + log.info("Aborting transaction for record on topic {} as requested by connector", record.topic()); + log.trace("Last record in aborted transaction: {}", record); + abortTransaction(); + // We abort the transaction, which causes all the records up to this point to be dropped, but we still want to + // commit offsets so that the task doesn't see the same records all over again + return true; + } + return transactionContext.shouldCommitOn(record); + } + + private void abortTransaction() { + producer.abortTransaction(); + transactionMetrics.abortTransaction(); + transactionOpen = false; + } + }; + default: + throw new IllegalArgumentException("Unrecognized transaction boundary: " + boundary); + } + } + + TransactionMetricsGroup transactionMetricsGroup() { + return transactionMetrics; + } + + + static class TransactionMetricsGroup { + private final Sensor transactionSize; + private int size; + private final ConnectMetrics.MetricGroup metricGroup; + + public TransactionMetricsGroup(ConnectorTaskId id, ConnectMetrics connectMetrics) { + ConnectMetricsRegistry registry = connectMetrics.registry(); + metricGroup = connectMetrics.group(registry.sourceTaskGroupName(), + registry.connectorTagName(), id.connector(), + registry.taskTagName(), Integer.toString(id.task())); + + transactionSize = metricGroup.sensor("transaction-size"); + transactionSize.add(metricGroup.metricName(registry.transactionSizeAvg), new Avg()); + transactionSize.add(metricGroup.metricName(registry.transactionSizeMin), new Min()); + transactionSize.add(metricGroup.metricName(registry.transactionSizeMax), new Max()); + } + + void addRecord() { + size++; + } + + void abortTransaction() { + size = 0; + } + + void commitTransaction() { + transactionSize.record(size); + size = 0; + } + + protected ConnectMetrics.MetricGroup metricGroup() { + return metricGroup; + } + + } + +} diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Herder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Herder.java index 945797f8ceaee..21da7c6ea8454 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Herder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Herder.java @@ -137,6 +137,17 @@ public interface Herder { */ void putTaskConfigs(String connName, List> configs, Callback callback, InternalRequestSignature requestSignature); + /** + * Fence out any older task generations for a source connector, and then write a record to the config topic + * indicating that it is safe to bring up a new generation of tasks. If that record is already present, do nothing + * and invoke the callback successfully. + * @param connName the name of the connector to fence out; must refer to a source connector + * @param callback callback to invoke upon completion + * @param requestSignature the signature of the request made for this connector; + * may be null if no signature was provided + */ + void fenceZombies(String connName, Callback callback, InternalRequestSignature requestSignature); + /** * Get a list of connectors currently running in this cluster. * @return A list of connector names diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SourceConnectorConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SourceConnectorConfig.java index 7cf5d67899e61..b58a01b87c57d 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SourceConnectorConfig.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SourceConnectorConfig.java @@ -20,21 +20,31 @@ import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.connect.runtime.isolation.Plugins; +import org.apache.kafka.connect.source.SourceTask; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; +import static org.apache.kafka.connect.runtime.SourceConnectorConfig.ExactlyOnceSupportLevel.REQUESTED; +import static org.apache.kafka.connect.runtime.SourceConnectorConfig.ExactlyOnceSupportLevel.REQUIRED; import static org.apache.kafka.connect.runtime.TopicCreationConfig.DEFAULT_TOPIC_CREATION_GROUP; import static org.apache.kafka.connect.runtime.TopicCreationConfig.DEFAULT_TOPIC_CREATION_PREFIX; import static org.apache.kafka.connect.runtime.TopicCreationConfig.EXCLUDE_REGEX_CONFIG; import static org.apache.kafka.connect.runtime.TopicCreationConfig.INCLUDE_REGEX_CONFIG; import static org.apache.kafka.connect.runtime.TopicCreationConfig.PARTITIONS_CONFIG; import static org.apache.kafka.connect.runtime.TopicCreationConfig.REPLICATION_FACTOR_CONFIG; +import static org.apache.kafka.connect.source.SourceTask.TransactionBoundary; +import static org.apache.kafka.connect.source.SourceTask.TransactionBoundary.CONNECTOR; +import static org.apache.kafka.connect.source.SourceTask.TransactionBoundary.DEFAULT; +import static org.apache.kafka.connect.source.SourceTask.TransactionBoundary.INTERVAL; +import static org.apache.kafka.connect.source.SourceTask.TransactionBoundary.POLL; +import static org.apache.kafka.common.utils.Utils.enumOptions; public class SourceConnectorConfig extends ConnectorConfig { @@ -47,6 +57,57 @@ public class SourceConnectorConfig extends ConnectorConfig { + "created by source connectors"; private static final String TOPIC_CREATION_GROUPS_DISPLAY = "Topic Creation Groups"; + protected static final String EXACTLY_ONCE_SUPPORT_GROUP = "Exactly Once Support"; + + public enum ExactlyOnceSupportLevel { + REQUESTED, + REQUIRED; + + public static ExactlyOnceSupportLevel fromProperty(String property) { + return valueOf(property.toUpperCase(Locale.ROOT).trim()); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + public static final String EXACTLY_ONCE_SUPPORT_CONFIG = "exactly.once.support"; + private static final String EXACTLY_ONCE_SUPPORT_DOC = "Permitted values are " + String.join(", ", enumOptions(ExactlyOnceSupportLevel.class)) + ". " + + "If set to \"" + REQUIRED + "\", forces a preflight check for the connector to ensure that it can provide exactly-once delivery " + + "with the given configuration. Some connectors may be capable of providing exactly-once delivery but not signal to " + + "Connect that they support this; in that case, documentation for the connector should be consulted carefully before " + + "creating it, and the value for this property should be set to \"" + REQUESTED + "\". " + + "Additionally, if the value is set to \"" + REQUIRED + "\" but the worker that performs preflight validation does not have " + + "exactly-once support enabled for source connectors, requests to create or validate the connector will fail."; + private static final String EXACTLY_ONCE_SUPPORT_DISPLAY = "Exactly once support"; + + public static final String TRANSACTION_BOUNDARY_CONFIG = SourceTask.TRANSACTION_BOUNDARY_CONFIG; + private static final String TRANSACTION_BOUNDARY_DOC = "Permitted values are: " + String.join(", ", enumOptions(TransactionBoundary.class)) + ". " + + "If set to '" + POLL + "', a new producer transaction will be started and committed for every batch of records that each task from " + + "this connector provides to Connect. If set to '" + CONNECTOR + "', relies on connector-defined transaction boundaries; note that " + + "not all connectors are capable of defining their own transaction boundaries, and in that case, attempts to create them with " + + "this value will fail. Finally, if set to '" + INTERVAL + "', commits transactions only after a user-defined time interval has passed."; + private static final String TRANSACTION_BOUNDARY_DISPLAY = "Transaction Boundary"; + + public static final String TRANSACTION_BOUNDARY_INTERVAL_CONFIG = "transaction.boundary.interval.ms"; + private static final String TRANSACTION_BOUNDARY_INTERVAL_DOC = "If '" + TRANSACTION_BOUNDARY_CONFIG + "' is set to '" + INTERVAL + + "', determines the interval for producer transaction commits by connector tasks. If unset, defaults to the value of the worker-level " + + "'" + WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG + "' property. Has no effect if a different " + + TRANSACTION_BOUNDARY_CONFIG + " is specified."; + private static final String TRANSACTION_BOUNDARY_INTERVAL_DISPLAY = "Transaction boundary interval"; + + protected static final String OFFSETS_TOPIC_GROUP = "offsets.topic"; + + public static final String OFFSETS_TOPIC_CONFIG = "offsets.storage.topic"; + private static final String OFFSETS_TOPIC_DOC = "The name of a separate offsets topic to use for this connector. " + + "If empty or not specified, the worker’s global offsets topic name will be used. " + + "If specified, the offsets topic will be created if it does not already exist on the Kafka cluster targeted by this connector " + + "(which may be different from the one used for the worker's global offsets topic if the bootstrap.servers property of the connector's producer " + + "has been overridden from the worker's). Only applicable in distributed mode; in standalone mode, setting this property will have no effect."; + private static final String OFFSETS_TOPIC_DISPLAY = "Offsets topic"; + private static class EnrichedSourceConnectorConfig extends ConnectorConfig { EnrichedSourceConnectorConfig(Plugins plugins, ConfigDef configDef, Map props) { super(plugins, configDef, props); @@ -58,23 +119,87 @@ public Object get(String key) { } } - private static ConfigDef config = SourceConnectorConfig.configDef(); + private final TransactionBoundary transactionBoundary; + private final Long transactionBoundaryInterval; private final EnrichedSourceConnectorConfig enrichedSourceConfig; + private final String offsetsTopic; public static ConfigDef configDef() { + ConfigDef.Validator atLeastZero = ConfigDef.Range.atLeast(0); int orderInGroup = 0; return new ConfigDef(ConnectorConfig.configDef()) - .define(TOPIC_CREATION_GROUPS_CONFIG, ConfigDef.Type.LIST, Collections.emptyList(), - ConfigDef.CompositeValidator.of(new ConfigDef.NonNullValidator(), ConfigDef.LambdaValidator.with( + .define( + TOPIC_CREATION_GROUPS_CONFIG, + ConfigDef.Type.LIST, + Collections.emptyList(), + ConfigDef.CompositeValidator.of( + new ConfigDef.NonNullValidator(), + ConfigDef.LambdaValidator.with( + (name, value) -> { + List groupAliases = (List) value; + if (groupAliases.size() > new HashSet<>(groupAliases).size()) { + throw new ConfigException(name, value, "Duplicate alias provided."); + } + }, + () -> "unique topic creation groups")), + ConfigDef.Importance.LOW, + TOPIC_CREATION_GROUPS_DOC, + TOPIC_CREATION_GROUP, + ++orderInGroup, + ConfigDef.Width.LONG, + TOPIC_CREATION_GROUPS_DISPLAY) + .define( + EXACTLY_ONCE_SUPPORT_CONFIG, + ConfigDef.Type.STRING, + REQUESTED.toString(), + ConfigDef.CaseInsensitiveValidString.in(enumOptions(ExactlyOnceSupportLevel.class)), + ConfigDef.Importance.MEDIUM, + EXACTLY_ONCE_SUPPORT_DOC, + EXACTLY_ONCE_SUPPORT_GROUP, + ++orderInGroup, + ConfigDef.Width.SHORT, + EXACTLY_ONCE_SUPPORT_DISPLAY) + .define( + TRANSACTION_BOUNDARY_CONFIG, + ConfigDef.Type.STRING, + DEFAULT.toString(), + ConfigDef.CaseInsensitiveValidString.in(enumOptions(TransactionBoundary.class)), + ConfigDef.Importance.MEDIUM, + TRANSACTION_BOUNDARY_DOC, + EXACTLY_ONCE_SUPPORT_GROUP, + ++orderInGroup, + ConfigDef.Width.SHORT, + TRANSACTION_BOUNDARY_DISPLAY) + .define( + TRANSACTION_BOUNDARY_INTERVAL_CONFIG, + ConfigDef.Type.LONG, + null, + ConfigDef.LambdaValidator.with( (name, value) -> { - List groupAliases = (List) value; - if (groupAliases.size() > new HashSet<>(groupAliases).size()) { - throw new ConfigException(name, value, "Duplicate alias provided."); + if (value == null) { + return; } + atLeastZero.ensureValid(name, value); }, - () -> "unique topic creation groups")), - ConfigDef.Importance.LOW, TOPIC_CREATION_GROUPS_DOC, TOPIC_CREATION_GROUP, - ++orderInGroup, ConfigDef.Width.LONG, TOPIC_CREATION_GROUPS_DISPLAY); + atLeastZero::toString + ), + ConfigDef.Importance.LOW, + TRANSACTION_BOUNDARY_INTERVAL_DOC, + EXACTLY_ONCE_SUPPORT_GROUP, + ++orderInGroup, + ConfigDef.Width.SHORT, + TRANSACTION_BOUNDARY_INTERVAL_DISPLAY) + .define( + OFFSETS_TOPIC_CONFIG, + ConfigDef.Type.STRING, + null, + new ConfigDef.NonEmptyString(), + ConfigDef.Importance.LOW, + OFFSETS_TOPIC_DOC, + OFFSETS_TOPIC_GROUP, + orderInGroup = 1, + ConfigDef.Width.LONG, + OFFSETS_TOPIC_DISPLAY); } public static ConfigDef embedDefaultGroup(ConfigDef baseConfigDef) { @@ -116,9 +241,9 @@ public static ConfigDef enrich(ConfigDef baseConfigDef, Map prop } public SourceConnectorConfig(Plugins plugins, Map props, boolean createTopics) { - super(plugins, config, props); + super(plugins, configDef(), props); if (createTopics && props.entrySet().stream().anyMatch(e -> e.getKey().startsWith(TOPIC_CREATION_PREFIX))) { - ConfigDef defaultConfigDef = embedDefaultGroup(config); + ConfigDef defaultConfigDef = embedDefaultGroup(configDef()); // This config is only used to set default values for partitions and replication // factor from the default group and otherwise it remains unused AbstractConfig defaultGroup = new AbstractConfig(defaultConfigDef, props, false); @@ -135,6 +260,13 @@ public SourceConnectorConfig(Plugins plugins, Map props, boolean } else { enrichedSourceConfig = null; } + transactionBoundary = TransactionBoundary.fromProperty(getString(TRANSACTION_BOUNDARY_CONFIG)); + transactionBoundaryInterval = getLong(TRANSACTION_BOUNDARY_INTERVAL_CONFIG); + offsetsTopic = getString(OFFSETS_TOPIC_CONFIG); + } + + public static boolean usesTopicCreation(Map props) { + return props.entrySet().stream().anyMatch(e -> e.getKey().startsWith(TOPIC_CREATION_PREFIX)); } @Override @@ -142,6 +274,18 @@ public Object get(String key) { return enrichedSourceConfig != null ? enrichedSourceConfig.get(key) : super.get(key); } + public TransactionBoundary transactionBoundary() { + return transactionBoundary; + } + + public Long transactionBoundaryInterval() { + return transactionBoundaryInterval; + } + + public String offsetsTopic() { + return offsetsTopic; + } + /** * Returns whether this configuration uses topic creation properties. * @@ -181,6 +325,6 @@ public Map topicCreationOtherConfigs(String group) { } public static void main(String[] args) { - System.out.println(config.toHtml(4, config -> "sourceconnectorconfigs_" + config)); + System.out.println(configDef().toHtml(4, config -> "sourceconnectorconfigs_" + config)); } } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SubmittedRecords.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SubmittedRecords.java index 6cdd2c1842b7b..941dc13ec1933 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SubmittedRecords.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/SubmittedRecords.java @@ -35,7 +35,7 @@ * source offsets. Records are tracked in the order in which they are submitted, which should match the order they were * returned from {@link SourceTask#poll()}. The latest-eligible offsets for each source partition can be retrieved via * {@link #committableOffsets()}, where every record up to and including the record for each returned offset has been - * either {@link SubmittedRecord#ack() acknowledged} or {@link #removeLastOccurrence(SubmittedRecord) removed}. + * either {@link SubmittedRecord#ack() acknowledged} or {@link SubmittedRecord#drop dropped}. * Note that this class is not thread-safe, though a {@link SubmittedRecord} can be * {@link SubmittedRecord#ack() acknowledged} from a different thread. */ @@ -54,13 +54,13 @@ public SubmittedRecords() { /** * Enqueue a new source record before dispatching it to a producer. * The returned {@link SubmittedRecord} should either be {@link SubmittedRecord#ack() acknowledged} in the - * producer callback, or {@link #removeLastOccurrence(SubmittedRecord) removed} if the record could not be successfully + * producer callback, or {@link SubmittedRecord#drop() dropped} if the record could not be successfully * sent to the producer. - * + * * @param record the record about to be dispatched; may not be null but may have a null * {@link SourceRecord#sourcePartition()} and/or {@link SourceRecord#sourceOffset()} * @return a {@link SubmittedRecord} that can be either {@link SubmittedRecord#ack() acknowledged} once ack'd by - * the producer, or {@link #removeLastOccurrence removed} if synchronously rejected by the producer + * the producer, or {@link SubmittedRecord#drop() dropped} if synchronously rejected by the producer */ @SuppressWarnings("unchecked") public SubmittedRecord submit(SourceRecord record) { @@ -78,32 +78,6 @@ SubmittedRecord submit(Map partition, Map offset return result; } - /** - * Remove a source record and do not take it into account any longer when tracking offsets. - * Useful if the record has been synchronously rejected by the producer. - * If multiple instances of the same {@link SubmittedRecord} have been submitted already, only the first one found - * (traversing from the end of the deque backward) will be removed. - * @param record the {@link #submit previously-submitted} record to stop tracking; may not be null - * @return whether an instance of the record was removed - */ - public boolean removeLastOccurrence(SubmittedRecord record) { - Deque deque = records.get(record.partition()); - if (deque == null) { - log.warn("Attempted to remove record from submitted queue for partition {}, but no records with that partition appear to have been submitted", record.partition()); - return false; - } - boolean result = deque.removeLastOccurrence(record); - if (deque.isEmpty()) { - records.remove(record.partition()); - } - if (result) { - messageAcked(); - } else { - log.warn("Attempted to remove record from submitted queue for partition {}, but the record has not been submitted or has already been removed", record.partition()); - } - return result; - } - /** * Clear out any acknowledged records at the head of the deques and return a {@link CommittableOffsets snapshot} of the offsets and offset metadata * accrued between the last time this method was invoked and now. This snapshot can be {@link CommittableOffsets#updatedWith(CommittableOffsets) combined} @@ -187,7 +161,7 @@ private synchronized void messageAcked() { } } - class SubmittedRecord { + public class SubmittedRecord { private final Map partition; private final Map offset; private final AtomicBoolean acked; @@ -208,6 +182,34 @@ public void ack() { } } + /** + * Remove this record and do not take it into account any longer when tracking offsets. + * Useful if the record has been synchronously rejected by the producer. + * If multiple instances of this record have been submitted already, only the first one found + * (traversing from the end of the deque backward) will be removed. + *

+ * This is not safe to be called from a different thread + * than what called {@link SubmittedRecords#submit(SourceRecord)}. + * @return whether this instance was dropped + */ + public boolean drop() { + Deque deque = records.get(partition); + if (deque == null) { + log.warn("Attempted to remove record from submitted queue for partition {}, but no records with that partition appear to have been submitted", partition); + return false; + } + boolean result = deque.removeLastOccurrence(this); + if (deque.isEmpty()) { + records.remove(partition); + } + if (result) { + messageAcked(); + } else { + log.warn("Attempted to remove record from submitted queue for partition {}, but the record has not been submitted or has already been removed", partition); + } + return result; + } + private boolean acked() { return acked.get(); } @@ -337,4 +339,4 @@ public CommittableOffsets updatedWith(CommittableOffsets newerOffsets) { ); } } -} +} \ No newline at end of file diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java index 11af818cb135b..15af9c423e062 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Worker.java @@ -16,11 +16,16 @@ */ package org.apache.kafka.connect.runtime; +import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.FenceProducersOptions; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.IsolationLevel; +import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.MetricNameTemplate; import org.apache.kafka.common.config.ConfigValue; import org.apache.kafka.common.config.provider.ConfigProvider; @@ -35,7 +40,6 @@ import org.apache.kafka.connect.json.JsonConverter; import org.apache.kafka.connect.json.JsonConverterConfig; import org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; import org.apache.kafka.connect.runtime.errors.DeadLetterQueueReporter; import org.apache.kafka.connect.runtime.errors.ErrorHandlingMetrics; import org.apache.kafka.connect.runtime.errors.ErrorReporter; @@ -44,15 +48,18 @@ import org.apache.kafka.connect.runtime.errors.WorkerErrantRecordReporter; import org.apache.kafka.connect.runtime.isolation.Plugins; import org.apache.kafka.connect.runtime.isolation.Plugins.ClassLoaderUsage; +import org.apache.kafka.connect.runtime.rest.resources.ConnectorsResource; import org.apache.kafka.connect.sink.SinkRecord; import org.apache.kafka.connect.sink.SinkTask; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; import org.apache.kafka.connect.storage.CloseableOffsetStorageReader; +import org.apache.kafka.connect.storage.ClusterConfigState; +import org.apache.kafka.connect.storage.ConnectorOffsetBackingStore; import org.apache.kafka.connect.storage.Converter; import org.apache.kafka.connect.storage.HeaderConverter; +import org.apache.kafka.connect.storage.KafkaOffsetBackingStore; import org.apache.kafka.connect.storage.OffsetBackingStore; -import org.apache.kafka.connect.storage.OffsetStorageReader; import org.apache.kafka.connect.storage.OffsetStorageReaderImpl; import org.apache.kafka.connect.storage.OffsetStorageWriter; import org.apache.kafka.connect.util.Callback; @@ -70,14 +77,23 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; /** *

@@ -107,11 +123,11 @@ public class Worker { private final WorkerConfig config; private final Converter internalKeyConverter; private final Converter internalValueConverter; - private final OffsetBackingStore offsetBackingStore; + private final OffsetBackingStore globalOffsetBackingStore; private final ConcurrentMap connectors = new ConcurrentHashMap<>(); private final ConcurrentMap tasks = new ConcurrentHashMap<>(); - private SourceTaskOffsetCommitter sourceTaskOffsetCommitter; + private Optional sourceTaskOffsetCommitter; private final WorkerConfigTransformer workerConfigTransformer; private final ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy; @@ -120,9 +136,9 @@ public Worker( Time time, Plugins plugins, WorkerConfig config, - OffsetBackingStore offsetBackingStore, + OffsetBackingStore globalOffsetBackingStore, ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy) { - this(workerId, time, plugins, config, offsetBackingStore, Executors.newCachedThreadPool(), connectorClientConfigOverridePolicy); + this(workerId, time, plugins, config, globalOffsetBackingStore, Executors.newCachedThreadPool(), connectorClientConfigOverridePolicy); } @SuppressWarnings("deprecation") @@ -131,7 +147,7 @@ public Worker( Time time, Plugins plugins, WorkerConfig config, - OffsetBackingStore offsetBackingStore, + OffsetBackingStore globalOffsetBackingStore, ExecutorService executorService, ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy ) { @@ -149,8 +165,8 @@ public Worker( this.internalKeyConverter = plugins.newInternalConverter(true, JsonConverter.class.getName(), internalConverterConfig); this.internalValueConverter = plugins.newInternalConverter(false, JsonConverter.class.getName(), internalConverterConfig); - this.offsetBackingStore = offsetBackingStore; - this.offsetBackingStore.configure(config); + this.globalOffsetBackingStore = globalOffsetBackingStore; + this.globalOffsetBackingStore.configure(config); this.workerConfigTransformer = initConfigTransformer(); @@ -184,8 +200,11 @@ protected Herder herder() { public void start() { log.info("Worker starting"); - offsetBackingStore.start(); - sourceTaskOffsetCommitter = new SourceTaskOffsetCommitter(config); + globalOffsetBackingStore.start(); + + sourceTaskOffsetCommitter = config.exactlyOnceSourceEnabled() + ? Optional.empty() + : Optional.of(new SourceTaskOffsetCommitter(config)); connectorStatusMetricsGroup = new ConnectorStatusMetricsGroup(metrics, tasks, herder); @@ -212,9 +231,9 @@ public void stop() { } long timeoutMs = limit - time.milliseconds(); - sourceTaskOffsetCommitter.close(timeoutMs); + sourceTaskOffsetCommitter.ifPresent(committer -> committer.close(timeoutMs)); - offsetBackingStore.stop(); + globalOffsetBackingStore.stop(); metrics.stop(); log.info("Worker stopped"); @@ -264,12 +283,22 @@ public void startConnector( log.info("Creating connector {} of type {}", connName, connClass); final Connector connector = plugins.newConnector(connClass); - final ConnectorConfig connConfig = ConnectUtils.isSinkConnector(connector) - ? new SinkConnectorConfig(plugins, connProps) - : new SourceConnectorConfig(plugins, connProps, config.topicCreationEnable()); - - final OffsetStorageReader offsetReader = new OffsetStorageReaderImpl( - offsetBackingStore, connName, internalKeyConverter, internalValueConverter); + final ConnectorConfig connConfig; + final CloseableOffsetStorageReader offsetReader; + if (ConnectUtils.isSinkConnector(connector)) { + connConfig = new SinkConnectorConfig(plugins, connProps); + offsetReader = null; + } else { + SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, connProps, config.topicCreationEnable()); + connConfig = sourceConfig; + + // Set up the offset backing store for this connector instance + ConnectorOffsetBackingStore offsetStore = config.exactlyOnceSourceEnabled() + ? offsetStoreForExactlyOnceSourceConnector(sourceConfig, connName, connector) + : offsetStoreForRegularSourceConnector(sourceConfig, connName, connector); + offsetStore.configure(config); + offsetReader = new OffsetStorageReaderImpl(offsetStore, connName, internalKeyConverter, internalValueConverter); + } workerConnector = new WorkerConnector( connName, connector, connConfig, ctx, metrics, connectorStatusListener, offsetReader, connectorLoader); log.info("Instantiated connector {} with version {} of type {}", connName, connector.version(), connector.getClass()); @@ -476,22 +505,95 @@ public boolean isRunning(String connName) { } /** - * Start a task managed by this worker. + * Start a sink task managed by this worker. + * + * @param id the task ID. + * @param configState the most recent {@link ClusterConfigState} known to the worker + * @param connProps the connector properties. + * @param taskProps the tasks properties. + * @param statusListener a listener for the runtime status transitions of the task. + * @param initialState the initial state of the connector. + * @return true if the task started successfully. + */ + public boolean startSinkTask( + ConnectorTaskId id, + ClusterConfigState configState, + Map connProps, + Map taskProps, + TaskStatus.Listener statusListener, + TargetState initialState + ) { + return startTask(id, connProps, taskProps, statusListener, + new SinkTaskBuilder(id, configState, statusListener, initialState)); + } + + /** + * Start a source task managed by this worker using older behavior that does not provide exactly-once support. * * @param id the task ID. + * @param configState the most recent {@link ClusterConfigState} known to the worker * @param connProps the connector properties. * @param taskProps the tasks properties. * @param statusListener a listener for the runtime status transitions of the task. * @param initialState the initial state of the connector. * @return true if the task started successfully. */ - public boolean startTask( + public boolean startSourceTask( ConnectorTaskId id, ClusterConfigState configState, Map connProps, Map taskProps, TaskStatus.Listener statusListener, TargetState initialState + ) { + return startTask(id, connProps, taskProps, statusListener, + new SourceTaskBuilder(id, configState, statusListener, initialState)); + } + + /** + * Start a source task with exactly-once support managed by this worker. + * + * @param id the task ID. + * @param configState the most recent {@link ClusterConfigState} known to the worker + * @param connProps the connector properties. + * @param taskProps the tasks properties. + * @param statusListener a listener for the runtime status transitions of the task. + * @param initialState the initial state of the connector. + * @param preProducerCheck a preflight check that should be performed before the task initializes its transactional producer. + * @param postProducerCheck a preflight check that should be performed after the task initializes its transactional producer, + * but before producing any source records or offsets. + * @return true if the task started successfully. + */ + public boolean startExactlyOnceSourceTask( + ConnectorTaskId id, + ClusterConfigState configState, + Map connProps, + Map taskProps, + TaskStatus.Listener statusListener, + TargetState initialState, + Runnable preProducerCheck, + Runnable postProducerCheck + ) { + return startTask(id, connProps, taskProps, statusListener, + new ExactlyOnceSourceTaskBuilder(id, configState, statusListener, initialState, preProducerCheck, postProducerCheck)); + } + + /** + * Start a task managed by this worker. + * + * @param id the task ID. + * @param connProps the connector properties. + * @param taskProps the tasks properties. + * @param statusListener a listener for the runtime status transitions of the task. + * @param taskBuilder the {@link TaskBuilder} used to create the {@link WorkerTask} that manages the lifecycle of the task. + * @return true if the task started successfully. + */ + private boolean startTask( + ConnectorTaskId id, + Map connProps, + Map taskProps, + TaskStatus.Listener statusListener, + TaskBuilder taskBuilder ) { final WorkerTask workerTask; final TaskStatus.Listener taskStatusListener = workerMetricsGroup.wrapStatusListener(statusListener); @@ -542,8 +644,15 @@ public boolean startTask( log.info("Set up the header converter {} for task {} using the connector config", headerConverter.getClass(), id); } - workerTask = buildWorkerTask(configState, connConfig, id, task, taskStatusListener, - initialState, keyConverter, valueConverter, headerConverter, connectorLoader); + workerTask = taskBuilder + .withTask(task) + .withConnectorConfig(connConfig) + .withKeyConverter(keyConverter) + .withValueConverter(valueConverter) + .withHeaderConverter(headerConverter) + .withClassloader(connectorLoader) + .build(); + workerTask.initialize(taskConfig); Plugins.compareAndSwapLoaders(savedLoader); } catch (Throwable t) { @@ -562,80 +671,87 @@ public boolean startTask( executor.submit(workerTask); if (workerTask instanceof WorkerSourceTask) { - sourceTaskOffsetCommitter.schedule(id, (WorkerSourceTask) workerTask); + sourceTaskOffsetCommitter.ifPresent(committer -> committer.schedule(id, (WorkerSourceTask) workerTask)); } return true; } } - private WorkerTask buildWorkerTask(ClusterConfigState configState, - ConnectorConfig connConfig, - ConnectorTaskId id, - Task task, - TaskStatus.Listener statusListener, - TargetState initialState, - Converter keyConverter, - Converter valueConverter, - HeaderConverter headerConverter, - ClassLoader loader) { - ErrorHandlingMetrics errorHandlingMetrics = errorHandlingMetrics(id); - final Class connectorClass = plugins.connectorClass( - connConfig.getString(ConnectorConfig.CONNECTOR_CLASS_CONFIG)); - RetryWithToleranceOperator retryWithToleranceOperator = new RetryWithToleranceOperator(connConfig.errorRetryTimeout(), - connConfig.errorMaxDelayInMillis(), connConfig.errorToleranceType(), Time.SYSTEM); - retryWithToleranceOperator.metrics(errorHandlingMetrics); - - // Decide which type of worker task we need based on the type of task. - if (task instanceof SourceTask) { - SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, - connConfig.originalsStrings(), config.topicCreationEnable()); - retryWithToleranceOperator.reporters(sourceTaskReporters(id, sourceConfig, errorHandlingMetrics)); - TransformationChain transformationChain = new TransformationChain<>(sourceConfig.transformations(), retryWithToleranceOperator); - log.info("Initializing: {}", transformationChain); - CloseableOffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetBackingStore, id.connector(), - internalKeyConverter, internalValueConverter); - OffsetStorageWriter offsetWriter = new OffsetStorageWriter(offsetBackingStore, id.connector(), - internalKeyConverter, internalValueConverter); - Map producerProps = producerConfigs(id, "connector-producer-" + id, config, sourceConfig, connectorClass, - connectorClientConfigOverridePolicy, kafkaClusterId); - KafkaProducer producer = new KafkaProducer<>(producerProps); - TopicAdmin admin; - Map topicCreationGroups; - if (config.topicCreationEnable() && sourceConfig.usesTopicCreation()) { - Map adminProps = adminConfigs(id, "connector-adminclient-" + id, config, - sourceConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId); - admin = new TopicAdmin(adminProps); - topicCreationGroups = TopicCreationGroup.configuredGroups(sourceConfig); - } else { - admin = null; - topicCreationGroups = null; - } - - // Note we pass the configState as it performs dynamic transformations under the covers - return new WorkerSourceTask(id, (SourceTask) task, statusListener, initialState, keyConverter, valueConverter, - headerConverter, transformationChain, producer, admin, topicCreationGroups, - offsetReader, offsetWriter, config, configState, metrics, loader, time, retryWithToleranceOperator, herder.statusBackingStore(), executor); - } else if (task instanceof SinkTask) { - TransformationChain transformationChain = new TransformationChain<>(connConfig.transformations(), retryWithToleranceOperator); - log.info("Initializing: {}", transformationChain); - SinkConnectorConfig sinkConfig = new SinkConnectorConfig(plugins, connConfig.originalsStrings()); - retryWithToleranceOperator.reporters(sinkTaskReporters(id, sinkConfig, errorHandlingMetrics, connectorClass)); - WorkerErrantRecordReporter workerErrantRecordReporter = createWorkerErrantRecordReporter(sinkConfig, retryWithToleranceOperator, - keyConverter, valueConverter, headerConverter); - - Map consumerProps = consumerConfigs(id, config, connConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId); - KafkaConsumer consumer = new KafkaConsumer<>(consumerProps); + /** + * Using the admin principal for this connector, perform a round of zombie fencing that disables transactional producers + * for the specified number of source tasks from sending any more records. + * @param connName the name of the connector + * @param numTasks the number of tasks to fence out + * @param connProps the configuration of the connector; may not be null + * @return a {@link KafkaFuture} that will complete when the producers have all been fenced out, or the attempt has failed + */ + public KafkaFuture fenceZombies(String connName, int numTasks, Map connProps) { + return fenceZombies(connName, numTasks, connProps, Admin::create); + } - return new WorkerSinkTask(id, (SinkTask) task, statusListener, initialState, config, configState, metrics, keyConverter, - valueConverter, headerConverter, transformationChain, consumer, loader, time, - retryWithToleranceOperator, workerErrantRecordReporter, herder.statusBackingStore()); - } else { - log.error("Tasks must be a subclass of either SourceTask or SinkTask and current is {}", task); - throw new ConnectException("Tasks must be a subclass of either SourceTask or SinkTask"); + // Allows us to mock out the Admin client for testing + KafkaFuture fenceZombies(String connName, int numTasks, Map connProps, Function, Admin> adminFactory) { + log.debug("Fencing out {} task producers for source connector {}", numTasks, connName); + try (LoggingContext loggingContext = LoggingContext.forConnector(connName)) { + ClassLoader savedLoader = plugins.currentThreadLoader(); + try { + String connType = connProps.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG); + ClassLoader connectorLoader = plugins.delegatingLoader().connectorLoader(connType); + savedLoader = Plugins.compareAndSwapLoaders(connectorLoader); + final SourceConnectorConfig connConfig = new SourceConnectorConfig(plugins, connProps, config.topicCreationEnable()); + final Class connClass = plugins.connectorClass( + connConfig.getString(ConnectorConfig.CONNECTOR_CLASS_CONFIG)); + + Map adminConfig = adminConfigs( + connName, + "connector-worker-adminclient-" + connName, + config, + connConfig, + connClass, + connectorClientConfigOverridePolicy, + kafkaClusterId, + ConnectorType.SOURCE); + Admin admin = adminFactory.apply(adminConfig); + + Collection transactionalIds = IntStream.range(0, numTasks) + .mapToObj(i -> new ConnectorTaskId(connName, i)) + .map(this::transactionalId) + .collect(Collectors.toList()); + FenceProducersOptions fencingOptions = new FenceProducersOptions() + .timeoutMs((int) ConnectorsResource.REQUEST_TIMEOUT_MS); + return admin.fenceProducers(transactionalIds, fencingOptions).all().whenComplete((ignored, error) -> { + if (error != null) + log.debug("Finished fencing out {} task producers for source connector {}", numTasks, connName); + Utils.closeQuietly(admin, "Zombie fencing admin for connector " + connName); + }); + } finally { + Plugins.compareAndSwapLoaders(savedLoader); + } } } - static Map producerConfigs(ConnectorTaskId id, + static Map exactlyOnceSourceTaskProducerConfigs(ConnectorTaskId id, + WorkerConfig config, + ConnectorConfig connConfig, + Class connectorClass, + ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy, + String clusterId) { + Map result = baseProducerConfigs(id.connector(), "connector-producer-" + id, config, connConfig, connectorClass, connectorClientConfigOverridePolicy, clusterId); + ConnectUtils.ensureProperty( + result, ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true", + "for connectors when exactly-once source support is enabled", + false + ); + String transactionalId = transactionalId(config.groupId(), id.connector(), id.task()); + ConnectUtils.ensureProperty( + result, ProducerConfig.TRANSACTIONAL_ID_CONFIG, transactionalId, + "for connectors when exactly-once source support is enabled", + true + ); + return result; + } + + static Map baseProducerConfigs(String connName, String defaultClientId, WorkerConfig config, ConnectorConfig connConfig, @@ -643,7 +759,7 @@ static Map producerConfigs(ConnectorTaskId id, ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy, String clusterId) { Map producerProps = new HashMap<>(); - producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Utils.join(config.getList(WorkerConfig.BOOTSTRAP_SERVERS_CONFIG), ",")); + producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.bootstrapServers()); producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); // These settings will execute infinite retries on retriable exceptions. They *may* be overridden via configs passed to the worker, @@ -660,7 +776,7 @@ static Map producerConfigs(ConnectorTaskId id, // Connector-specified overrides Map producerOverrides = - connectorClientConfigOverrides(id, connConfig, connectorClass, ConnectorConfig.CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX, + connectorClientConfigOverrides(connName, connConfig, connectorClass, ConnectorConfig.CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX, ConnectorType.SOURCE, ConnectorClientConfigRequest.ClientType.PRODUCER, connectorClientConfigOverridePolicy); producerProps.putAll(producerOverrides); @@ -668,20 +784,56 @@ static Map producerConfigs(ConnectorTaskId id, return producerProps; } - static Map consumerConfigs(ConnectorTaskId id, + static Map exactlyOnceSourceOffsetsConsumerConfigs(String connName, + String defaultClientId, + WorkerConfig config, + ConnectorConfig connConfig, + Class connectorClass, + ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy, + String clusterId) { + Map result = baseConsumerConfigs( + connName, defaultClientId, config, connConfig, connectorClass, + connectorClientConfigOverridePolicy, clusterId, ConnectorType.SOURCE); + ConnectUtils.ensureProperty( + result, ConsumerConfig.ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT), + "for source connectors' offset consumers when exactly-once source support is enabled", + false + ); + return result; + } + + static Map regularSourceOffsetsConsumerConfigs(String connName, + String defaultClientId, + WorkerConfig config, + ConnectorConfig connConfig, + Class connectorClass, + ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy, + String clusterId) { + Map result = baseConsumerConfigs( + connName, defaultClientId, config, connConfig, connectorClass, + connectorClientConfigOverridePolicy, clusterId, ConnectorType.SOURCE); + // Users can disable this if they want to; it won't affect delivery guarantees since the task isn't exactly-once anyways + result.putIfAbsent( + ConsumerConfig.ISOLATION_LEVEL_CONFIG, + IsolationLevel.READ_COMMITTED.toString().toLowerCase(Locale.ROOT)); + return result; + } + + static Map baseConsumerConfigs(String connName, + String defaultClientId, WorkerConfig config, ConnectorConfig connConfig, Class connectorClass, ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy, - String clusterId) { + String clusterId, + ConnectorType connectorType) { // Include any unknown worker configs so consumer configs can be set globally on the worker // and through to the task Map consumerProps = new HashMap<>(); - consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, SinkUtils.consumerGroupId(id.connector())); - consumerProps.put(ConsumerConfig.CLIENT_ID_CONFIG, "connector-consumer-" + id); - consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, - Utils.join(config.getList(WorkerConfig.BOOTSTRAP_SERVERS_CONFIG), ",")); + consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, SinkUtils.consumerGroupId(connName)); + consumerProps.put(ConsumerConfig.CLIENT_ID_CONFIG, defaultClientId); + consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, config.bootstrapServers()); consumerProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); @@ -692,21 +844,22 @@ static Map consumerConfigs(ConnectorTaskId id, ConnectUtils.addMetricsContextProperties(consumerProps, config, clusterId); // Connector-specified overrides Map consumerOverrides = - connectorClientConfigOverrides(id, connConfig, connectorClass, ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX, - ConnectorType.SINK, ConnectorClientConfigRequest.ClientType.CONSUMER, + connectorClientConfigOverrides(connName, connConfig, connectorClass, ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX, + connectorType, ConnectorClientConfigRequest.ClientType.CONSUMER, connectorClientConfigOverridePolicy); consumerProps.putAll(consumerOverrides); return consumerProps; } - static Map adminConfigs(ConnectorTaskId id, + static Map adminConfigs(String connName, String defaultClientId, WorkerConfig config, ConnectorConfig connConfig, Class connectorClass, ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy, - String clusterId) { + String clusterId, + ConnectorType connectorType) { Map adminProps = new HashMap<>(); // Use the top-level worker configs to retain backwards compatibility with older releases which // did not require a prefix for connector admin client configs in the worker configuration file @@ -718,8 +871,7 @@ static Map adminConfigs(ConnectorTaskId id, && !e.getKey().startsWith("producer.") && !e.getKey().startsWith("consumer.")) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, - Utils.join(config.getList(WorkerConfig.BOOTSTRAP_SERVERS_CONFIG), ",")); + adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, config.bootstrapServers()); adminProps.put(AdminClientConfig.CLIENT_ID_CONFIG, defaultClientId); adminProps.putAll(nonPrefixedWorkerConfigs); @@ -728,8 +880,8 @@ static Map adminConfigs(ConnectorTaskId id, // Connector-specified overrides Map adminOverrides = - connectorClientConfigOverrides(id, connConfig, connectorClass, ConnectorConfig.CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX, - ConnectorType.SINK, ConnectorClientConfigRequest.ClientType.ADMIN, + connectorClientConfigOverrides(connName, connConfig, connectorClass, ConnectorConfig.CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX, + connectorType, ConnectorClientConfigRequest.ClientType.ADMIN, connectorClientConfigOverridePolicy); adminProps.putAll(adminOverrides); @@ -739,7 +891,7 @@ static Map adminConfigs(ConnectorTaskId id, return adminProps; } - private static Map connectorClientConfigOverrides(ConnectorTaskId id, + private static Map connectorClientConfigOverrides(String connName, ConnectorConfig connConfig, Class connectorClass, String clientConfigPrefix, @@ -748,7 +900,7 @@ private static Map connectorClientConfigOverrides(ConnectorTaskI ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy) { Map clientOverrides = connConfig.originalsWithPrefix(clientConfigPrefix); ConnectorClientConfigRequest connectorClientConfigRequest = new ConnectorClientConfigRequest( - id.connector(), + connName, connectorType, connectorClass, clientOverrides, @@ -764,6 +916,14 @@ private static Map connectorClientConfigOverrides(ConnectorTaskI return clientOverrides; } + private String transactionalId(ConnectorTaskId id) { + return transactionalId(config.groupId(), id.connector(), id.task()); + } + + public static String transactionalId(String groupId, String connector, int taskId) { + return String.format("%s-%s-%d", groupId, connector, taskId); + } + ErrorHandlingMetrics errorHandlingMetrics(ConnectorTaskId id) { return new ErrorHandlingMetrics(id, metrics); } @@ -778,9 +938,9 @@ private List sinkTaskReporters(ConnectorTaskId id, SinkConnectorC // check if topic for dead letter queue exists String topic = connConfig.dlqTopicName(); if (topic != null && !topic.isEmpty()) { - Map producerProps = producerConfigs(id, "connector-dlq-producer-" + id, config, connConfig, connectorClass, + Map producerProps = baseProducerConfigs(id.connector(), "connector-dlq-producer-" + id, config, connConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId); - Map adminProps = adminConfigs(id, "connector-dlq-adminclient-", config, connConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId); + Map adminProps = adminConfigs(id.connector(), "connector-dlq-adminclient-", config, connConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId, ConnectorType.SINK); DeadLetterQueueReporter reporter = DeadLetterQueueReporter.createAndSetup(adminProps, id, connConfig, producerProps, errorHandlingMetrics); reporters.add(reporter); @@ -822,7 +982,7 @@ private void stopTask(ConnectorTaskId taskId) { log.info("Stopping task {}", task.id()); if (task instanceof WorkerSourceTask) - sourceTaskOffsetCommitter.remove(task.id()); + sourceTaskOffsetCommitter.ifPresent(committer -> committer.remove(task.id())); ClassLoader savedLoader = plugins.currentThreadLoader(); try { @@ -980,6 +1140,526 @@ WorkerMetricsGroup workerMetricsGroup() { return workerMetricsGroup; } + abstract class TaskBuilder { + + private final ConnectorTaskId id; + private final ClusterConfigState configState; + private final TaskStatus.Listener statusListener; + private final TargetState initialState; + + private Task task = null; + private ConnectorConfig connectorConfig = null; + private Converter keyConverter = null; + private Converter valueConverter = null; + private HeaderConverter headerConverter = null; + private ClassLoader classLoader = null; + + public TaskBuilder(ConnectorTaskId id, + ClusterConfigState configState, + TaskStatus.Listener statusListener, + TargetState initialState) { + this.id = id; + this.configState = configState; + this.statusListener = statusListener; + this.initialState = initialState; + } + + public TaskBuilder withTask(Task task) { + this.task = task; + return this; + } + + public TaskBuilder withConnectorConfig(ConnectorConfig connectorConfig) { + this.connectorConfig = connectorConfig; + return this; + } + + public TaskBuilder withKeyConverter(Converter keyConverter) { + this.keyConverter = keyConverter; + return this; + } + + public TaskBuilder withValueConverter(Converter valueConverter) { + this.valueConverter = valueConverter; + return this; + } + + public TaskBuilder withHeaderConverter(HeaderConverter headerConverter) { + this.headerConverter = headerConverter; + return this; + } + + public TaskBuilder withClassloader(ClassLoader classLoader) { + this.classLoader = classLoader; + return this; + } + + public WorkerTask build() { + Objects.requireNonNull(task, "Task cannot be null"); + Objects.requireNonNull(connectorConfig, "Connector config used by task cannot be null"); + Objects.requireNonNull(keyConverter, "Key converter used by task cannot be null"); + Objects.requireNonNull(valueConverter, "Value converter used by task cannot be null"); + Objects.requireNonNull(headerConverter, "Header converter used by task cannot be null"); + Objects.requireNonNull(classLoader, "Classloader used by task cannot be null"); + + ErrorHandlingMetrics errorHandlingMetrics = errorHandlingMetrics(id); + final Class connectorClass = plugins.connectorClass( + connectorConfig.getString(ConnectorConfig.CONNECTOR_CLASS_CONFIG)); + RetryWithToleranceOperator retryWithToleranceOperator = new RetryWithToleranceOperator(connectorConfig.errorRetryTimeout(), + connectorConfig.errorMaxDelayInMillis(), connectorConfig.errorToleranceType(), Time.SYSTEM); + retryWithToleranceOperator.metrics(errorHandlingMetrics); + + return doBuild(task, id, configState, statusListener, initialState, + connectorConfig, keyConverter, valueConverter, headerConverter, classLoader, + errorHandlingMetrics, connectorClass, retryWithToleranceOperator); + } + + abstract WorkerTask doBuild(Task task, + ConnectorTaskId id, + ClusterConfigState configState, + TaskStatus.Listener statusListener, + TargetState initialState, + ConnectorConfig connectorConfig, + Converter keyConverter, + Converter valueConverter, + HeaderConverter headerConverter, + ClassLoader classLoader, + ErrorHandlingMetrics errorHandlingMetrics, + Class connectorClass, + RetryWithToleranceOperator retryWithToleranceOperator); + + } + + class SinkTaskBuilder extends TaskBuilder { + public SinkTaskBuilder(ConnectorTaskId id, + ClusterConfigState configState, + TaskStatus.Listener statusListener, + TargetState initialState) { + super(id, configState, statusListener, initialState); + } + + @Override + public WorkerTask doBuild(Task task, + ConnectorTaskId id, + ClusterConfigState configState, + TaskStatus.Listener statusListener, + TargetState initialState, + ConnectorConfig connectorConfig, + Converter keyConverter, + Converter valueConverter, + HeaderConverter headerConverter, + ClassLoader classLoader, + ErrorHandlingMetrics errorHandlingMetrics, + Class connectorClass, + RetryWithToleranceOperator retryWithToleranceOperator) { + + TransformationChain transformationChain = new TransformationChain<>(connectorConfig.transformations(), retryWithToleranceOperator); + log.info("Initializing: {}", transformationChain); + SinkConnectorConfig sinkConfig = new SinkConnectorConfig(plugins, connectorConfig.originalsStrings()); + retryWithToleranceOperator.reporters(sinkTaskReporters(id, sinkConfig, errorHandlingMetrics, connectorClass)); + WorkerErrantRecordReporter workerErrantRecordReporter = createWorkerErrantRecordReporter(sinkConfig, retryWithToleranceOperator, + keyConverter, valueConverter, headerConverter); + + Map consumerProps = baseConsumerConfigs( + id.connector(), "connector-consumer-" + id, config, connectorConfig, connectorClass, + connectorClientConfigOverridePolicy, kafkaClusterId, ConnectorType.SINK); + KafkaConsumer consumer = new KafkaConsumer<>(consumerProps); + + return new WorkerSinkTask(id, (SinkTask) task, statusListener, initialState, config, configState, metrics, keyConverter, + valueConverter, headerConverter, transformationChain, consumer, classLoader, time, + retryWithToleranceOperator, workerErrantRecordReporter, herder.statusBackingStore()); + } + } + + class SourceTaskBuilder extends TaskBuilder { + public SourceTaskBuilder(ConnectorTaskId id, + ClusterConfigState configState, + TaskStatus.Listener statusListener, + TargetState initialState) { + super(id, configState, statusListener, initialState); + } + + @Override + public WorkerTask doBuild(Task task, + ConnectorTaskId id, + ClusterConfigState configState, + TaskStatus.Listener statusListener, + TargetState initialState, + ConnectorConfig connectorConfig, + Converter keyConverter, + Converter valueConverter, + HeaderConverter headerConverter, + ClassLoader classLoader, + ErrorHandlingMetrics errorHandlingMetrics, + Class connectorClass, + RetryWithToleranceOperator retryWithToleranceOperator) { + + SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, + connectorConfig.originalsStrings(), config.topicCreationEnable()); + retryWithToleranceOperator.reporters(sourceTaskReporters(id, sourceConfig, errorHandlingMetrics)); + TransformationChain transformationChain = new TransformationChain<>(sourceConfig.transformations(), retryWithToleranceOperator); + log.info("Initializing: {}", transformationChain); + + Map producerProps = baseProducerConfigs(id.connector(), "connector-producer-" + id, config, sourceConfig, connectorClass, + connectorClientConfigOverridePolicy, kafkaClusterId); + KafkaProducer producer = new KafkaProducer<>(producerProps); + + // Prepare to create a topic admin if the task requires one, but do not actually create an instance + // until/unless one is needed + final AtomicReference topicAdmin = new AtomicReference<>(); + final Supplier topicAdminCreator = () -> topicAdmin.updateAndGet(existingAdmin -> { + if (existingAdmin != null) { + return existingAdmin; + } + Map adminOverrides = adminConfigs(id.connector(), "connector-adminclient-" + id, config, + sourceConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId, ConnectorType.SOURCE); + Admin adminClient = Admin.create(adminOverrides); + return new TopicAdmin(adminOverrides.get(BOOTSTRAP_SERVERS_CONFIG), adminClient); + }); + + Map topicCreationGroups; + if (config.topicCreationEnable() && sourceConfig.usesTopicCreation()) { + topicCreationGroups = TopicCreationGroup.configuredGroups(sourceConfig); + // Create a topic admin that the task can use for topic creation + topicAdminCreator.get(); + } else { + topicCreationGroups = null; + } + + // Set up the offset backing store for this task instance + ConnectorOffsetBackingStore offsetStore = offsetStoreForRegularSourceTask( + id, sourceConfig, connectorClass, producer, producerProps, topicAdminCreator); + offsetStore.configure(config); + + CloseableOffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetStore, id.connector(), internalKeyConverter, internalValueConverter); + OffsetStorageWriter offsetWriter = new OffsetStorageWriter(offsetStore, id.connector(), internalKeyConverter, internalValueConverter); + + // Note we pass the configState as it performs dynamic transformations under the covers + return new WorkerSourceTask(id, (SourceTask) task, statusListener, initialState, keyConverter, valueConverter, + headerConverter, transformationChain, producer, topicAdmin.get(), topicCreationGroups, + offsetReader, offsetWriter, offsetStore, config, configState, metrics, classLoader, time, + retryWithToleranceOperator, herder.statusBackingStore(), executor); + } + } + + class ExactlyOnceSourceTaskBuilder extends TaskBuilder { + private final Runnable preProducerCheck; + private final Runnable postProducerCheck; + + public ExactlyOnceSourceTaskBuilder(ConnectorTaskId id, + ClusterConfigState configState, + TaskStatus.Listener statusListener, + TargetState initialState, + Runnable preProducerCheck, + Runnable postProducerCheck) { + super(id, configState, statusListener, initialState); + this.preProducerCheck = preProducerCheck; + this.postProducerCheck = postProducerCheck; + } + + @Override + public WorkerTask doBuild(Task task, + ConnectorTaskId id, + ClusterConfigState configState, + TaskStatus.Listener statusListener, + TargetState initialState, + ConnectorConfig connectorConfig, + Converter keyConverter, + Converter valueConverter, + HeaderConverter headerConverter, + ClassLoader classLoader, + ErrorHandlingMetrics errorHandlingMetrics, + Class connectorClass, + RetryWithToleranceOperator retryWithToleranceOperator) { + + SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, + connectorConfig.originalsStrings(), config.topicCreationEnable()); + retryWithToleranceOperator.reporters(sourceTaskReporters(id, sourceConfig, errorHandlingMetrics)); + TransformationChain transformationChain = new TransformationChain<>(sourceConfig.transformations(), retryWithToleranceOperator); + log.info("Initializing: {}", transformationChain); + + // Create a topic admin that the task will use for its offsets topic and, potentially, automatic topic creation + Map adminOverrides = adminConfigs(id.connector(), "connector-adminclient-" + id, config, + sourceConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId, ConnectorType.SOURCE); + Admin adminClient = Admin.create(adminOverrides); + TopicAdmin topicAdmin = new TopicAdmin(adminOverrides.get(BOOTSTRAP_SERVERS_CONFIG), adminClient); + + Map topicCreationGroups; + if (config.topicCreationEnable() && sourceConfig.usesTopicCreation()) { + topicCreationGroups = TopicCreationGroup.configuredGroups(sourceConfig); + } else { + topicCreationGroups = null; + } + + Map producerProps = exactlyOnceSourceTaskProducerConfigs( + id, config, sourceConfig, connectorClass, + connectorClientConfigOverridePolicy, kafkaClusterId); + KafkaProducer producer = new KafkaProducer<>(producerProps); + + // Set up the offset backing store for this task instance + ConnectorOffsetBackingStore offsetStore = offsetStoreForExactlyOnceSourceTask( + id, sourceConfig, connectorClass, producer, producerProps, topicAdmin); + offsetStore.configure(config); + + CloseableOffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetStore, id.connector(), internalKeyConverter, internalValueConverter); + OffsetStorageWriter offsetWriter = new OffsetStorageWriter(offsetStore, id.connector(), internalKeyConverter, internalValueConverter); + + // Note we pass the configState as it performs dynamic transformations under the covers + return new ExactlyOnceWorkerSourceTask(id, (SourceTask) task, statusListener, initialState, keyConverter, valueConverter, + headerConverter, transformationChain, producer, topicAdmin, topicCreationGroups, + offsetReader, offsetWriter, offsetStore, config, configState, metrics, classLoader, time, retryWithToleranceOperator, + herder.statusBackingStore(), sourceConfig, executor, preProducerCheck, postProducerCheck); + } + } + + // Visible for testing + ConnectorOffsetBackingStore offsetStoreForRegularSourceConnector( + SourceConnectorConfig sourceConfig, + String connName, + Connector connector + ) { + String connectorSpecificOffsetsTopic = sourceConfig.offsetsTopic(); + + Map producerProps = baseProducerConfigs(connName, "connector-producer-" + connName, config, sourceConfig, connector.getClass(), + connectorClientConfigOverridePolicy, kafkaClusterId); + + // We use a connector-specific store (i.e., a dedicated KafkaOffsetBackingStore for this connector) + // if the worker supports per-connector offsets topics (which may be the case in distributed but not standalone mode, for example) + // and if the connector is explicitly configured with an offsets topic + final boolean usesConnectorSpecificStore = connectorSpecificOffsetsTopic != null + && config.connectorOffsetsTopicsPermitted(); + + if (usesConnectorSpecificStore) { + Map consumerProps; + if (config.exactlyOnceSourceEnabled()) { + consumerProps = exactlyOnceSourceOffsetsConsumerConfigs( + connName, "connector-consumer-" + connName, config, sourceConfig, connector.getClass(), + connectorClientConfigOverridePolicy, kafkaClusterId); + } else { + consumerProps = regularSourceOffsetsConsumerConfigs( + connName, "connector-consumer-" + connName, config, sourceConfig, connector.getClass(), + connectorClientConfigOverridePolicy, kafkaClusterId); + } + KafkaConsumer consumer = new KafkaConsumer<>(consumerProps); + + Map adminOverrides = adminConfigs(connName, "connector-adminclient-" + connName, config, + sourceConfig, connector.getClass(), connectorClientConfigOverridePolicy, kafkaClusterId, ConnectorType.SOURCE); + + TopicAdmin admin = new TopicAdmin(adminOverrides); + KafkaOffsetBackingStore connectorStore = + KafkaOffsetBackingStore.forConnector(connectorSpecificOffsetsTopic, consumer, admin); + + // If the connector's offsets topic is the same as the worker-global offsets topic, there's no need to construct + // an offset store that has a primary and a secondary store which both read from that same topic. + // So, if the user has explicitly configured the connector with a connector-specific offsets topic + // but we know that that topic is the same as the worker-global offsets topic, we ignore the worker-global + // offset store and build a store backed exclusively by a connector-specific offsets store. + // It may seem reasonable to instead build a store backed exclusively by the worker-global offset store, but that + // would prevent users from being able to customize the config properties used for the Kafka clients that + // access the offsets topic, and we would not be able to establish reasonable defaults like setting + // isolation.level=read_committed for the offsets topic consumer for this connector + if (sameOffsetTopicAsWorker(connectorSpecificOffsetsTopic, producerProps)) { + return ConnectorOffsetBackingStore.withOnlyConnectorStore( + () -> LoggingContext.forConnector(connName), + connectorStore, + connectorSpecificOffsetsTopic, + admin + ); + } else { + return ConnectorOffsetBackingStore.withConnectorAndWorkerStores( + () -> LoggingContext.forConnector(connName), + globalOffsetBackingStore, + connectorStore, + connectorSpecificOffsetsTopic, + admin + ); + } + } else { + return ConnectorOffsetBackingStore.withOnlyWorkerStore( + () -> LoggingContext.forConnector(connName), + globalOffsetBackingStore, + config.offsetsTopic() + ); + } + } + + // Visible for testing + ConnectorOffsetBackingStore offsetStoreForExactlyOnceSourceConnector( + SourceConnectorConfig sourceConfig, + String connName, + Connector connector + ) { + String connectorSpecificOffsetsTopic = Optional.ofNullable(sourceConfig.offsetsTopic()).orElse(config.offsetsTopic()); + + Map consumerProps = exactlyOnceSourceOffsetsConsumerConfigs( + connName, "connector-consumer-" + connName, config, sourceConfig, connector.getClass(), + connectorClientConfigOverridePolicy, kafkaClusterId); + KafkaConsumer consumer = new KafkaConsumer<>(consumerProps); + + Map adminOverrides = adminConfigs(connName, "connector-adminclient-" + connName, config, + sourceConfig, connector.getClass(), connectorClientConfigOverridePolicy, kafkaClusterId, ConnectorType.SOURCE); + + TopicAdmin admin = new TopicAdmin(adminOverrides); + KafkaOffsetBackingStore connectorStore = + KafkaOffsetBackingStore.forConnector(connectorSpecificOffsetsTopic, consumer, admin); + + // If the connector's offsets topic is the same as the worker-global offsets topic, there's no need to construct + // an offset store that has a primary and a secondary store which both read from that same topic. + // So, even if the user has explicitly configured the connector with a connector-specific offsets topic, + // if we know that that topic is the same as the worker-global offsets topic, we ignore the worker-global + // offset store and build a store backed exclusively by a connector-specific offsets store. + // It may seem reasonable to instead build a store backed exclusively by the worker-global offset store, but that + // would prevent users from being able to customize the config properties used for the Kafka clients that + // access the offsets topic, and may lead to confusion for them when tasks are created for the connector + // since they will all have their own dedicated offsets stores anyways + Map producerProps = baseProducerConfigs(connName, "connector-producer-" + connName, config, sourceConfig, connector.getClass(), + connectorClientConfigOverridePolicy, kafkaClusterId); + if (sameOffsetTopicAsWorker(connectorSpecificOffsetsTopic, producerProps)) { + return ConnectorOffsetBackingStore.withOnlyConnectorStore( + () -> LoggingContext.forConnector(connName), + connectorStore, + connectorSpecificOffsetsTopic, + admin + ); + } else { + return ConnectorOffsetBackingStore.withConnectorAndWorkerStores( + () -> LoggingContext.forConnector(connName), + globalOffsetBackingStore, + connectorStore, + connectorSpecificOffsetsTopic, + admin + ); + } + } + + // Visible for testing + ConnectorOffsetBackingStore offsetStoreForRegularSourceTask( + ConnectorTaskId id, + SourceConnectorConfig sourceConfig, + Class connectorClass, + Producer producer, + Map producerProps, + Supplier topicAdminSupplier + ) { + String connectorSpecificOffsetsTopic = sourceConfig.offsetsTopic(); + // We use a connector-specific store (i.e., a dedicated KafkaOffsetBackingStore for this task) + // if the worker supports per-connector offsets topics (which may be the case in distributed mode but not standalone, for example) + // and the user has explicitly specified an offsets topic for the connector + final boolean usesConnectorSpecificStore = connectorSpecificOffsetsTopic != null + && config.connectorOffsetsTopicsPermitted(); + + if (usesConnectorSpecificStore) { + TopicAdmin topicAdmin = topicAdminSupplier.get(); + + Map consumerProps = regularSourceOffsetsConsumerConfigs( + id.connector(), "connector-consumer-" + id, config, sourceConfig, connectorClass, + connectorClientConfigOverridePolicy, kafkaClusterId); + KafkaConsumer consumer = new KafkaConsumer<>(consumerProps); + + KafkaOffsetBackingStore connectorStore = + KafkaOffsetBackingStore.forTask(sourceConfig.offsetsTopic(), producer, consumer, topicAdmin); + + // If the connector's offsets topic is the same as the worker-global offsets topic, there's no need to construct + // an offset store that has a primary and a secondary store which both read from that same topic. + // So, if the user has (implicitly or explicitly) configured the connector with a connector-specific offsets topic + // but we know that that topic is the same as the worker-global offsets topic, we ignore the worker-global + // offset store and build a store backed exclusively by a connector-specific offsets store. + // It may seem reasonable to instead build a store backed exclusively by the worker-global offset store, but that + // would prevent users from being able to customize the config properties used for the Kafka clients that + // access the offsets topic, and we would not be able to establish reasonable defaults like setting + // isolation.level=read_committed for the offsets topic consumer for this task + if (sameOffsetTopicAsWorker(sourceConfig.offsetsTopic(), producerProps)) { + return ConnectorOffsetBackingStore.withOnlyConnectorStore( + () -> LoggingContext.forTask(id), + connectorStore, + connectorSpecificOffsetsTopic, + topicAdmin + ); + } else { + return ConnectorOffsetBackingStore.withConnectorAndWorkerStores( + () -> LoggingContext.forTask(id), + globalOffsetBackingStore, + connectorStore, + connectorSpecificOffsetsTopic, + topicAdmin + ); + } + } else { + return ConnectorOffsetBackingStore.withOnlyWorkerStore( + () -> LoggingContext.forTask(id), + globalOffsetBackingStore, + config.offsetsTopic() + ); + } + } + + // Visible for testing + ConnectorOffsetBackingStore offsetStoreForExactlyOnceSourceTask( + ConnectorTaskId id, + SourceConnectorConfig sourceConfig, + Class connectorClass, + Producer producer, + Map producerProps, + TopicAdmin topicAdmin + ) { + Map consumerProps = exactlyOnceSourceOffsetsConsumerConfigs( + id.connector(), "connector-consumer-" + id, config, sourceConfig, connectorClass, + connectorClientConfigOverridePolicy, kafkaClusterId); + KafkaConsumer consumer = new KafkaConsumer<>(consumerProps); + + String connectorOffsetsTopic = Optional.ofNullable(sourceConfig.offsetsTopic()).orElse(config.offsetsTopic()); + + KafkaOffsetBackingStore connectorStore = + KafkaOffsetBackingStore.forTask(connectorOffsetsTopic, producer, consumer, topicAdmin); + + // If the connector's offsets topic is the same as the worker-global offsets topic, there's no need to construct + // an offset store that has a primary and a secondary store which both read from that same topic. + // So, if the user has (implicitly or explicitly) configured the connector with a connector-specific offsets topic + // but we know that that topic is the same as the worker-global offsets topic, we ignore the worker-global + // offset store and build a store backed exclusively by a connector-specific offsets store. + // We cannot under any circumstances build an offset store backed exclusively by the worker-global offset store + // as that would prevent us from being able to write source records and source offset information for the task + // with the same producer, and therefore, in the same transaction. + if (sameOffsetTopicAsWorker(connectorOffsetsTopic, producerProps)) { + return ConnectorOffsetBackingStore.withOnlyConnectorStore( + () -> LoggingContext.forTask(id), + connectorStore, + connectorOffsetsTopic, + topicAdmin + ); + } else { + return ConnectorOffsetBackingStore.withConnectorAndWorkerStores( + () -> LoggingContext.forTask(id), + globalOffsetBackingStore, + connectorStore, + connectorOffsetsTopic, + topicAdmin + ); + } + } + + /** + * Gives a best-effort guess for whether the given offsets topic is the same topic as the worker-global offsets topic. + * Even if the name of the topic is the same as the name of the worker's offsets topic, the two may still be different topics + * if the connector is configured to produce to a different Kafka cluster than the one that hosts the worker's offsets topic. + * @param offsetsTopic the name of the offsets topic for the connector + * @param producerProps the producer configuration for the connector + * @return whether it appears that the connector's offsets topic is the same topic as the worker-global offsets topic. + * If {@code true}, it is guaranteed that the two are the same; + * if {@code false}, it is likely but not guaranteed that the two are not the same + */ + private boolean sameOffsetTopicAsWorker(String offsetsTopic, Map producerProps) { + // We can check the offset topic name and the Kafka cluster's bootstrap servers, + // although this isn't exact and can lead to some false negatives if the user + // provides an overridden bootstrap servers value for their producer that is different than + // the worker's but still resolves to the same Kafka cluster used by the worker. + // At the moment this is probably adequate, especially since we don't want to put + // a network ping to a remote Kafka cluster inside the herder's tick thread (which is where this + // logic takes place right now) in case that takes a while. + return offsetsTopic.equals(config.offsetsTopic()) + && config.bootstrapServers().equals(producerProps.get(BOOTSTRAP_SERVERS_CONFIG)); + } + static class ConnectorStatusMetricsGroup { private final ConnectMetrics connectMetrics; private final ConnectMetricsRegistry registry; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java index 73b743bbe114c..7919323d8e594 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConfig.java @@ -110,7 +110,8 @@ public class WorkerConfig extends AbstractConfig { private static final String OFFSET_COMMIT_TIMEOUT_MS_DOC = "Maximum number of milliseconds to wait for records to flush and partition offset data to be" + " committed to offset storage before cancelling the process and restoring the offset " - + "data to be committed in a future attempt."; + + "data to be committed in a future attempt. This property has no effect for source connectors " + + "running with exactly-once support."; public static final long OFFSET_COMMIT_TIMEOUT_MS_DEFAULT = 5000L; public static final String LISTENERS_CONFIG = "listeners"; @@ -342,6 +343,15 @@ private void logPluginPathConfigProviderWarning(Map rawOriginals } } + /** + * @return the {@link CommonClientConfigs#BOOTSTRAP_SERVERS_CONFIG bootstrap servers} property + * used by the worker when instantiating Kafka clients for connectors and tasks (unless overridden) + * and its internal topics (if running in distributed mode) + */ + public String bootstrapServers() { + return String.join(",", getList(BOOTSTRAP_SERVERS_CONFIG)); + } + public Integer getRebalanceTimeout() { return null; } @@ -350,6 +360,54 @@ public boolean topicCreationEnable() { return getBoolean(TOPIC_CREATION_ENABLE_CONFIG); } + /** + * Whether this worker is configured with exactly-once support for source connectors. + * The default implementation returns {@code false} and should be overridden by subclasses + * if the worker mode for the subclass provides exactly-once support for source connectors. + * @return whether exactly-once support is enabled for source connectors on this worker + */ + public boolean exactlyOnceSourceEnabled() { + return false; + } + + /** + * Get the internal topic used by this worker to store source connector offsets. + * The default implementation returns {@code null} and should be overridden by subclasses + * if the worker mode for the subclass uses an internal offsets topic. + * @return the name of the internal offsets topic, or {@code null} if the worker does not use + * an internal offsets topic + */ + public String offsetsTopic() { + return null; + } + + /** + * Determine whether this worker supports per-connector source offsets topics. + * The default implementation returns {@code false} and should be overridden by subclasses + * if the worker mode for the subclass supports per-connector offsets topics. + * @return whether the worker supports per-connector offsets topics + */ + public boolean connectorOffsetsTopicsPermitted() { + return false; + } + + /** + * @return the offset commit interval for tasks created by this worker + */ + public long offsetCommitInterval() { + return getLong(OFFSET_COMMIT_INTERVAL_MS_CONFIG); + } + + /** + * Get the {@link CommonClientConfigs#GROUP_ID_CONFIG group ID} used by this worker to form a cluster. + * The default implementation returns {@code null} and should be overridden by subclasses + * if the worker mode for the subclass is capable of forming a cluster using Kafka's group coordination API. + * @return the group ID for the worker's cluster, or {@code null} if the worker is not capable of forming a cluster. + */ + public String groupId() { + return null; + } + @Override protected Map postProcessParsedConfig(final Map parsedValues) { return CommonClientConfigs.postProcessReconnectBackoffConfigs(this, parsedValues); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java index 09b57fd42a851..607f5db5d290b 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerConnector.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.connect.runtime; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.connect.connector.Connector; import org.apache.kafka.connect.connector.ConnectorContext; import org.apache.kafka.connect.errors.ConnectException; @@ -23,6 +24,7 @@ import org.apache.kafka.connect.runtime.isolation.Plugins; import org.apache.kafka.connect.sink.SinkConnectorContext; import org.apache.kafka.connect.source.SourceConnectorContext; +import org.apache.kafka.connect.storage.CloseableOffsetStorageReader; import org.apache.kafka.connect.storage.OffsetStorageReader; import org.apache.kafka.connect.util.Callback; import org.apache.kafka.connect.util.ConnectUtils; @@ -74,7 +76,7 @@ private enum State { private volatile boolean cancelled; // indicates whether the Worker has cancelled the connector (e.g. because of slow shutdown) private State state; - private final OffsetStorageReader offsetStorageReader; + private final CloseableOffsetStorageReader offsetStorageReader; public WorkerConnector(String connName, Connector connector, @@ -82,7 +84,7 @@ public WorkerConnector(String connName, CloseableConnectorContext ctx, ConnectMetrics metrics, ConnectorStatus.Listener statusListener, - OffsetStorageReader offsetStorageReader, + CloseableOffsetStorageReader offsetStorageReader, ClassLoader loader) { this.connName = connName; this.config = connectorConfig.originalsStrings(); @@ -165,6 +167,7 @@ void initialize() { SinkConnectorConfig.validate(config); connector.initialize(new WorkerSinkConnectorContext()); } else { + Objects.requireNonNull(offsetStorageReader, "Offset reader cannot be null for source connectors"); connector.initialize(new WorkerSourceConnectorContext(offsetStorageReader)); } } catch (Throwable t) { @@ -271,8 +274,9 @@ void doShutdown() { state = State.FAILED; statusListener.onFailure(connName, t); } finally { - ctx.close(); - metrics.close(); + Utils.closeQuietly(ctx, "Connector context for " + connName); + Utils.closeQuietly(metrics, "Connector metrics for " + connName); + Utils.closeQuietly(offsetStorageReader, "Offset reader for " + connName); } } @@ -281,7 +285,9 @@ public synchronized void cancel() { // instance is being abandoned and we won't update the status on its behalf any more // after this since a new instance may be started soon statusListener.onShutdown(connName); - ctx.close(); + Utils.closeQuietly(ctx, "Connector context for " + connName); + // Preemptively close the offset reader in case the connector is blocked on an offset read. + Utils.closeQuietly(offsetStorageReader, "Offset reader for " + connName); cancelled = true; } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java index 72ee749d14d5e..2ca0f9a3cc866 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java @@ -40,7 +40,7 @@ import org.apache.kafka.connect.header.ConnectHeaders; import org.apache.kafka.connect.header.Headers; import org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator; import org.apache.kafka.connect.runtime.errors.Stage; import org.apache.kafka.connect.runtime.errors.WorkerErrantRecordReporter; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTaskContext.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTaskContext.java index 724b02ec56724..125d9e1094667 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTaskContext.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTaskContext.java @@ -19,7 +19,7 @@ import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.connect.errors.IllegalWorkerStateException; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.sink.ErrantRecordReporter; import org.apache.kafka.connect.sink.SinkTaskContext; import org.slf4j.Logger; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java index b036396d2aa8b..5fbfd536e95fb 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java @@ -16,50 +16,31 @@ */ package org.apache.kafka.connect.runtime; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.header.internals.RecordHeaders; -import org.apache.kafka.common.metrics.Sensor; -import org.apache.kafka.common.metrics.stats.Avg; -import org.apache.kafka.common.metrics.stats.CumulativeSum; -import org.apache.kafka.common.metrics.stats.Max; -import org.apache.kafka.common.metrics.stats.Rate; -import org.apache.kafka.common.metrics.stats.Value; import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.errors.RetriableException; -import org.apache.kafka.connect.header.Header; -import org.apache.kafka.connect.header.Headers; -import org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup; -import org.apache.kafka.connect.runtime.SubmittedRecords.SubmittedRecord; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator; import org.apache.kafka.connect.runtime.errors.Stage; import org.apache.kafka.connect.runtime.errors.ToleranceType; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; import org.apache.kafka.connect.storage.CloseableOffsetStorageReader; +import org.apache.kafka.connect.storage.ClusterConfigState; +import org.apache.kafka.connect.storage.ConnectorOffsetBackingStore; import org.apache.kafka.connect.storage.Converter; import org.apache.kafka.connect.storage.HeaderConverter; import org.apache.kafka.connect.storage.OffsetStorageWriter; import org.apache.kafka.connect.storage.StatusBackingStore; -import org.apache.kafka.connect.util.ConnectUtils; import org.apache.kafka.connect.util.ConnectorTaskId; import org.apache.kafka.connect.util.TopicAdmin; -import org.apache.kafka.connect.util.TopicCreation; import org.apache.kafka.connect.util.TopicCreationGroup; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.time.Duration; -import java.util.List; import java.util.Map; -import java.util.concurrent.CountDownLatch; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.Future; @@ -68,40 +49,16 @@ import java.util.concurrent.atomic.AtomicReference; import static org.apache.kafka.connect.runtime.SubmittedRecords.CommittableOffsets; -import static org.apache.kafka.connect.runtime.WorkerConfig.TOPIC_TRACKING_ENABLE_CONFIG; /** * WorkerTask that uses a SourceTask to ingest data into Kafka. */ -class WorkerSourceTask extends WorkerTask { +class WorkerSourceTask extends AbstractWorkerSourceTask { private static final Logger log = LoggerFactory.getLogger(WorkerSourceTask.class); - private static final long SEND_FAILED_BACKOFF_MS = 100; - - private final WorkerConfig workerConfig; - private final SourceTask task; - private final ClusterConfigState configState; - private final Converter keyConverter; - private final Converter valueConverter; - private final HeaderConverter headerConverter; - private final TransformationChain transformationChain; - private final KafkaProducer producer; - private final TopicAdmin admin; - private final CloseableOffsetStorageReader offsetReader; - private final OffsetStorageWriter offsetWriter; - private final Executor closeExecutor; - private final SourceTaskMetricsGroup sourceTaskMetricsGroup; - private final AtomicReference producerSendException; - private final boolean isTopicTrackingEnabled; - private final TopicCreation topicCreation; - - private List toSend; private volatile CommittableOffsets committableOffsets; private final SubmittedRecords submittedRecords; - private final CountDownLatch stopRequestedLatch; - - private Map taskConfig; - private boolean started = false; + private final AtomicReference producerSendException; public WorkerSourceTask(ConnectorTaskId id, SourceTask task, @@ -111,11 +68,12 @@ public WorkerSourceTask(ConnectorTaskId id, Converter valueConverter, HeaderConverter headerConverter, TransformationChain transformationChain, - KafkaProducer producer, + Producer producer, TopicAdmin admin, Map topicGroups, CloseableOffsetStorageReader offsetReader, OffsetStorageWriter offsetWriter, + ConnectorOffsetBackingStore offsetBackingStore, WorkerConfig workerConfig, ClusterConfigState configState, ConnectMetrics connectMetrics, @@ -125,355 +83,115 @@ public WorkerSourceTask(ConnectorTaskId id, StatusBackingStore statusBackingStore, Executor closeExecutor) { - super(id, statusListener, initialState, loader, connectMetrics, - retryWithToleranceOperator, time, statusBackingStore); - - this.workerConfig = workerConfig; - this.task = task; - this.configState = configState; - this.keyConverter = keyConverter; - this.valueConverter = valueConverter; - this.headerConverter = headerConverter; - this.transformationChain = transformationChain; - this.producer = producer; - this.admin = admin; - this.offsetReader = offsetReader; - this.offsetWriter = offsetWriter; - this.closeExecutor = closeExecutor; - - this.toSend = null; + super(id, task, statusListener, initialState, keyConverter, valueConverter, headerConverter, transformationChain, + new WorkerSourceTaskContext(offsetReader, id, configState, null), producer, + admin, topicGroups, offsetReader, offsetWriter, offsetBackingStore, workerConfig, connectMetrics, loader, + time, retryWithToleranceOperator, statusBackingStore, closeExecutor); + this.committableOffsets = CommittableOffsets.EMPTY; this.submittedRecords = new SubmittedRecords(); - this.stopRequestedLatch = new CountDownLatch(1); - this.sourceTaskMetricsGroup = new SourceTaskMetricsGroup(id, connectMetrics); this.producerSendException = new AtomicReference<>(); - this.isTopicTrackingEnabled = workerConfig.getBoolean(TOPIC_TRACKING_ENABLE_CONFIG); - this.topicCreation = TopicCreation.newTopicCreation(workerConfig, topicGroups); } @Override - public void initialize(TaskConfig taskConfig) { - try { - this.taskConfig = taskConfig.originalsStrings(); - } catch (Throwable t) { - log.error("{} Task failed initialization and will not be started.", this, t); - onFailure(t); - } + protected void prepareToInitializeTask() { + // No-op } @Override - protected void close() { - if (started) { - try { - task.stop(); - } catch (Throwable t) { - log.warn("Could not stop task", t); - } - } - - closeProducer(Duration.ofSeconds(30)); - - if (admin != null) { - try { - admin.close(Duration.ofSeconds(30)); - } catch (Throwable t) { - log.warn("Failed to close admin client on time", t); - } - } - Utils.closeQuietly(transformationChain, "transformation chain"); - Utils.closeQuietly(retryWithToleranceOperator, "retry operator"); + protected void prepareToEnterSendLoop() { + // No-op } @Override - public void removeMetrics() { - try { - sourceTaskMetricsGroup.close(); - } finally { - super.removeMetrics(); - } + protected void beginSendIteration() { + updateCommittableOffsets(); } @Override - public void cancel() { - super.cancel(); - offsetReader.close(); - // We proactively close the producer here as the main work thread for the task may - // be blocked indefinitely in a call to Producer::send if automatic topic creation is - // not enabled on either the connector or the Kafka cluster. Closing the producer should - // unblock it in that case and allow shutdown to proceed normally. - // With a duration of 0, the producer's own shutdown logic should be fairly quick, - // but closing user-pluggable classes like interceptors may lag indefinitely. So, we - // call close on a separate thread in order to avoid blocking the herder's tick thread. - closeExecutor.execute(() -> closeProducer(Duration.ZERO)); + protected void prepareToPollTask() { + maybeThrowProducerSendException(); } @Override - public void stop() { - super.stop(); - stopRequestedLatch.countDown(); + protected void recordDropped(SourceRecord record) { + commitTaskRecord(record, null); } @Override - protected void initializeAndStart() { - // If we try to start the task at all by invoking initialize, then count this as - // "started" and expect a subsequent call to the task's stop() method - // to properly clean up any resources allocated by its initialize() or - // start() methods. If the task throws an exception during stop(), - // the worst thing that happens is another exception gets logged for an already- - // failed task - started = true; - task.initialize(new WorkerSourceTaskContext(offsetReader, this, configState)); - task.start(taskConfig); - log.info("{} Source task finished initialization and start", this); + protected Optional prepareToSendRecord( + SourceRecord sourceRecord, + ProducerRecord producerRecord + ) { + maybeThrowProducerSendException(); + return Optional.of(submittedRecords.submit(sourceRecord)); } @Override - public void execute() { - try { - log.info("{} Executing source task", this); - while (!isStopping()) { - updateCommittableOffsets(); - - if (shouldPause()) { - onPause(); - if (awaitUnpause()) { - onResume(); - } - continue; - } - - maybeThrowProducerSendException(); - if (toSend == null) { - log.trace("{} Nothing to send to Kafka. Polling source for additional records", this); - long start = time.milliseconds(); - toSend = poll(); - if (toSend != null) { - recordPollReturned(toSend.size(), time.milliseconds() - start); - } - } - - if (toSend == null) - continue; - log.trace("{} About to send {} records to Kafka", this, toSend.size()); - if (!sendRecords()) - stopRequestedLatch.await(SEND_FAILED_BACKOFF_MS, TimeUnit.MILLISECONDS); - } - } catch (InterruptedException e) { - // Ignore and allow to exit. - } finally { - submittedRecords.awaitAllMessages( - workerConfig.getLong(WorkerConfig.OFFSET_COMMIT_TIMEOUT_MS_CONFIG), - TimeUnit.MILLISECONDS - ); - // It should still be safe to commit offsets since any exception would have - // simply resulted in not getting more records but all the existing records should be ok to flush - // and commit offsets. Worst case, task.flush() will also throw an exception causing the offset commit - // to fail. - updateCommittableOffsets(); - commitOffsets(); - } + protected void recordDispatched(SourceRecord record) { + // No-op } - private void closeProducer(Duration duration) { - if (producer != null) { - try { - producer.close(duration); - } catch (Throwable t) { - log.warn("Could not close producer for {}", id, t); - } - } - } - - private void maybeThrowProducerSendException() { - if (producerSendException.get() != null) { - throw new ConnectException( - "Unrecoverable exception from producer send callback", - producerSendException.get() - ); - } - } - - private void updateCommittableOffsets() { - CommittableOffsets newOffsets = submittedRecords.committableOffsets(); - synchronized (this) { - this.committableOffsets = this.committableOffsets.updatedWith(newOffsets); - } - } - - protected List poll() throws InterruptedException { - try { - return task.poll(); - } catch (RetriableException | org.apache.kafka.common.errors.RetriableException e) { - log.warn("{} failed to poll records from SourceTask. Will retry operation.", this, e); - // Do nothing. Let the framework poll whenever it's ready. - return null; - } - } - - /** - * Convert the source record into a producer record. - * - * @param record the transformed record - * @return the producer record which can sent over to Kafka. A null is returned if the input is null or - * if an error was encountered during any of the converter stages. - */ - private ProducerRecord convertTransformedRecord(SourceRecord record) { - if (record == null) { - return null; - } - - RecordHeaders headers = retryWithToleranceOperator.execute(() -> convertHeaderFor(record), Stage.HEADER_CONVERTER, headerConverter.getClass()); - - byte[] key = retryWithToleranceOperator.execute(() -> keyConverter.fromConnectData(record.topic(), headers, record.keySchema(), record.key()), - Stage.KEY_CONVERTER, keyConverter.getClass()); - - byte[] value = retryWithToleranceOperator.execute(() -> valueConverter.fromConnectData(record.topic(), headers, record.valueSchema(), record.value()), - Stage.VALUE_CONVERTER, valueConverter.getClass()); - - if (retryWithToleranceOperator.failed()) { - return null; - } - - return new ProducerRecord<>(record.topic(), record.kafkaPartition(), - ConnectUtils.checkAndConvertTimestamp(record.timestamp()), key, value, headers); + @Override + protected void batchDispatched() { + // No-op } - /** - * Try to send a batch of records. If a send fails and is retriable, this saves the remainder of the batch so it can - * be retried after backing off. If a send fails and is not retriable, this will throw a ConnectException. - * @return true if all messages were sent, false if some need to be retried - */ - private boolean sendRecords() { - int processed = 0; - recordBatch(toSend.size()); - final SourceRecordWriteCounter counter = - toSend.size() > 0 ? new SourceRecordWriteCounter(toSend.size(), sourceTaskMetricsGroup) : null; - for (final SourceRecord preTransformRecord : toSend) { - maybeThrowProducerSendException(); - - retryWithToleranceOperator.sourceRecord(preTransformRecord); - final SourceRecord record = transformationChain.apply(preTransformRecord); - final ProducerRecord producerRecord = convertTransformedRecord(record); - if (producerRecord == null || retryWithToleranceOperator.failed()) { - counter.skipRecord(); - commitTaskRecord(preTransformRecord, null); - continue; - } - - log.trace("{} Appending record to the topic {} with key {}, value {}", this, record.topic(), record.key(), record.value()); - SubmittedRecord submittedRecord = submittedRecords.submit(record); - try { - maybeCreateTopic(record.topic()); - final String topic = producerRecord.topic(); - producer.send( - producerRecord, - (recordMetadata, e) -> { - if (e != null) { - if (retryWithToleranceOperator.getErrorToleranceType() == ToleranceType.ALL) { - log.trace("Ignoring failed record send: {} failed to send record to {}: ", - WorkerSourceTask.this, topic, e); - // executeFailed here allows the use of existing logging infrastructure/configuration - retryWithToleranceOperator.executeFailed(Stage.KAFKA_PRODUCE, WorkerSourceTask.class, - preTransformRecord, e); - commitTaskRecord(preTransformRecord, null); - } else { - log.error("{} failed to send record to {}: ", WorkerSourceTask.this, topic, e); - log.trace("{} Failed record: {}", WorkerSourceTask.this, preTransformRecord); - producerSendException.compareAndSet(null, e); - } - } else { - submittedRecord.ack(); - counter.completeRecord(); - log.trace("{} Wrote record successfully: topic {} partition {} offset {}", - WorkerSourceTask.this, - recordMetadata.topic(), recordMetadata.partition(), - recordMetadata.offset()); - commitTaskRecord(preTransformRecord, recordMetadata); - if (isTopicTrackingEnabled) { - recordActiveTopic(producerRecord.topic()); - } - } - }); - } catch (RetriableException | org.apache.kafka.common.errors.RetriableException e) { - log.warn("{} Failed to send record to topic '{}' and partition '{}'. Backing off before retrying: ", - this, producerRecord.topic(), producerRecord.partition(), e); - toSend = toSend.subList(processed, toSend.size()); - submittedRecords.removeLastOccurrence(submittedRecord); - counter.retryRemaining(); - return false; - } catch (ConnectException e) { - log.warn("{} Failed to send record to topic '{}' and partition '{}' due to an unrecoverable exception: ", - this, producerRecord.topic(), producerRecord.partition(), e); - log.trace("{} Failed to send {} with unrecoverable exception: ", this, producerRecord, e); - throw e; - } catch (KafkaException e) { - throw new ConnectException("Unrecoverable exception trying to send", e); - } - processed++; - } - toSend = null; - return true; + @Override + protected void recordSent( + SourceRecord sourceRecord, + ProducerRecord producerRecord, + RecordMetadata recordMetadata + ) { + commitTaskRecord(sourceRecord, recordMetadata); } - // Due to transformations that may change the destination topic of a record (such as - // RegexRouter) topic creation can not be batched for multiple topics - private void maybeCreateTopic(String topic) { - if (!topicCreation.isTopicCreationRequired(topic)) { - log.trace("Topic creation by the connector is disabled or the topic {} was previously created." + - "If auto.create.topics.enable is enabled on the broker, " + - "the topic will be created with default settings", topic); - return; - } - log.info("The task will send records to topic '{}' for the first time. Checking " - + "whether topic exists", topic); - Map existing = admin.describeTopics(topic); - if (!existing.isEmpty()) { - log.info("Topic '{}' already exists.", topic); - topicCreation.addTopic(topic); - return; - } - - log.info("Creating topic '{}'", topic); - TopicCreationGroup topicGroup = topicCreation.findFirstGroup(topic); - log.debug("Topic '{}' matched topic creation group: {}", topic, topicGroup); - NewTopic newTopic = topicGroup.newTopic(topic); - - TopicAdmin.TopicCreationResponse response = admin.createOrFindTopics(newTopic); - if (response.isCreated(newTopic.name())) { - topicCreation.addTopic(topic); - log.info("Created topic '{}' using creation group {}", newTopic, topicGroup); - } else if (response.isExisting(newTopic.name())) { - topicCreation.addTopic(topic); - log.info("Found existing topic '{}'", newTopic); + @Override + protected void producerSendFailed( + boolean synchronous, + ProducerRecord producerRecord, + SourceRecord preTransformRecord, + Exception e + ) { + if (synchronous) { + throw new ConnectException("Unrecoverable exception trying to send", e); + } + + String topic = producerRecord.topic(); + if (retryWithToleranceOperator.getErrorToleranceType() == ToleranceType.ALL) { + log.trace( + "Ignoring failed record send: {} failed to send record to {}: ", + WorkerSourceTask.this, + topic, + e + ); + // executeFailed here allows the use of existing logging infrastructure/configuration + retryWithToleranceOperator.executeFailed( + Stage.KAFKA_PRODUCE, + WorkerSourceTask.class, + preTransformRecord, + e + ); + commitTaskRecord(preTransformRecord, null); } else { - // The topic still does not exist and could not be created, so treat it as a task failure - log.warn("Request to create new topic '{}' failed", topic); - throw new ConnectException("Task failed to create new topic " + newTopic + ". Ensure " - + "that the task is authorized to create topics or that the topic exists and " - + "restart the task"); - } - } - - private RecordHeaders convertHeaderFor(SourceRecord record) { - Headers headers = record.headers(); - RecordHeaders result = new RecordHeaders(); - if (headers != null) { - String topic = record.topic(); - for (Header header : headers) { - String key = header.key(); - byte[] rawHeader = headerConverter.fromConnectHeader(topic, key, header.schema(), header.value()); - result.add(key, rawHeader); - } + log.error("{} failed to send record to {}: ", WorkerSourceTask.this, topic, e); + log.trace("{} Failed record: {}", WorkerSourceTask.this, preTransformRecord); + producerSendException.compareAndSet(null, e); } - return result; } - private void commitTaskRecord(SourceRecord record, RecordMetadata metadata) { - try { - task.commitRecord(record, metadata); - } catch (Throwable t) { - log.error("{} Exception thrown while calling task.commitRecord()", this, t); - } + @Override + protected void finalOffsetCommit(boolean failed) { + // It should still be safe to commit offsets since any exception would have + // simply resulted in not getting more records but all the existing records should be ok to flush + // and commit offsets. Worst case, task.commit() will also throw an exception causing the offset + // commit to fail. + submittedRecords.awaitAllMessages( + workerConfig.getLong(WorkerConfig.OFFSET_COMMIT_TIMEOUT_MS_CONFIG), + TimeUnit.MILLISECONDS + ); + updateCommittableOffsets(); + commitOffsets(); } public boolean commitOffsets() { @@ -491,9 +209,9 @@ public boolean commitOffsets() { } if (committableOffsets.isEmpty()) { - log.info("{} Either no records were produced by the task since the last offset commit, " - + "or every record has been filtered out by a transformation " - + "or dropped due to transformation or conversion errors.", + log.info("{} Either no records were produced by the task since the last offset commit, " + + "or every record has been filtered out by a transformation " + + "or dropped due to transformation or conversion errors.", this ); // We continue with the offset commit process here instead of simply returning immediately @@ -510,8 +228,8 @@ public boolean commitOffsets() { committableOffsets.largestDequeSize() ); } else { - log.debug("{} There are currently no pending messages for this offset commit; " - + "all messages dispatched to the task's producer since the last commit have been acknowledged", + log.debug("{} There are currently no pending messages for this offset commit; " + + "all messages dispatched to the task's producer since the last commit have been acknowledged", this ); } @@ -582,11 +300,19 @@ public boolean commitOffsets() { return true; } - private void commitSourceTask() { - try { - this.task.commit(); - } catch (Throwable t) { - log.error("{} Exception thrown while calling task.commit()", this, t); + private void updateCommittableOffsets() { + CommittableOffsets newOffsets = submittedRecords.committableOffsets(); + synchronized (this) { + this.committableOffsets = this.committableOffsets.updatedWith(newOffsets); + } + } + + private void maybeThrowProducerSendException() { + if (producerSendException.get() != null) { + throw new ConnectException( + "Unrecoverable exception from producer send callback", + producerSendException.get() + ); } } @@ -597,101 +323,4 @@ public String toString() { '}'; } - protected void recordPollReturned(int numRecordsInBatch, long duration) { - sourceTaskMetricsGroup.recordPoll(numRecordsInBatch, duration); - } - - SourceTaskMetricsGroup sourceTaskMetricsGroup() { - return sourceTaskMetricsGroup; - } - - static class SourceRecordWriteCounter { - private final SourceTaskMetricsGroup metricsGroup; - private final int batchSize; - private boolean completed = false; - private int counter; - public SourceRecordWriteCounter(int batchSize, SourceTaskMetricsGroup metricsGroup) { - assert batchSize > 0; - assert metricsGroup != null; - this.batchSize = batchSize; - counter = batchSize; - this.metricsGroup = metricsGroup; - } - public void skipRecord() { - if (counter > 0 && --counter == 0) { - finishedAllWrites(); - } - } - public void completeRecord() { - if (counter > 0 && --counter == 0) { - finishedAllWrites(); - } - } - public void retryRemaining() { - finishedAllWrites(); - } - private void finishedAllWrites() { - if (!completed) { - metricsGroup.recordWrite(batchSize - counter); - completed = true; - } - } - } - - static class SourceTaskMetricsGroup { - private final MetricGroup metricGroup; - private final Sensor sourceRecordPoll; - private final Sensor sourceRecordWrite; - private final Sensor sourceRecordActiveCount; - private final Sensor pollTime; - private int activeRecordCount; - - public SourceTaskMetricsGroup(ConnectorTaskId id, ConnectMetrics connectMetrics) { - ConnectMetricsRegistry registry = connectMetrics.registry(); - metricGroup = connectMetrics.group(registry.sourceTaskGroupName(), - registry.connectorTagName(), id.connector(), - registry.taskTagName(), Integer.toString(id.task())); - // remove any previously created metrics in this group to prevent collisions. - metricGroup.close(); - - sourceRecordPoll = metricGroup.sensor("source-record-poll"); - sourceRecordPoll.add(metricGroup.metricName(registry.sourceRecordPollRate), new Rate()); - sourceRecordPoll.add(metricGroup.metricName(registry.sourceRecordPollTotal), new CumulativeSum()); - - sourceRecordWrite = metricGroup.sensor("source-record-write"); - sourceRecordWrite.add(metricGroup.metricName(registry.sourceRecordWriteRate), new Rate()); - sourceRecordWrite.add(metricGroup.metricName(registry.sourceRecordWriteTotal), new CumulativeSum()); - - pollTime = metricGroup.sensor("poll-batch-time"); - pollTime.add(metricGroup.metricName(registry.sourceRecordPollBatchTimeMax), new Max()); - pollTime.add(metricGroup.metricName(registry.sourceRecordPollBatchTimeAvg), new Avg()); - - sourceRecordActiveCount = metricGroup.sensor("source-record-active-count"); - sourceRecordActiveCount.add(metricGroup.metricName(registry.sourceRecordActiveCount), new Value()); - sourceRecordActiveCount.add(metricGroup.metricName(registry.sourceRecordActiveCountMax), new Max()); - sourceRecordActiveCount.add(metricGroup.metricName(registry.sourceRecordActiveCountAvg), new Avg()); - } - - void close() { - metricGroup.close(); - } - - void recordPoll(int batchSize, long duration) { - sourceRecordPoll.record(batchSize); - pollTime.record(duration); - activeRecordCount += batchSize; - sourceRecordActiveCount.record(activeRecordCount); - } - - void recordWrite(int recordCount) { - sourceRecordWrite.record(recordCount); - activeRecordCount -= recordCount; - activeRecordCount = Math.max(0, activeRecordCount); - sourceRecordActiveCount.record(activeRecordCount); - } - - protected MetricGroup metricGroup() { - return metricGroup; - } - } -} +} \ No newline at end of file diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTaskContext.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTaskContext.java index fe1409b282aa0..d58e98e057443 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTaskContext.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTaskContext.java @@ -16,33 +16,42 @@ */ package org.apache.kafka.connect.runtime; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.source.SourceTaskContext; import org.apache.kafka.connect.storage.OffsetStorageReader; +import org.apache.kafka.connect.util.ConnectorTaskId; import java.util.Map; public class WorkerSourceTaskContext implements SourceTaskContext { private final OffsetStorageReader reader; - private final WorkerSourceTask task; + private final ConnectorTaskId id; private final ClusterConfigState configState; + private final WorkerTransactionContext transactionContext; public WorkerSourceTaskContext(OffsetStorageReader reader, - WorkerSourceTask task, - ClusterConfigState configState) { + ConnectorTaskId id, + ClusterConfigState configState, + WorkerTransactionContext transactionContext) { this.reader = reader; - this.task = task; + this.id = id; this.configState = configState; + this.transactionContext = transactionContext; } @Override public Map configs() { - return configState.taskConfig(task.id()); + return configState.taskConfig(id); } @Override public OffsetStorageReader offsetStorageReader() { return reader; } + + @Override + public WorkerTransactionContext transactionContext() { + return transactionContext; + } } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java index 0d893f56ee568..0dc8c7d91c351 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTask.java @@ -52,10 +52,10 @@ abstract class WorkerTask implements Runnable { private static final Logger log = LoggerFactory.getLogger(WorkerTask.class); private static final String THREAD_NAME_PREFIX = "task-thread-"; - protected final ConnectorTaskId id; private final TaskStatus.Listener statusListener; + private final StatusBackingStore statusBackingStore; + protected final ConnectorTaskId id; protected final ClassLoader loader; - protected final StatusBackingStore statusBackingStore; protected final Time time; private final CountDownLatch shutdownLatch = new CountDownLatch(1); private final TaskMetricsGroup taskMetricsGroup; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTransactionContext.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTransactionContext.java new file mode 100644 index 0000000000000..1d64e9f0f5c28 --- /dev/null +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerTransactionContext.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.runtime; + +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.TransactionContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +public class WorkerTransactionContext implements TransactionContext { + + private static final Logger log = LoggerFactory.getLogger(WorkerTransactionContext.class); + + private final Set commitableRecords = new HashSet<>(); + private final Set abortableRecords = new HashSet<>(); + private volatile boolean batchCommitRequested = false; + private volatile boolean batchAbortRequested = false; + + @Override + public synchronized void commitTransaction() { + batchCommitRequested = true; + } + + @Override + public synchronized void commitTransaction(SourceRecord record) { + Objects.requireNonNull(record, "Source record used to define transaction boundaries may not be null"); + commitableRecords.add(record); + } + + @Override + public synchronized void abortTransaction() { + batchAbortRequested = true; + } + + @Override + public void abortTransaction(SourceRecord record) { + Objects.requireNonNull(record, "Source record used to define transaction boundaries may not be null"); + abortableRecords.add(record); + } + + public synchronized boolean shouldCommitBatch() { + checkBatchRequestsConsistency(); + boolean result = batchCommitRequested; + batchCommitRequested = false; + return result; + } + + public synchronized boolean shouldAbortBatch() { + checkBatchRequestsConsistency(); + boolean result = batchAbortRequested; + batchAbortRequested = false; + return result; + } + + public synchronized boolean shouldCommitOn(SourceRecord record) { + // We could perform this check in the connector-facing methods (such as commitTransaction(SourceRecord)), + // but the connector might swallow that exception. + // This way, we can fail the task unconditionally, which is warranted since the alternative may lead to data loss. + // Essentially, instead of telling the task that it screwed up and trusting it to do the right thing, we rat on it to the + // worker and let it get punished accordingly. + checkRecordRequestConsistency(record); + return commitableRecords.remove(record); + } + + public synchronized boolean shouldAbortOn(SourceRecord record) { + checkRecordRequestConsistency(record); + return abortableRecords.remove(record); + } + + private synchronized void checkBatchRequestsConsistency() { + if (batchCommitRequested && batchAbortRequested) { + throw new IllegalStateException("Connector requested both commit and abort of same transaction"); + } + } + + private synchronized void checkRecordRequestConsistency(SourceRecord record) { + if (commitableRecords.contains(record) && abortableRecords.contains(record)) { + log.trace("Connector will fail as it has requested both commit and abort of transaction for same record: {}", record); + throw new IllegalStateException(String.format( + "Connector requested both commit and abort of same record against topic/partition %s/%s", + record.topic(), record.kafkaPartition() + )); + } + } + +} diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java index 0823fbcc30ad3..5a39b5af3ad0f 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedConfig.java @@ -17,6 +17,9 @@ package org.apache.kafka.connect.runtime.distributed; import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.config.TopicConfig; @@ -31,9 +34,12 @@ import java.security.NoSuchAlgorithmException; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.kafka.common.config.ConfigDef.Range.atLeast; import static org.apache.kafka.common.config.ConfigDef.Range.between; @@ -192,6 +198,44 @@ public class DistributedConfig extends WorkerConfig { public static final String INTER_WORKER_VERIFICATION_ALGORITHMS_DOC = "A list of permitted algorithms for verifying internal requests"; public static final List INTER_WORKER_VERIFICATION_ALGORITHMS_DEFAULT = Collections.singletonList(INTER_WORKER_SIGNATURE_ALGORITHM_DEFAULT); + private enum ExactlyOnceSourceSupport { + DISABLED(false), + PREPARING(true), + ENABLED(true); + + public final boolean usesTransactionalLeader; + + ExactlyOnceSourceSupport(boolean usesTransactionalLeader) { + this.usesTransactionalLeader = usesTransactionalLeader; + } + + public static List options() { + return Stream.of(values()).map(ExactlyOnceSourceSupport::toString).collect(Collectors.toList()); + } + + public static ExactlyOnceSourceSupport fromProperty(String property) { + return ExactlyOnceSourceSupport.valueOf(property.toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + public static final String EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG = "exactly.once.source.support"; + public static final String EXACTLY_ONCE_SOURCE_SUPPORT_DOC = "Whether to enable exactly-once support for source connectors in the cluster " + + "by using transactions to write source records and their source offsets, and by proactively fencing out old task generations before bringing up new ones. " + + "If this feature is enabled, consumers of the topics to which exactly-once source connectors write should be configured with the " + + ConsumerConfig.ISOLATION_LEVEL_CONFIG + " property set to '" + IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT) + "'. " + + "Note that this must be enabled on every worker in a cluster in order for exactly-once delivery to be guaranteed, " + + "and that some source connectors may still not be able to provide exactly-once delivery guarantees even with this support enabled. " + + "Permitted values are \"" + ExactlyOnceSourceSupport.DISABLED + "\", \"" + ExactlyOnceSourceSupport.PREPARING + "\", and \"" + ExactlyOnceSourceSupport.ENABLED + "\". " + + "In order to safely enable exactly-once support for source connectors, " + + "all workers in the cluster must first be updated to use the \"preparing\" value for this property. " + + "Once this has been done, a second update of all of the workers in the cluster should be performed to change the value of this property to \"enabled\"."; + public static final String EXACTLY_ONCE_SOURCE_SUPPORT_DEFAULT = ExactlyOnceSourceSupport.DISABLED.toString(); + @SuppressWarnings("unchecked") private static final ConfigDef CONFIG = baseConfigDef() .define(GROUP_ID_CONFIG, @@ -213,6 +257,12 @@ public class DistributedConfig extends WorkerConfig { Math.toIntExact(TimeUnit.SECONDS.toMillis(3)), ConfigDef.Importance.HIGH, HEARTBEAT_INTERVAL_MS_DOC) + .define(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, + ConfigDef.Type.STRING, + EXACTLY_ONCE_SOURCE_SUPPORT_DEFAULT, + ConfigDef.CaseInsensitiveValidString.in(ExactlyOnceSourceSupport.options().toArray(new String[0])), + ConfigDef.Importance.HIGH, + EXACTLY_ONCE_SOURCE_SUPPORT_DOC) .define(CommonClientConfigs.METADATA_MAX_AGE_CONFIG, ConfigDef.Type.LONG, TimeUnit.MINUTES.toMillis(5), @@ -396,13 +446,58 @@ public class DistributedConfig extends WorkerConfig { ConfigDef.Importance.LOW, INTER_WORKER_VERIFICATION_ALGORITHMS_DOC); + private final ExactlyOnceSourceSupport exactlyOnceSourceSupport; + @Override public Integer getRebalanceTimeout() { return getInt(DistributedConfig.REBALANCE_TIMEOUT_MS_CONFIG); } + @Override + public boolean exactlyOnceSourceEnabled() { + return exactlyOnceSourceSupport == ExactlyOnceSourceSupport.ENABLED; + } + + /** + * @return whether the Connect cluster's leader should use a transactional producer to perform writes to the config + * topic, which is useful for ensuring that zombie leaders are fenced out and unable to write to the topic after a + * new leader has been elected. + */ + public boolean transactionalLeaderEnabled() { + return exactlyOnceSourceSupport.usesTransactionalLeader; + } + + /** + * @return the {@link ProducerConfig#TRANSACTIONAL_ID_CONFIG transactional ID} to use for the worker's producer if + * the worker is the leader of the cluster and is + * {@link #transactionalLeaderEnabled() configured to use a transactional producer}. + */ + public String transactionalProducerId() { + return transactionalProducerId(groupId()); + } + + public static String transactionalProducerId(String groupId) { + return "connect-cluster-" + groupId; + } + + @Override + public String offsetsTopic() { + return getString(OFFSET_STORAGE_TOPIC_CONFIG); + } + + @Override + public boolean connectorOffsetsTopicsPermitted() { + return true; + } + + @Override + public String groupId() { + return getString(GROUP_ID_CONFIG); + } + public DistributedConfig(Map props) { super(CONFIG, props); + exactlyOnceSourceSupport = ExactlyOnceSourceSupport.fromProperty(getString(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG)); getInternalRequestKeyGenerator(); // Check here for a valid key size + key algorithm to fail fast if either are invalid validateKeyAlgorithmAndVerificationAlgorithms(); } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java index 5aa327ed61e5c..ca6bed977083a 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java @@ -17,8 +17,10 @@ package org.apache.kafka.connect.runtime.distributed; import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigValue; +import org.apache.kafka.common.errors.ProducerFencedException; import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Avg; @@ -29,7 +31,6 @@ import org.apache.kafka.common.utils.ThreadUtils; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; -import org.apache.kafka.connect.connector.Connector; import org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy; import org.apache.kafka.connect.errors.AlreadyExistsException; import org.apache.kafka.connect.errors.ConnectException; @@ -54,14 +55,21 @@ import org.apache.kafka.connect.runtime.rest.RestClient; import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo; import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo; +import org.apache.kafka.connect.runtime.rest.entities.ConnectorType; import org.apache.kafka.connect.runtime.rest.entities.TaskInfo; import org.apache.kafka.connect.runtime.rest.errors.BadRequestException; import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException; import org.apache.kafka.connect.sink.SinkConnector; +import org.apache.kafka.connect.source.ConnectorTransactionBoundaries; +import org.apache.kafka.connect.source.ExactlyOnceSupport; +import org.apache.kafka.connect.source.SourceConnector; +import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.storage.ConfigBackingStore; import org.apache.kafka.connect.storage.StatusBackingStore; import org.apache.kafka.connect.util.Callback; import org.apache.kafka.connect.util.ConnectorTaskId; +import org.apache.kafka.connect.util.FutureCallback; import org.apache.kafka.connect.util.SinkUtils; import org.slf4j.Logger; @@ -84,6 +92,7 @@ import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingDeque; @@ -138,6 +147,7 @@ public class DistributedHerder extends AbstractHerder implements Runnable { private static final long FORWARD_REQUEST_SHUTDOWN_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(10); private static final long START_AND_STOP_SHUTDOWN_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(1); private static final long RECONFIGURE_CONNECTOR_TASKS_BACKOFF_MS = 250; + private static final long CONFIG_TOPIC_WRITE_PRIVILEGES_BACKOFF_MS = 250; private static final int START_STOP_THREAD_POOL_SIZE = 8; private static final short BACKOFF_RETRIES = 5; @@ -156,8 +166,9 @@ public class DistributedHerder extends AbstractHerder implements Runnable { private final List keySignatureVerificationAlgorithms; private final KeyGenerator keyGenerator; + // Visible for testing + ExecutorService forwardRequestExecutor; private final ExecutorService herderExecutor; - private final ExecutorService forwardRequestExecutor; private final ExecutorService startAndStopExecutor; private final WorkerGroupMember member; private final AtomicBoolean stopping; @@ -184,7 +195,9 @@ public class DistributedHerder extends AbstractHerder implements Runnable { // Similarly collect target state changes (when observed by the config storage listener) for handling in the // herder's main thread. private Set connectorTargetStateChanges = new HashSet<>(); + private final Map activeZombieFencings = new HashMap<>(); private boolean needsReconfigRebalance; + private volatile boolean fencedFromConfigTopic; private volatile int generation; private volatile long scheduledRebalance; private volatile SecretKey sessionKey; @@ -196,6 +209,10 @@ public class DistributedHerder extends AbstractHerder implements Runnable { // The latest pending restart request for each named connector final Map pendingRestartRequests = new HashMap<>(); + // The thread that the herder's tick loop runs on. Would be final, but cannot be set in the constructor, + // and it's also useful to be able to modify it for testing + Thread herderThread; + private final DistributedConfig config; /** @@ -284,6 +301,7 @@ public DistributedHerder(DistributedConfig config, configState = ClusterConfigState.EMPTY; rebalanceResolved = true; // If we still need to follow up after a rebalance occurred, starting up tasks needsReconfigRebalance = false; + fencedFromConfigTopic = false; canReadConfigs = true; // We didn't try yet, but Configs are readable until proven otherwise scheduledRebalance = Long.MAX_VALUE; keyExpiration = Long.MAX_VALUE; @@ -316,6 +334,7 @@ public void start() { public void run() { try { log.info("Herder starting"); + herderThread = Thread.currentThread(); startServices(); @@ -368,18 +387,38 @@ public void tick() { return; } + if (fencedFromConfigTopic) { + if (isLeader()) { + // We were accidentally fenced out, possibly by a zombie leader + try { + log.debug("Reclaiming write privileges for config topic after being fenced out"); + configBackingStore.claimWritePrivileges(); + fencedFromConfigTopic = false; + log.debug("Successfully reclaimed write privileges for config topic after being fenced out"); + } catch (Exception e) { + log.warn("Unable to claim write privileges for config topic. Will backoff and possibly retry if still the leader", e); + backoff(CONFIG_TOPIC_WRITE_PRIVILEGES_BACKOFF_MS); + return; + } + } else { + log.trace("Relinquished write privileges for config topic after being fenced out, since worker is no longer the leader of the cluster"); + // We were meant to be fenced out because we fell out of the group and a new leader was elected + fencedFromConfigTopic = false; + } + } + long now = time.milliseconds(); if (checkForKeyRotation(now)) { log.debug("Distributing new session key"); keyExpiration = Long.MAX_VALUE; try { - configBackingStore.putSessionKey(new SessionKey( - keyGenerator.generateKey(), - now - )); + SessionKey newSessionKey = new SessionKey(keyGenerator.generateKey(), now); + if (!writeToConfigTopic(() -> configBackingStore.putSessionKey(newSessionKey))) { + return; + } } catch (Exception e) { - log.info("Failed to write new session key to config topic; forcing a read to the end of the config topic before possibly retrying"); + log.info("Failed to write new session key to config topic; forcing a read to the end of the config topic before possibly retrying", e); canReadConfigs = false; return; } @@ -404,12 +443,7 @@ public void tick() { break; } - try { - next.action().call(); - next.callback().onCompletion(null, null); - } catch (Throwable t) { - next.callback().onCompletion(t, null); - } + runRequest(next.action(), next.callback()); } // Process all pending connector restart requests @@ -488,6 +522,12 @@ private boolean checkForKeyRotation(long now) { SecretKey key; long expiration; synchronized (this) { + // This happens on startup; the snapshot contains the session key, + // but no callback in the config update listener has been fired for it yet. + if (sessionKey == null && configState.sessionKey() != null) { + sessionKey = configState.sessionKey().key(); + keyExpiration = configState.sessionKey().creationTimestamp() + keyRotationIntervalMs; + } key = sessionKey; expiration = keyExpiration; } @@ -507,10 +547,6 @@ private boolean checkForKeyRotation(long now) { + "than required by current worker configuration. Distributing new key now."); return true; } - } else if (key == null && configState.sessionKey() != null) { - // This happens on startup for follower workers; the snapshot contains the session key, - // but no callback in the config update listener has been fired for it yet. - sessionKey = configState.sessionKey().key(); } } return false; @@ -680,11 +716,25 @@ private void processTaskConfigUpdatesWithIncrementalCooperative(Set connectorsWhoseTasksToStop = taskConfigUpdates.stream() .map(ConnectorTaskId::connector).collect(Collectors.toSet()); + stopReconfiguredTasks(connectorsWhoseTasksToStop); + } + + private void stopReconfiguredTasks(Set connectors) { + Set localTasks = assignment == null + ? Collections.emptySet() + : new HashSet<>(assignment.tasks()); List tasksToStop = localTasks.stream() - .filter(taskId -> connectorsWhoseTasksToStop.contains(taskId.connector())) + .filter(taskId -> connectors.contains(taskId.connector())) .collect(Collectors.toList()); - log.info("Handling task config update by restarting tasks {}", tasksToStop); + + if (tasksToStop.isEmpty()) { + // The rest of the method would essentially be a no-op so this isn't strictly necessary, + // but it prevents an unnecessary log message from being emitted + return; + } + + log.info("Handling task config update by stopping tasks {}, which will be restarted after rebalance if still assigned to this worker", tasksToStop); worker.stopAndAwaitTasks(tasksToStop); tasksToRestart.addAll(tasksToStop); } @@ -832,7 +882,9 @@ public void deleteConnectorConfig(final String connName, final Callback configBackingStore.removeConnectorConfig(connName))) { + throw new ConnectException("Failed to remove connector configuration from config topic since worker was fenced out"); + } callback.onCompletion(null, new Created<>(false, null)); } return null; @@ -842,21 +894,132 @@ public void deleteConnectorConfig(final String connName, final Callback validateBasicConnectorConfig(Connector connector, - ConfigDef configDef, - Map config) { - Map validatedConfig = super.validateBasicConnectorConfig(connector, configDef, config); - if (connector instanceof SinkConnector) { - ConfigValue validatedName = validatedConfig.get(ConnectorConfig.NAME_CONFIG); - String name = (String) validatedName.value(); - if (workerGroupId.equals(SinkUtils.consumerGroupId(name))) { - validatedName.addErrorMessage("Consumer group for sink connector named " + name + - " conflicts with Connect worker group " + workerGroupId); + protected Map validateSinkConnectorConfig(SinkConnector connector, ConfigDef configDef, Map config) { + Map result = super.validateSinkConnectorConfig(connector, configDef, config); + validateSinkConnectorGroupId(result); + return result; + } + + @Override + protected Map validateSourceConnectorConfig(SourceConnector connector, ConfigDef configDef, Map config) { + Map result = super.validateSourceConnectorConfig(connector, configDef, config); + validateSourceConnectorExactlyOnceSupport(config, result, connector); + validateSourceConnectorTransactionBoundary(config, result, connector); + return result; + } + + + private void validateSinkConnectorGroupId(Map validatedConfig) { + ConfigValue validatedName = validatedConfig.get(ConnectorConfig.NAME_CONFIG); + String name = (String) validatedName.value(); + if (workerGroupId.equals(SinkUtils.consumerGroupId(name))) { + validatedName.addErrorMessage("Consumer group for sink connector named " + name + + " conflicts with Connect worker group " + workerGroupId); + } + } + + private void validateSourceConnectorExactlyOnceSupport( + Map rawConfig, + Map validatedConfig, + SourceConnector connector) { + ConfigValue validatedExactlyOnceSupport = validatedConfig.get(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG); + if (validatedExactlyOnceSupport.errorMessages().isEmpty()) { + // Should be safe to parse the enum from the user-provided value since it's passed validation so far + SourceConnectorConfig.ExactlyOnceSupportLevel exactlyOnceSupportLevel = + SourceConnectorConfig.ExactlyOnceSupportLevel.fromProperty(Objects.toString(validatedExactlyOnceSupport.value())); + if (SourceConnectorConfig.ExactlyOnceSupportLevel.REQUIRED.equals(exactlyOnceSupportLevel)) { + if (!config.exactlyOnceSourceEnabled()) { + validatedExactlyOnceSupport.addErrorMessage("This worker does not have exactly-once source support enabled."); + } + + try { + ExactlyOnceSupport exactlyOnceSupport = connector.exactlyOnceSupport(rawConfig); + if (!ExactlyOnceSupport.SUPPORTED.equals(exactlyOnceSupport)) { + final String validationErrorMessage; + // Would do a switch here but that doesn't permit matching on null values + if (exactlyOnceSupport == null) { + validationErrorMessage = "The connector does not implement the API required for preflight validation of exactly-once " + + "source support. Please consult the documentation for the connector to determine whether it supports exactly-once " + + "guarantees, and then consider reconfiguring the connector to use the value \"" + + SourceConnectorConfig.ExactlyOnceSupportLevel.REQUESTED + + "\" for this property (which will disable this preflight check and allow the connector to be created)."; + } else if (ExactlyOnceSupport.UNSUPPORTED.equals(exactlyOnceSupport)) { + validationErrorMessage = "The connector does not support exactly-once delivery guarantees with the provided configuration."; + } else { + throw new ConnectException("Unexpected value returned from SourceConnector::exactlyOnceSupport: " + exactlyOnceSupport); + } + validatedExactlyOnceSupport.addErrorMessage(validationErrorMessage); + } + } catch (Exception e) { + log.error("Failed while validating connector support for exactly-once guarantees", e); + String validationErrorMessage = "An unexpected error occurred during validation"; + String failureMessage = e.getMessage(); + if (failureMessage != null && !failureMessage.trim().isEmpty()) { + validationErrorMessage += ": " + failureMessage.trim(); + } else { + validationErrorMessage += "; please see the worker logs for more details."; + } + validatedExactlyOnceSupport.addErrorMessage(validationErrorMessage); + } + } + } + } + + private void validateSourceConnectorTransactionBoundary( + Map rawConfig, + Map validatedConfig, + SourceConnector connector) { + ConfigValue validatedTransactionBoundary = validatedConfig.get(SourceConnectorConfig.TRANSACTION_BOUNDARY_CONFIG); + if (validatedTransactionBoundary.errorMessages().isEmpty()) { + // Should be safe to parse the enum from the user-provided value since it's passed validation so far + SourceTask.TransactionBoundary transactionBoundary = + SourceTask.TransactionBoundary.fromProperty(Objects.toString(validatedTransactionBoundary.value())); + if (SourceTask.TransactionBoundary.CONNECTOR.equals(transactionBoundary)) { + try { + ConnectorTransactionBoundaries connectorTransactionSupport = connector.canDefineTransactionBoundaries(rawConfig); + if (!ConnectorTransactionBoundaries.SUPPORTED.equals(connectorTransactionSupport)) { + validatedTransactionBoundary.addErrorMessage( + "The connector does not support connector-defined transaction boundaries with the given configuration. " + + "Please reconfigure it to use a different transaction boundary definition."); + } + } catch (Exception e) { + log.error("Failed while validating connector support for defining its own transaction boundaries", e); + String validationErrorMessage = "An unexpected error occurred during validation"; + String failureMessage = e.getMessage(); + if (failureMessage != null && !failureMessage.trim().isEmpty()) { + validationErrorMessage += ": " + failureMessage.trim(); + } else { + validationErrorMessage += "; please see the worker logs for more details."; + } + validatedTransactionBoundary.addErrorMessage(validationErrorMessage); + } } } - return validatedConfig; } + @Override + protected boolean connectorUsesAdmin(org.apache.kafka.connect.health.ConnectorType connectorType, Map connProps) { + if (super.connectorUsesAdmin(connectorType, connProps)) { + return true; + } else if (connectorType == org.apache.kafka.connect.health.ConnectorType.SOURCE) { + return config.exactlyOnceSourceEnabled() + || !connProps.getOrDefault(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, "").trim().isEmpty(); + } else { + return false; + } + } + + @Override + protected boolean connectorUsesConsumer(org.apache.kafka.connect.health.ConnectorType connectorType, Map connProps) { + if (super.connectorUsesConsumer(connectorType, connProps)) { + return true; + } else if (connectorType == org.apache.kafka.connect.health.ConnectorType.SOURCE) { + return config.exactlyOnceSourceEnabled() + || !connProps.getOrDefault(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, "").trim().isEmpty(); + } else { + return false; + } + } @Override public void putConnectorConfig(final String connName, final Map config, final boolean allowReplace, @@ -891,13 +1054,15 @@ public void putConnectorConfig(final String connName, final Map } log.trace("Submitting connector config {} {} {}", connName, allowReplace, configState.connectors()); - configBackingStore.putConnectorConfig(connName, config); + if (!writeToConfigTopic(() -> configBackingStore.putConnectorConfig(connName, config))) { + throw new ConnectException("Failed to write connector config to config topic since worker was fenced out"); + } // Note that we use the updated connector config despite the fact that we don't have an updated // snapshot yet. The existing task info should still be accurate. ConnectorInfo info = new ConnectorInfo(connName, config, configState.tasks(connName), // validateConnectorConfig have checked the existence of CONNECTOR_CLASS_CONFIG - connectorTypeForClass(config.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG))); + connectorTypeForConfig(config)); callback.onCompletion(null, new Created<>(!exists, info)); return null; }, @@ -956,31 +1121,8 @@ public void taskConfigs(final String connName, final Callback> ca @Override public void putTaskConfigs(final String connName, final List> configs, final Callback callback, InternalRequestSignature requestSignature) { log.trace("Submitting put task configuration request {}", connName); - if (internalRequestValidationEnabled()) { - ConnectRestException requestValidationError = null; - if (requestSignature == null) { - requestValidationError = new BadRequestException("Internal request missing required signature"); - } else if (!keySignatureVerificationAlgorithms.contains(requestSignature.keyAlgorithm())) { - requestValidationError = new BadRequestException(String.format( - "This worker does not support the '%s' key signing algorithm used by other workers. " - + "This worker is currently configured to use: %s. " - + "Check that all workers' configuration files permit the same set of signature algorithms, " - + "and correct any misconfigured worker and restart it.", - requestSignature.keyAlgorithm(), - keySignatureVerificationAlgorithms - )); - } else { - if (!requestSignature.isValid(sessionKey)) { - requestValidationError = new ConnectRestException( - Response.Status.FORBIDDEN, - "Internal request contained invalid signature." - ); - } - } - if (requestValidationError != null) { - callback.onCompletion(requestValidationError, null); - return; - } + if (requestNotSignedProperly(requestSignature, callback)) { + return; } addRequest( @@ -990,7 +1132,7 @@ public void putTaskConfigs(final String connName, final List else if (!configState.contains(connName)) callback.onCompletion(new NotFoundException("Connector " + connName + " not found"), null); else { - configBackingStore.putTaskConfigs(connName, configs); + putTaskConfigs(connName, configs); callback.onCompletion(null, null); } return null; @@ -999,6 +1141,129 @@ else if (!configState.contains(connName)) ); } + private void putTaskConfigs(final String connName, final List> configs) { + if (!writeToConfigTopic(() -> configBackingStore.putTaskConfigs(connName, configs))) { + throw new ConnectException("Failed to write task configs to config topic since worker was fenced out"); + } + } + + // Another worker has forwarded a request to this worker (which it believes is the leader) to perform a round of zombie fencing + @Override + public void fenceZombies(final String connName, final Callback callback, InternalRequestSignature requestSignature) { + log.trace("Submitting zombie fencing request {}", connName); + if (requestNotSignedProperly(requestSignature, callback)) { + return; + } + + fenceZombies(connName, callback); + } + + // A task on this worker requires a round of zombie fencing + void fenceZombies(final ConnectorTaskId id, Callback callback) { + log.trace("Performing preflight zombie check for task {}", id); + fenceZombies(id.connector(), (error, ignored) -> { + if (error == null) { + callback.onCompletion(null, null); + return; + } + + if (error instanceof NotLeaderException) { + String forwardedUrl = ((NotLeaderException) error).forwardUrl() + "connectors/" + id.connector() + "/fence"; + log.trace("Forwarding zombie fencing request for connector {} to leader at {}", id.connector(), forwardedUrl); + forwardRequestExecutor.execute(() -> { + try { + RestClient.httpRequest(forwardedUrl, "PUT", null, null, null, config, sessionKey, requestSignatureAlgorithm); + callback.onCompletion(null, null); + } catch (Throwable t) { + callback.onCompletion(t, null); + } + }); + return; + } + + if (!(error instanceof ConnectException)) { + error = new ConnectException("Failed to perform zombie fencing", error); + } + callback.onCompletion(error, null); + }); + } + + // Visible for testing + void fenceZombies(final String connName, final Callback callback) { + addRequest( + () -> { + log.trace("Performing zombie fencing request {}", connName); + if (!isLeader()) + callback.onCompletion(new NotLeaderException("Only the leader may perform zombie fencing.", leaderUrl()), null); + else if (!configState.contains(connName)) + callback.onCompletion(new NotFoundException("Connector " + connName + " not found"), null); + else if (!isSourceConnector(connName)) + callback.onCompletion(new BadRequestException("Connector " + connName + " is not a source connector"), null); + else { + try { + configBackingStore.refresh(1, TimeUnit.MINUTES); + } catch (TimeoutException e) { + throw new ConnectException("Failed to read to end of config topic before performing zombie fencing", e); + } + + configState = configBackingStore.snapshot(); + int taskCount = configState.taskCount(connName); + Integer taskCountRecord = configState.taskCountRecord(connName); + + synchronized (DistributedHerder.this) { + // Check first to see if we have to do a fencing. The control flow is a little awkward here (why not stick this in + // an else block lower down?) but we can't synchronize around the body below since that may contain a synchronous + // write to the config topic. + if (configState.pendingFencing(connName) && taskCountRecord != null + && (taskCountRecord != 1 || taskCount != 1)) { + int taskGen = configState.taskConfigGeneration(connName); + ZombieFencing activeZombieFencing = activeZombieFencings.get(connName); + if (activeZombieFencing == null) { + activeZombieFencing = new ZombieFencing(connName, taskCountRecord, taskCount, taskGen); + activeZombieFencings.put(connName, activeZombieFencing); + // Immediately after the fencing and necessary followup work (i.e., writing the task count record to the config topic) + // is complete, remove this from the list of active fencings + activeZombieFencing.addCallback((ignored, error) -> { + synchronized (DistributedHerder.this) { + activeZombieFencings.remove(connName); + } + }); + } + activeZombieFencing.addCallback(callback); + return null; + } + } + + if (!configState.pendingFencing(connName)) { + // If the latest task count record for the connector is present after the latest set of task configs, there's no need to + // do any zombie fencing or write a new task count record to the config topic + log.debug("Skipping zombie fencing round for connector {} as all old task generations have already been fenced out", connName); + } else { + if (taskCountRecord == null) { + // If there is no task count record present for the connector, no transactional producers should have been brought up for it, + // so there's nothing to fence--but we do need to write a task count record now so that we know to fence those tasks if/when + // the connector is reconfigured + log.debug("Skipping zombie fencing round but writing task count record for connector {} " + + "as it is being brought up for the first time with exactly-once source support", connName); + } else { + // If the last generation of tasks only had one task, and the next generation only has one, then the new task will automatically + // fence out the older task if it's still running; no need to fence here, but again, we still need to write a task count record + log.debug("Skipping zombie fencing round but writing task count record for connector {} " + + "as both the most recent and the current generation of task configs only contain one task", connName); + } + if (!writeToConfigTopic(() -> configBackingStore.putTaskCountRecord(connName, taskCount))) { + throw new ConnectException("Failed to write connector task count record to config topic since worker was fenced out"); + } + } + callback.onCompletion(null, null); + return null; + } + return null; + }, + forwardErrorCallback(callback) + ); + } + @Override public void restartConnector(final String connName, final Callback callback) { restartConnector(0, connName, callback); @@ -1211,6 +1476,29 @@ private String leaderUrl() { return assignment.leaderUrl(); } + /** + * Perform an action that writes to the config topic, and if it fails because the leader has been fenced out, make note of that + * fact so that we can try to reclaim write ownership (if still the leader of the cluster) in a subsequent iteration of the tick loop. + * Note that it is not necessary to wrap every write to the config topic in this method, only the writes that should be performed + * exclusively by the leader. For example, {@link ConfigBackingStore#putTargetState(String, TargetState)} does not require this + * method, as it can be invoked by any worker in the cluster. + * @param write the action that writes to the config topic, such as {@link ConfigBackingStore#putSessionKey(SessionKey)} or + * {@link ConfigBackingStore#putConnectorConfig(String, Map)}. + * @return {@code true} if the write succeeded, and {@code false} if the write failed because the worker was fenced out. All other + * failures will be propagated as exceptions. + */ + private boolean writeToConfigTopic(Runnable write) { + try { + write.run(); + return true; + } catch (ProducerFencedException e) { + log.info("Worker was fenced out from writing to config topic; this is likely because a new leader has been elected in the cluster. " + + "Will rejoin group if necessary and, if still the leader, reclaim write privileges for the config topic by fencing out any active writers."); + fencedFromConfigTopic = true; + return false; + } + } + /** * Handle post-assignment operations, either trying to resolve issues that kept assignment from completing, getting * this node into sync and its work started. @@ -1418,14 +1706,58 @@ private static Collection assignmentDifference(Collection update, Coll private boolean startTask(ConnectorTaskId taskId) { log.info("Starting task {}", taskId); - return worker.startTask( - taskId, - configState, - configState.connectorConfig(taskId.connector()), - configState.taskConfig(taskId), - this, - configState.targetState(taskId.connector()) - ); + Map connProps = configState.connectorConfig(taskId.connector()); + switch (connectorTypeForConfig(connProps)) { + case SINK: + return worker.startSinkTask( + taskId, + configState, + connProps, + configState.taskConfig(taskId), + this, + configState.targetState(taskId.connector()) + ); + case SOURCE: + if (config.exactlyOnceSourceEnabled()) { + int taskGeneration = configState.taskConfigGeneration(taskId.connector()); + return worker.startExactlyOnceSourceTask( + taskId, + configState, + connProps, + configState.taskConfig(taskId), + this, + configState.targetState(taskId.connector()), + () -> { + FutureCallback preflightFencing = new FutureCallback<>(); + fenceZombies(taskId, preflightFencing); + try { + preflightFencing.get(); + } catch (InterruptedException e) { + throw new ConnectException("Interrupted while attempting to perform round of zombie fencing", e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof ConnectException) { + throw (ConnectException) cause; + } else { + throw new ConnectException("Failed to perform round of zombie fencing", cause); + } + } + }, + () -> verifyTaskGenerationAndOwnership(taskId, taskGeneration) + ); + } else { + return worker.startSourceTask( + taskId, + configState, + connProps, + configState.taskConfig(taskId), + this, + configState.targetState(taskId.connector()) + ); + } + default: + throw new ConnectException("Failed to start task " + taskId + " since it is not a recognizable type (source or sink)"); + } } private Callable getTaskStartingCallable(final ConnectorTaskId taskId) { @@ -1583,7 +1915,7 @@ private void reconfigureConnector(final String connName, final Callback cb if (changed) { List> rawTaskProps = reverseTransform(connName, configState, taskProps); if (isLeader()) { - configBackingStore.putTaskConfigs(connName, rawTaskProps); + putTaskConfigs(connName, rawTaskProps); cb.onCompletion(null, null); } else { // We cannot forward the request on the same thread because this reconfiguration can happen as a result of connector @@ -1618,6 +1950,53 @@ private void reconfigureConnector(final String connName, final Callback cb } } + private void verifyTaskGenerationAndOwnership(ConnectorTaskId id, int initialTaskGen) { + try { + log.debug("Reading to end of config topic to ensure it is still safe to bring up source task {} with exactly-once support", id); + configBackingStore.refresh(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + // The timeout is quite large. This should never happen + throw new ConnectException("Failed to read to end of config topic", e); + } + + FutureCallback verifyCallback = new FutureCallback<>(); + + addRequest( + () -> verifyTaskGenerationAndOwnership(id, initialTaskGen, verifyCallback), + forwardErrorCallback(verifyCallback) + ); + + try { + verifyCallback.get(); + } catch (InterruptedException e) { + throw new ConnectException("Interrupted while performing preflight check for task " + id, e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof ConnectException) { + throw (ConnectException) cause; + } else { + throw new ConnectException("Failed to perform preflight check for task " + id, cause); + } + } + } + + // Visible for testing + Void verifyTaskGenerationAndOwnership(ConnectorTaskId id, int initialTaskGen, Callback callback) { + Integer currentTaskGen = configBackingStore.snapshot().taskConfigGeneration(id.connector()); + if (!Objects.equals(initialTaskGen, currentTaskGen)) { + throw new ConnectException("Cannot start source task " + + id + " with exactly-once support as the connector has already generated a new set of task configs"); + } + + if (!assignment.tasks().contains(id)) { + throw new ConnectException("Cannot start source task " + + id + " as it has already been revoked from this worker"); + } + + callback.onCompletion(null, null); + return null; + } + private boolean checkRebalanceNeeded(Callback callback) { // Raise an error if we are expecting a rebalance to begin. This prevents us from forwarding requests // based on stale leadership or assignment information @@ -1628,6 +2007,23 @@ private boolean checkRebalanceNeeded(Callback callback) { return false; } + /** + * Execute the given action and subsequent callback immediately if the current thread is the herder's tick thread, + * or use them to create and store a {@link DistributedHerderRequest} on the request queue and return the resulting request + * if not. + * @param action the action that should be run on the herder's tick thread + * @param callback the callback that should be invoked once the action is complete + * @return a new {@link DistributedHerderRequest} if one has been created and added to the request queue, and {@code null} otherwise + */ + DistributedHerderRequest addOrRunRequest(Callable action, Callback callback) { + if (Thread.currentThread().equals(herderThread)) { + runRequest(action, callback); + return null; + } else { + return addRequest(action, callback); + } + } + DistributedHerderRequest addRequest(Callable action, Callback callback) { return addRequest(0, action, callback); } @@ -1640,6 +2036,15 @@ DistributedHerderRequest addRequest(long delayMs, Callable action, Callbac return req; } + private void runRequest(Callable action, Callback callback) { + try { + action.call(); + callback.onCompletion(null, null); + } catch (Throwable t) { + callback.onCompletion(t, null); + } + } + private boolean internalRequestValidationEnabled() { return internalRequestValidationEnabled(member.currentProtocolVersion()); } @@ -1692,7 +2097,7 @@ public void onTaskConfigUpdate(Collection tasks) { log.info("Tasks {} configs updated", tasks); // Stage the update and wake up the work thread. - // The set of tasks is recorder for incremental cooperative rebalancing, in which + // The set of tasks is recorded for incremental cooperative rebalancing, in which // tasks don't get restarted unless they are balanced between workers. // With eager rebalancing there's no need to record the set of tasks because task reconfigs // always need a rebalance to ensure offsets get committed. In eager rebalancing the @@ -1703,6 +2108,20 @@ public void onTaskConfigUpdate(Collection tasks) { needsReconfigRebalance = true; taskConfigUpdates.addAll(tasks); } + tasks.stream() + .map(ConnectorTaskId::connector) + .distinct() + .forEach(connName -> { + synchronized (this) { + ZombieFencing activeFencing = activeZombieFencings.get(connName); + if (activeFencing != null) { + activeFencing.completeExceptionally(new ConnectRestException( + Response.Status.CONFLICT.getStatusCode(), + "Failed to complete zombie fencing because a new set of task configs was generated" + )); + } + } + }); member.wakeup(); } @@ -1892,12 +2311,20 @@ public void onAssigned(ExtendedAssignment assignment, int generation) { herderMetrics.rebalanceStarted(time.milliseconds()); } - // Delete the statuses of all connectors and tasks removed prior to the start of this rebalance. This - // has to be done after the rebalance completes to avoid race conditions as the previous generation - // attempts to change the state to UNASSIGNED after tasks have been stopped. if (isLeader()) { + // Delete the statuses of all connectors and tasks removed prior to the start of this rebalance. This + // has to be done after the rebalance completes to avoid race conditions as the previous generation + // attempts to change the state to UNASSIGNED after tasks have been stopped. updateDeletedConnectorStatus(); updateDeletedTaskStatus(); + // As the leader, we're now allowed to write directly to the config topic for important things like + // connector configs, session keys, and task count records + try { + configBackingStore.claimWritePrivileges(); + } catch (Exception e) { + fencedFromConfigTopic = true; + log.error("Unable to claim write privileges for config topic after being elected leader during rebalance", e); + } } // We *must* interrupt any poll() call since this could occur when the poll starts, and we might then @@ -1965,6 +2392,122 @@ private void resetActiveTopics(Collection connectors, Collection callback) { + if (internalRequestValidationEnabled()) { + ConnectRestException requestValidationError = null; + if (requestSignature == null) { + requestValidationError = new BadRequestException("Internal request missing required signature"); + } else if (!keySignatureVerificationAlgorithms.contains(requestSignature.keyAlgorithm())) { + requestValidationError = new BadRequestException(String.format( + "This worker does not support the '%s' key signing algorithm used by other workers. " + + "This worker is currently configured to use: %s. " + + "Check that all workers' configuration files permit the same set of signature algorithms, " + + "and correct any misconfigured worker and restart it.", + requestSignature.keyAlgorithm(), + keySignatureVerificationAlgorithms + )); + } else { + if (!requestSignature.isValid(sessionKey)) { + requestValidationError = new ConnectRestException( + Response.Status.FORBIDDEN, + "Internal request contained invalid signature." + ); + } + } + if (requestValidationError != null) { + callback.onCompletion(requestValidationError, null); + return true; + } + } + + return false; + } + + /** + * Represents an active zombie fencing: that is, an in-progress attempt to invoke + * {@link Worker#fenceZombies(String, int, Map)} and then, if successful, write a new task count + * record to the config topic. + */ + class ZombieFencing { + private final String connName; + private final int tasksToRecord; + private final int taskGen; + private final FutureCallback fencingFollowup; + private final KafkaFuture fencingFuture; + + public ZombieFencing(String connName, int tasksToFence, int tasksToRecord, int taskGen) { + this.connName = connName; + this.tasksToRecord = tasksToRecord; + this.taskGen = taskGen; + this.fencingFollowup = new FutureCallback<>(); + this.fencingFuture = worker.fenceZombies(connName, tasksToFence, configState.connectorConfig(connName)).thenApply(ignored -> { + // This callback will be called on the same thread that invokes KafkaFuture::thenApply if + // the future is already completed. Since that thread is the herder tick thread, we don't need + // to perform follow-up logic through an additional herder request (and if we tried, it would lead + // to deadlock) + addOrRunRequest( + this::onZombieFencingSuccess, + fencingFollowup + ); + awaitFollowup(); + return null; + }); + } + + // Invoked after the worker has successfully fenced out the producers of old task generations using an admin client + // Note that work here will be performed on the herder's tick thread, so it should not block for very long + private Void onZombieFencingSuccess() throws TimeoutException { + configBackingStore.refresh(1, TimeUnit.MINUTES); + configState = configBackingStore.snapshot(); + if (taskGen < configState.taskConfigGeneration(connName)) { + throw new ConnectRestException( + Response.Status.CONFLICT.getStatusCode(), + "Fencing failed because new task configurations were generated for the connector"); + } else { + if (!writeToConfigTopic(() -> configBackingStore.putTaskCountRecord(connName, tasksToRecord))) { + throw new ConnectException("Failed to write connector task count record to config topic since worker was fenced out"); + } + } + return null; + } + + private void awaitFollowup() { + try { + fencingFollowup.get(); + } catch (InterruptedException e) { + throw new ConnectException("Interrupted while performing zombie fencing", e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof ConnectException) { + throw (ConnectException) cause; + } else { + throw new ConnectException("Failed to perform round of zombie fencing", cause); + } + } + } + + public void completeExceptionally(Throwable t) { + fencingFollowup.onCompletion(t, null); + } + + public void addCallback(Callback callback) { + fencingFuture.whenComplete((ignored, error) -> { + if (error != null && !(error instanceof ConnectException)) { + callback.onCompletion( + new ConnectException("Failed to perform zombie fencing", error), + null + ); + } else { + callback.onCompletion(error, null); + } + }); + } + } + class HerderMetrics { private final MetricGroup metricGroup; private final Sensor rebalanceCompletedCounts; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/EagerAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/EagerAssignor.java index b6dbd09147805..4620e3d218bd6 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/EagerAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/EagerAssignor.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.utils.CircularIterator; import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.util.ConnectorTaskId; import org.slf4j.Logger; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index f7fa55d8b3e9a..17c3b8af41eb2 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.utils.Time; import org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.ConnectorsAndTasks; import org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.WorkerLoad; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.util.ConnectorTaskId; import org.slf4j.Logger; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java index e407a30decf98..bafcb3fcf1b88 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java @@ -25,6 +25,7 @@ import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.storage.ConfigBackingStore; import org.apache.kafka.connect.util.ConnectorTaskId; import org.slf4j.Logger; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerRebalanceListener.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerRebalanceListener.java index 93d03272813b6..416800cb7cd32 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerRebalanceListener.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerRebalanceListener.java @@ -31,7 +31,7 @@ public interface WorkerRebalanceListener { void onAssigned(ExtendedAssignment assignment, int generation); /** - * Invoked when a rebalance operation starts, revoking ownership for the set of connectors + * Invoked when a rebalance operation starts or completes, revoking ownership for the set of connectors * and tasks. Depending on the Connect protocol version, the collection of revoked connectors * or tasks might refer to all or some of the connectors and tasks running on the worker. */ diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/RetryWithToleranceOperator.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/RetryWithToleranceOperator.java index 0e15ced99b922..46aa085ebdd4d 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/RetryWithToleranceOperator.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/errors/RetryWithToleranceOperator.java @@ -114,7 +114,6 @@ public synchronized Future executeFailed(Stage stage, Class executingCl public synchronized Future executeFailed(Stage stage, Class executingClass, SourceRecord sourceRecord, Throwable error) { - markAsFailed(); context.sourceRecord(sourceRecord); context.currentContext(stage, executingClass); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java index 81c5a846998b1..88d8e39015427 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestClient.java @@ -112,14 +112,15 @@ public static HttpResponse httpRequest(String url, String method, HttpHea if (serializedBody != null) { req.content(new StringContentProvider(serializedBody, StandardCharsets.UTF_8), "application/json"); - if (sessionKey != null && requestSignatureAlgorithm != null) { - InternalRequestSignature.addToRequest( - sessionKey, - serializedBody.getBytes(StandardCharsets.UTF_8), - requestSignatureAlgorithm, - req - ); - } + } + + if (sessionKey != null && requestSignatureAlgorithm != null) { + InternalRequestSignature.addToRequest( + sessionKey, + serializedBody != null ? serializedBody.getBytes(StandardCharsets.UTF_8) : null, + requestSignatureAlgorithm, + req + ); } ContentResponse res = req.send(); @@ -132,7 +133,10 @@ public static HttpResponse httpRequest(String url, String method, HttpHea ErrorMessage errorMessage = JSON_SERDE.readValue(res.getContentAsString(), ErrorMessage.class); throw new ConnectRestException(responseCode, errorMessage.errorCode(), errorMessage.message()); } else if (responseCode >= 200 && responseCode < 300) { - T result = JSON_SERDE.readValue(res.getContentAsString(), responseFormat); + String responseBody = res.getContentAsString(); + T result = responseFormat != null + ? JSON_SERDE.readValue(responseBody, responseFormat) + : null; return new HttpResponse<>(responseCode, convertHttpFieldsToMap(res.getHeaders()), result); } else { throw new ConnectRestException(Response.Status.INTERNAL_SERVER_ERROR, diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java index 18b10c9fad897..4887fc376e8b1 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java @@ -320,6 +320,18 @@ public void putTaskConfigs(final @PathParam("connector") String connector, completeOrForwardRequest(cb, "/connectors/" + connector + "/tasks", "POST", headers, taskConfigs, forward); } + @PUT + @Path("/{connector}/fence") + public Response fenceZombies(final @PathParam("connector") String connector, + final @Context HttpHeaders headers, + final @QueryParam("forward") Boolean forward, + final byte[] requestBody) throws Throwable { + FutureCallback cb = new FutureCallback<>(); + herder.fenceZombies(connector, cb, InternalRequestSignature.fromHeaders(requestBody, headers)); + completeOrForwardRequest(cb, "/connectors/" + connector + "/fence", "PUT", headers, requestBody, forward); + return Response.ok().build(); + } + @GET @Path("/{connector}/tasks/{task}/status") public ConnectorStateInfo.TaskState getTaskStatus(final @PathParam("connector") String connector, diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java index dac389ba0e346..ff3eb7a8b8a0a 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java @@ -31,7 +31,7 @@ import org.apache.kafka.connect.runtime.SourceConnectorConfig; import org.apache.kafka.connect.runtime.TargetState; import org.apache.kafka.connect.runtime.Worker; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.runtime.rest.InternalRequestSignature; import org.apache.kafka.connect.runtime.rest.entities.ConfigInfos; import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo; @@ -265,6 +265,11 @@ public void putTaskConfigs(String connName, List> configs, C throw new UnsupportedOperationException("Kafka Connect in standalone mode does not support externally setting task configurations."); } + @Override + public void fenceZombies(String connName, Callback callback, InternalRequestSignature requestSignature) { + throw new UnsupportedOperationException("Kafka Connect in standalone mode does not support exactly-once source connectors."); + } + @Override public synchronized void restartTask(ConnectorTaskId taskId, Callback cb) { if (!configState.contains(taskId.connector())) @@ -275,9 +280,8 @@ public synchronized void restartTask(ConnectorTaskId taskId, Callback cb) cb.onCompletion(new NotFoundException("Task " + taskId + " not found", null), null); Map connConfigProps = configState.connectorConfig(taskId.connector()); - TargetState targetState = configState.targetState(taskId.connector()); worker.stopAndAwaitTask(taskId); - if (worker.startTask(taskId, configState, connConfigProps, taskConfigProps, this, targetState)) + if (startTask(taskId, connConfigProps)) cb.onCompletion(null, null); else cb.onCompletion(new ConnectException("Failed to start task: " + taskId), null); @@ -372,12 +376,36 @@ private void createConnectorTasks(String connName) { } private void createConnectorTasks(String connName, Collection taskIds) { - TargetState initialState = configState.targetState(connName); Map connConfigs = configState.connectorConfig(connName); for (ConnectorTaskId taskId : taskIds) { - Map taskConfigMap = configState.taskConfig(taskId); - worker.startTask(taskId, configState, connConfigs, taskConfigMap, this, initialState); + startTask(taskId, connConfigs); + } + } + + private boolean startTask(ConnectorTaskId taskId, Map connProps) { + switch (connectorTypeForClass(connProps.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG))) { + case SINK: + return worker.startSinkTask( + taskId, + configState, + connProps, + configState.taskConfig(taskId), + this, + configState.targetState(taskId.connector()) + ); + case SOURCE: + return worker.startSourceTask( + taskId, + configState, + connProps, + configState.taskConfig(taskId), + this, + configState.targetState(taskId.connector()) + ); + default: + throw new ConnectException("Failed to start task " + taskId + " since it is not a recognizable type (source or sink)"); } + } private void removeConnectorTasks(String connName) { diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/CloseableOffsetStorageReader.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/CloseableOffsetStorageReader.java index b90273936bfb0..05187ff109bdd 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/CloseableOffsetStorageReader.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/CloseableOffsetStorageReader.java @@ -27,7 +27,9 @@ public interface CloseableOffsetStorageReader extends Closeable, OffsetStorageRe * {@link Future#cancel(boolean) Cancel} all outstanding offset read requests, and throw an * exception in all current and future calls to {@link #offsets(Collection)} and * {@link #offset(Map)}. This is useful for unblocking task threads which need to shut down but - * are blocked on offset reads. + * are blocked on offset reads. Additionally, release any resources allocated specifically for + * tracking this connector's offsets. This method should be idempotent; after it is successfully + * invoked for the first time, successive invocations should have no effect. */ void close(); } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ClusterConfigState.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ClusterConfigState.java similarity index 81% rename from connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ClusterConfigState.java rename to connect/runtime/src/main/java/org/apache/kafka/connect/storage/ClusterConfigState.java index 717120d8508ee..5627cb75c5a23 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/ClusterConfigState.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ClusterConfigState.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.connect.runtime.distributed; +package org.apache.kafka.connect.storage; import org.apache.kafka.common.config.provider.ConfigProvider; import org.apache.kafka.connect.runtime.SessionKey; @@ -42,16 +42,22 @@ public class ClusterConfigState { Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptySet(), Collections.emptySet()); private final long offset; private final SessionKey sessionKey; - private final Map connectorTaskCounts; - private final Map> connectorConfigs; - private final Map connectorTargetStates; - private final Map> taskConfigs; - private final Set inconsistentConnectors; private final WorkerConfigTransformer configTransformer; + final Map connectorTaskCounts; + final Map> connectorConfigs; + final Map connectorTargetStates; + final Map> taskConfigs; + final Map connectorTaskCountRecords; + final Map connectorTaskConfigGenerations; + final Set connectorsPendingFencing; + final Set inconsistentConnectors; public ClusterConfigState(long offset, SessionKey sessionKey, @@ -59,6 +65,9 @@ public ClusterConfigState(long offset, Map> connectorConfigs, Map connectorTargetStates, Map> taskConfigs, + Map connectorTaskCountRecords, + Map connectorTaskConfigGenerations, + Set connectorsPendingFencing, Set inconsistentConnectors) { this(offset, sessionKey, @@ -66,6 +75,9 @@ public ClusterConfigState(long offset, connectorConfigs, connectorTargetStates, taskConfigs, + connectorTaskCountRecords, + connectorTaskConfigGenerations, + connectorsPendingFencing, inconsistentConnectors, null); } @@ -76,6 +88,9 @@ public ClusterConfigState(long offset, Map> connectorConfigs, Map connectorTargetStates, Map> taskConfigs, + Map connectorTaskCountRecords, + Map connectorTaskConfigGenerations, + Set connectorsPendingFencing, Set inconsistentConnectors, WorkerConfigTransformer configTransformer) { this.offset = offset; @@ -84,6 +99,9 @@ public ClusterConfigState(long offset, this.connectorConfigs = connectorConfigs; this.connectorTargetStates = connectorTargetStates; this.taskConfigs = taskConfigs; + this.connectorTaskCountRecords = connectorTaskCountRecords; + this.connectorTaskConfigGenerations = connectorTaskConfigGenerations; + this.connectorsPendingFencing = connectorsPendingFencing; this.inconsistentConnectors = inconsistentConnectors; this.configTransformer = configTransformer; } @@ -202,6 +220,15 @@ public int taskCount(String connectorName) { return count == null ? 0 : count; } + /** + * Get whether the connector requires a round of zombie fencing before + * a new generation of tasks can be brought up for it. + * @param connectorName name of the connector + */ + public boolean pendingFencing(String connectorName) { + return connectorsPendingFencing.contains(connectorName); + } + /** * Get the current set of task IDs for the specified connector. * @param connectorName the name of the connector to look up task configs for @@ -225,6 +252,25 @@ public List tasks(String connectorName) { return Collections.unmodifiableList(taskIds); } + /** + * Get the task count record for the connector, if one exists + * @param connector name of the connector + * @return the latest task count record for the connector, or {@code null} if none exists + */ + public Integer taskCountRecord(String connector) { + return connectorTaskCountRecords.get(connector); + } + + /** + * Get the generation number for the connector's task configurations, if one exists. + * Generation numbers increase monotonically each time a new set of task configurations is detected for the connector + * @param connector name of the connector + * @return the latest task config generation number for the connector, or {@code null} if none exists + */ + public Integer taskConfigGeneration(String connector) { + return connectorTaskConfigGenerations.get(connector); + } + /** * Get the set of connectors which have inconsistent data in this snapshot. These inconsistencies can occur due to * partially completed writes combined with log compaction. @@ -280,4 +326,4 @@ public int hashCode() { inconsistentConnectors, configTransformer); } -} +} \ No newline at end of file diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConfigBackingStore.java index 826f934ffd550..0965223157122 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConfigBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConfigBackingStore.java @@ -19,7 +19,6 @@ import org.apache.kafka.connect.runtime.RestartRequest; import org.apache.kafka.connect.runtime.SessionKey; import org.apache.kafka.connect.runtime.TargetState; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; import org.apache.kafka.connect.util.ConnectorTaskId; import java.util.Collection; @@ -90,6 +89,10 @@ public interface ConfigBackingStore { */ void putTargetState(String connector, TargetState state); + /** + * Store a new {@link SessionKey} that can be used to validate internal (i.e., non-user-triggered) inter-worker communication. + * @param sessionKey the session key to store + */ void putSessionKey(SessionKey sessionKey); /** @@ -98,6 +101,22 @@ public interface ConfigBackingStore { */ void putRestartRequest(RestartRequest restartRequest); + /** + * Record the number of tasks for the connector after a successful round of zombie fencing. + * @param connector name of the connector + * @param taskCount number of tasks used by the connector + */ + void putTaskCountRecord(String connector, int taskCount); + + /** + * Prepare to write to the backing config store. May be required by some implementations (such as those that only permit a single + * writer at a time across a cluster of workers) before performing mutative operations such as writing configurations, target states, etc. + * The default implementation is a no-op; it is the responsibility of the implementing class to override this and document any expectations for + * when it must be invoked. + */ + default void claimWritePrivileges() { + } + /** * Set an update listener to get notifications when there are config/target state * changes. diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConnectorOffsetBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConnectorOffsetBackingStore.java new file mode 100644 index 0000000000000..ca5b917b3b2f9 --- /dev/null +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConnectorOffsetBackingStore.java @@ -0,0 +1,337 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.storage; + +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.connect.runtime.WorkerConfig; +import org.apache.kafka.connect.util.Callback; +import org.apache.kafka.connect.util.LoggingContext; +import org.apache.kafka.connect.util.TopicAdmin; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; + +/** + * An {@link OffsetBackingStore} with support for reading from and writing to a worker-global + * offset backing store and/or a connector-specific offset backing store. + */ +public class ConnectorOffsetBackingStore implements OffsetBackingStore { + + private static final Logger log = LoggerFactory.getLogger(ConnectorOffsetBackingStore.class); + + /** + * Builds an offset store that uses a connector-specific offset topic as the primary store and + * the worker-global offset store as the secondary store. + * + * @param loggingContext a {@link Supplier} for the {@link LoggingContext} that should be used + * for messages logged by this offset store; may not be null, and may never return null + * @param workerStore the worker-global offset store; may not be null + * @param connectorStore the connector-specific offset store; may not be null + * @param connectorOffsetsTopic the name of the connector-specific offset topic; may not be null + * @param connectorStoreAdmin the topic admin to use for the connector-specific offset topic; may not be null + * @return an offset store backed primarily by the connector-specific offset topic and secondarily + * by the worker-global offset store; never null + */ + public static ConnectorOffsetBackingStore withConnectorAndWorkerStores( + Supplier loggingContext, + OffsetBackingStore workerStore, + KafkaOffsetBackingStore connectorStore, + String connectorOffsetsTopic, + TopicAdmin connectorStoreAdmin + ) { + Objects.requireNonNull(loggingContext); + Objects.requireNonNull(workerStore); + Objects.requireNonNull(connectorStore); + Objects.requireNonNull(connectorOffsetsTopic); + Objects.requireNonNull(connectorStoreAdmin); + return new ConnectorOffsetBackingStore( + Time.SYSTEM, + loggingContext, + connectorOffsetsTopic, + workerStore, + connectorStore, + connectorStoreAdmin + ); + } + + /** + * Builds an offset store that uses the worker-global offset store as the primary store, and no secondary store. + * + * @param loggingContext a {@link Supplier} for the {@link LoggingContext} that should be used + * for messages logged by this offset store; may not be null, and may never return null + * @param workerStore the worker-global offset store; may not be null + * @param workerOffsetsTopic the name of the worker-global offset topic; may be null if the worker + * does not use an offset topic for its offset store + * @return an offset store for the connector backed solely by the worker-global offset store; never null + */ + public static ConnectorOffsetBackingStore withOnlyWorkerStore( + Supplier loggingContext, + OffsetBackingStore workerStore, + String workerOffsetsTopic + ) { + Objects.requireNonNull(loggingContext); + Objects.requireNonNull(workerStore); + return new ConnectorOffsetBackingStore(Time.SYSTEM, loggingContext, workerOffsetsTopic, workerStore, null, null); + } + + /** + * Builds an offset store that uses a connector-specific offset topic as the primary store, and no secondary store. + * + * @param loggingContext a {@link Supplier} for the {@link LoggingContext} that should be used + * for messages logged by this offset store; may not be null, and may never return null + * @param connectorStore the connector-specific offset store; may not be null + * @param connectorOffsetsTopic the name of the connector-specific offset topic; may not be null + * @param connectorStoreAdmin the topic admin to use for the connector-specific offset topic; may not be null + * @return an offset store for the connector backed solely by the connector-specific offset topic; never null + */ + public static ConnectorOffsetBackingStore withOnlyConnectorStore( + Supplier loggingContext, + KafkaOffsetBackingStore connectorStore, + String connectorOffsetsTopic, + TopicAdmin connectorStoreAdmin + ) { + Objects.requireNonNull(loggingContext); + Objects.requireNonNull(connectorOffsetsTopic); + Objects.requireNonNull(connectorStoreAdmin); + return new ConnectorOffsetBackingStore( + Time.SYSTEM, + loggingContext, + connectorOffsetsTopic, + null, + connectorStore, + connectorStoreAdmin + ); + } + + private final Time time; + private final Supplier loggingContext; + private final String primaryOffsetsTopic; + private final Optional workerStore; + private final Optional connectorStore; + private final Optional connectorStoreAdmin; + + ConnectorOffsetBackingStore( + Time time, + Supplier loggingContext, + String primaryOffsetsTopic, + OffsetBackingStore workerStore, + KafkaOffsetBackingStore connectorStore, + TopicAdmin connectorStoreAdmin + ) { + if (workerStore == null && connectorStore == null) { + throw new IllegalArgumentException("At least one non-null offset store must be provided"); + } + this.time = time; + this.loggingContext = loggingContext; + this.primaryOffsetsTopic = primaryOffsetsTopic; + this.workerStore = Optional.ofNullable(workerStore); + this.connectorStore = Optional.ofNullable(connectorStore); + this.connectorStoreAdmin = Optional.ofNullable(connectorStoreAdmin); + } + + public String primaryOffsetsTopic() { + return primaryOffsetsTopic; + } + + /** + * If configured to use a connector-specific offset store, {@link OffsetBackingStore#start() start} that store. + * + *

The worker-global offset store is not modified; it is the caller's responsibility to ensure that it is started + * before calls to {@link #get(Collection)} and {@link #set(Map, Callback)} take place. + */ + @Override + public void start() { + // Worker offset store should already be started + connectorStore.ifPresent(OffsetBackingStore::start); + } + + /** + * If configured to use a connector-specific offset store, {@link OffsetBackingStore#start() stop} that store, + * and the {@link TopicAdmin} used by that store. + * + *

The worker-global offset store is not modified as it may be used for other connectors that either already exist, + * or will be created, on this worker. + */ + @Override + public void stop() { + // Worker offset store should not be stopped as it may be used for multiple connectors + connectorStore.ifPresent(OffsetBackingStore::stop); + connectorStoreAdmin.ifPresent(TopicAdmin::close); + } + + /** + * Get the offset values for the specified keys. + * + *

If configured to use a connector-specific offset store, priority is given to the values contained in that store, + * and the values in the worker-global offset store (if one is provided) are used as a fallback for keys that are not + * present in the connector-specific store. + * + *

If not configured to use a connector-specific offset store, only the values contained in the worker-global + * offset store are returned. + + * @param keys list of keys to look up + * @return future for the resulting map from key to value + */ + @Override + public Future> get(Collection keys) { + Future> workerGetFuture = getFromStore(workerStore, keys); + Future> connectorGetFuture = getFromStore(connectorStore, keys); + + return new Future>() { + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return workerGetFuture.cancel(mayInterruptIfRunning) + || connectorGetFuture.cancel(mayInterruptIfRunning); + } + + @Override + public boolean isCancelled() { + return workerGetFuture.isCancelled() + || connectorGetFuture.isCancelled(); + } + + @Override + public boolean isDone() { + return workerGetFuture.isDone() + && connectorGetFuture.isDone(); + } + + @Override + public Map get() throws InterruptedException, ExecutionException { + Map result = new HashMap<>(workerGetFuture.get()); + result.putAll(connectorGetFuture.get()); + return result; + } + + @Override + public Map get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + long endTime = time.milliseconds() + unit.toMillis(timeout); + Map result = new HashMap<>(workerGetFuture.get(timeout, unit)); + timeout = Math.max(1, endTime - time.milliseconds()); + result.putAll(connectorGetFuture.get(timeout, TimeUnit.MILLISECONDS)); + return result; + } + }; + } + + /** + * Store the specified offset key/value pairs. + * + *

If configured to use a connector-specific offset store, the returned {@link Future} corresponds to a + * write to that store, and the passed-in {@link Callback} is invoked once that write completes. If a worker-global + * store is provided, a secondary write is made to that store if the write to the connector-specific store + * succeeds. Errors with this secondary write are not reflected in the returned {@link Future} or the passed-in + * {@link Callback}; they are only logged as a warning to users. + * + *

If not configured to use a connector-specific offset store, the returned {@link Future} corresponds to a + * write to the worker-global offset store, and the passed-in {@link Callback} is invoked once that write completes. + + * @param values map from key to value + * @param callback callback to invoke on completion of the primary write + * @return void future for the primary write + */ + @Override + public Future set(Map values, Callback callback) { + final OffsetBackingStore primaryStore; + final OffsetBackingStore secondaryStore; + if (connectorStore.isPresent()) { + primaryStore = connectorStore.get(); + secondaryStore = workerStore.orElse(null); + } else if (workerStore.isPresent()) { + primaryStore = workerStore.get(); + secondaryStore = null; + } else { + // Should never happen since we check for this case in the constructor, but just in case, this should + // be more informative than the NPE that would otherwise be thrown + throw new IllegalStateException("At least one non-null offset store must be provided"); + } + + return primaryStore.set(values, (primaryWriteError, ignored) -> { + if (secondaryStore != null) { + if (primaryWriteError != null) { + log.trace("Skipping offsets write to secondary store because primary write has failed", primaryWriteError); + } else { + try { + // Invoke OffsetBackingStore::set but ignore the resulting future; we don't block on writes to this + // backing store. + secondaryStore.set(values, (secondaryWriteError, ignored2) -> { + try (LoggingContext context = loggingContext()) { + if (secondaryWriteError != null) { + log.warn("Failed to write offsets to secondary backing store", secondaryWriteError); + } else { + log.debug("Successfully flushed offsets to secondary backing store"); + } + } + }); + } catch (Exception e) { + log.warn("Failed to write offsets to secondary backing store", e); + } + } + } + try (LoggingContext context = loggingContext()) { + callback.onCompletion(primaryWriteError, ignored); + } + }); + } + + /** + * If configured to use a connector-specific offset store, + * {@link OffsetBackingStore#configure(WorkerConfig) configure} that store. + * + *

The worker-global offset store is not modified; it is the caller's responsibility to ensure that it is configured + * before calls to {@link #start()}, {@link #get(Collection)} and {@link #set(Map, Callback)} take place. + */ + @Override + public void configure(WorkerConfig config) { + // Worker offset store should already be configured + connectorStore.ifPresent(store -> store.configure(config)); + } + + // For testing + public boolean hasConnectorSpecificStore() { + return connectorStore.isPresent(); + } + + // For testing + public boolean hasWorkerGlobalStore() { + return workerStore.isPresent(); + } + + private LoggingContext loggingContext() { + LoggingContext result = loggingContext.get(); + Objects.requireNonNull(result); + return result; + } + + private static Future> getFromStore(Optional store, Collection keys) { + return store.map(s -> s.get(keys)).orElseGet(() -> CompletableFuture.completedFuture(Collections.emptyMap())); + } + +} diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java index e77bcfa5018c2..2be9704a8d9a2 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java @@ -19,9 +19,15 @@ import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.config.TopicConfig; +import org.apache.kafka.common.errors.ProducerFencedException; +import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.serialization.ByteArraySerializer; import org.apache.kafka.common.serialization.StringDeserializer; @@ -39,7 +45,6 @@ import org.apache.kafka.connect.runtime.TargetState; import org.apache.kafka.connect.runtime.WorkerConfig; import org.apache.kafka.connect.runtime.WorkerConfigTransformer; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; import org.apache.kafka.connect.runtime.distributed.DistributedConfig; import org.apache.kafka.connect.util.Callback; import org.apache.kafka.connect.util.ConnectUtils; @@ -51,6 +56,7 @@ import org.slf4j.LoggerFactory; import javax.crypto.spec.SecretKeySpec; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; @@ -58,6 +64,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -185,6 +192,12 @@ public static String COMMIT_TASKS_KEY(String connectorName) { return COMMIT_TASKS_PREFIX + connectorName; } + public static final String TASK_COUNT_RECORD_PREFIX = "tasks-count-"; + + public static String TASK_COUNT_RECORD_KEY(String connectorName) { + return TASK_COUNT_RECORD_PREFIX + connectorName; + } + public static final String SESSION_KEY_KEY = "session-key"; // Note that while using real serialization for values as we have here, but ad hoc string serialization for keys, @@ -201,6 +214,9 @@ public static String COMMIT_TASKS_KEY(String connectorName) { public static final Schema TARGET_STATE_V0 = SchemaBuilder.struct() .field("state", Schema.STRING_SCHEMA) .build(); + public static final Schema TASK_COUNT_RECORD_V0 = SchemaBuilder.struct() + .field("tasks", Schema.INT32_SCHEMA) + .build(); // The key is logically a byte array, but we can't use the JSON converter to (de-)serialize that without a schema. // So instead, we base 64-encode it before serializing and decode it after deserializing. public static final Schema SESSION_KEY_V0 = SchemaBuilder.struct() @@ -231,23 +247,25 @@ public static String RESTART_KEY(String connectorName) { private volatile boolean started; // Although updateListener is not final, it's guaranteed to be visible to any thread after its // initialization as long as we always read the volatile variable "started" before we access the listener. - private UpdateListener updateListener; + private ConfigBackingStore.UpdateListener updateListener; + + private final Map baseProducerProps; private final String topic; // Data is passed to the log already serialized. We use a converter to handle translating to/from generic Connect // format to serialized form private final KafkaBasedLog configLog; // Connector -> # of tasks - private final Map connectorTaskCounts = new HashMap<>(); + final Map connectorTaskCounts = new HashMap<>(); // Connector and task configs: name or id -> config map - private final Map> connectorConfigs = new HashMap<>(); - private final Map> taskConfigs = new HashMap<>(); + final Map> connectorConfigs = new HashMap<>(); + final Map> taskConfigs = new HashMap<>(); private final Supplier topicAdminSupplier; private SharedTopicAdmin ownTopicAdmin; // Set of connectors where we saw a task commit with an incomplete set of task config updates, indicating the data // is in an inconsistent state and we cannot safely use them until they have been refreshed. - private final Set inconsistent = new HashSet<>(); + final Set inconsistent = new HashSet<>(); // The most recently read offset. This does not take into account deferred task updates/commits, so we may have // outstanding data to be applied. private volatile long offset; @@ -257,22 +275,34 @@ public static String RESTART_KEY(String connectorName) { // Connector -> Map[ConnectorTaskId -> Configs] private final Map>> deferredTaskUpdates = new HashMap<>(); - private final Map connectorTargetStates = new HashMap<>(); + final Map connectorTargetStates = new HashMap<>(); + + final Map connectorTaskCountRecords = new HashMap<>(); + final Map connectorTaskConfigGenerations = new HashMap<>(); + final Set connectorsPendingFencing = new HashSet<>(); private final WorkerConfigTransformer configTransformer; + private final boolean usesFencableWriter; + private volatile Producer fencableProducer; + private final Map fencableProducerProps; + @Deprecated - public KafkaConfigBackingStore(Converter converter, WorkerConfig config, WorkerConfigTransformer configTransformer) { + public KafkaConfigBackingStore(Converter converter, DistributedConfig config, WorkerConfigTransformer configTransformer) { this(converter, config, configTransformer, null); } - public KafkaConfigBackingStore(Converter converter, WorkerConfig config, WorkerConfigTransformer configTransformer, Supplier adminSupplier) { + public KafkaConfigBackingStore(Converter converter, DistributedConfig config, WorkerConfigTransformer configTransformer, Supplier adminSupplier) { this.lock = new Object(); this.started = false; this.converter = converter; this.offset = -1; this.topicAdminSupplier = adminSupplier; + this.baseProducerProps = baseProducerProps(config); + this.fencableProducerProps = fencableProducerProps(config); + + this.usesFencableWriter = config.transactionalLeaderEnabled(); this.topic = config.getString(DistributedConfig.CONFIG_TOPIC_CONFIG); if (this.topic == null || this.topic.trim().length() == 0) throw new ConfigException("Must specify topic for connector configuration."); @@ -282,7 +312,7 @@ public KafkaConfigBackingStore(Converter converter, WorkerConfig config, WorkerC } @Override - public void setUpdateListener(UpdateListener listener) { + public void setUpdateListener(ConfigBackingStore.UpdateListener listener) { this.updateListener = listener; } @@ -291,7 +321,16 @@ public void start() { log.info("Starting KafkaConfigBackingStore"); // Before startup, callbacks are *not* invoked. You can grab a snapshot after starting -- just take care that // updates can continue to occur in the background - configLog.start(); + try { + configLog.start(); + } catch (UnsupportedVersionException e) { + throw new ConnectException( + "Enabling exactly-once support for source connectors requires a Kafka broker version that allows " + + "admin clients to read consumer offsets. Disable the worker's exactly-once support " + + "for source connectors, or user a new Kafka broker version.", + e + ); + } int partitionCount = configLog.partitionCount(); if (partitionCount > 1) { @@ -309,16 +348,70 @@ public void start() { @Override public void stop() { log.info("Closing KafkaConfigBackingStore"); - try { - configLog.stop(); - } finally { - if (ownTopicAdmin != null) { - ownTopicAdmin.close(); - } + + if (fencableProducer != null) { + Utils.closeQuietly(() -> fencableProducer.close(Duration.ZERO), "fencable producer for config topic"); } + Utils.closeQuietly(ownTopicAdmin, "admin for config topic"); + Utils.closeQuietly(configLog::stop, "KafkaBasedLog for config topic"); + log.info("Closed KafkaConfigBackingStore"); } + @Override + public void claimWritePrivileges() { + if (usesFencableWriter && fencableProducer == null) { + try { + fencableProducer = createFencableProducer(); + fencableProducer.initTransactions(); + } catch (Exception e) { + if (fencableProducer != null) { + Utils.closeQuietly(() -> fencableProducer.close(Duration.ZERO), "fencable producer for config topic"); + fencableProducer = null; + } + throw new ConnectException("Failed to create and initialize fencable producer for config topic", e); + } + } + } + + private Map baseProducerProps(WorkerConfig workerConfig) { + Map producerProps = new HashMap<>(workerConfig.originals()); + String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(workerConfig); + producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + producerProps.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, Integer.MAX_VALUE); + ConnectUtils.addMetricsContextProperties(producerProps, workerConfig, kafkaClusterId); + return producerProps; + } + + // Visible for testing + Map fencableProducerProps(DistributedConfig workerConfig) { + Map result = new HashMap<>(baseProducerProps(workerConfig)); + + // Always require producer acks to all to ensure durable writes + result.put(ProducerConfig.ACKS_CONFIG, "all"); + // Don't allow more than one in-flight request to prevent reordering on retry (if enabled) + result.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 1); + + ConnectUtils.ensureProperty( + result, ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true", + "for the worker's config topic producer when exactly-once source support is enabled or in preparation to be enabled", + false + ); + ConnectUtils.ensureProperty( + result, ProducerConfig.TRANSACTIONAL_ID_CONFIG, workerConfig.transactionalProducerId(), + "for the worker's config topic producer when exactly-once source support is enabled or in preparation to be enabled", + true + ); + + return result; + } + + // Visible in order to be mocked during testing + Producer createFencableProducer() { + return new KafkaProducer<>(fencableProducerProps); + } + /** * Get a snapshot of the current state of the cluster. */ @@ -334,6 +427,9 @@ public ClusterConfigState snapshot() { new HashMap<>(connectorConfigs), new HashMap<>(connectorTargetStates), new HashMap<>(taskConfigs), + new HashMap<>(connectorTaskCountRecords), + new HashMap<>(connectorTaskConfigGenerations), + new HashSet<>(connectorsPendingFencing), new HashSet<>(inconsistent), configTransformer ); @@ -349,10 +445,14 @@ public boolean contains(String connector) { /** * Write this connector configuration to persistent storage and wait until it has been acknowledged and read back by - * tailing the Kafka log with a consumer. + * tailing the Kafka log with a consumer. {@link #claimWritePrivileges()} must be successfully invoked before calling + * this method if the worker is configured to use a fencable producer for writes to the config topic. * * @param connector name of the connector to write data for * @param properties the configuration to write + * @throws ProducerFencedException if write privileges were claimed by another writer before this method was invoked + * @throws IllegalStateException if {@link #claimWritePrivileges()} is required, but was not successfully invoked before + * this method was called */ @Override public void putConnectorConfig(String connector, Map properties) { @@ -360,19 +460,29 @@ public void putConnectorConfig(String connector, Map properties) Struct connectConfig = new Struct(CONNECTOR_CONFIGURATION_V0); connectConfig.put("properties", properties); byte[] serializedConfig = converter.fromConnectData(topic, CONNECTOR_CONFIGURATION_V0, connectConfig); - updateConnectorConfig(connector, serializedConfig); + try { + maybeSendFencibly(CONNECTOR_KEY(connector), serializedConfig); + configLog.readToEnd().get(READ_TO_END_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + log.error("Failed to write connector configuration to Kafka: ", e); + throw new ConnectException("Error writing connector configuration to Kafka", e); + } } /** - * Remove configuration for a given connector. + * Remove configuration for a given connector. {@link #claimWritePrivileges()} must be successfully invoked before calling + * this method if the worker is configured to use a fencable producer for writes to the config topic. * @param connector name of the connector to remove + * @throws ProducerFencedException if write privileges were claimed by another writer before this method was invoked + * @throws IllegalStateException if {@link #claimWritePrivileges()} is required, but was not successfully invoked before + * this method was called */ @Override public void removeConnectorConfig(String connector) { log.debug("Removing connector configuration for connector '{}'", connector); try { - configLog.send(CONNECTOR_KEY(connector), null); - configLog.send(TARGET_STATE_KEY(connector), null); + maybeSendFencibly(CONNECTOR_KEY(connector), null); + maybeSendFencibly(TARGET_STATE_KEY(connector), null); configLog.readToEnd().get(READ_TO_END_TIMEOUT_MS, TimeUnit.MILLISECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { log.error("Failed to remove connector configuration from Kafka: ", e); @@ -385,24 +495,19 @@ public void removeTaskConfigs(String connector) { throw new UnsupportedOperationException("Removal of tasks is not currently supported"); } - private void updateConnectorConfig(String connector, byte[] serializedConfig) { - try { - configLog.send(CONNECTOR_KEY(connector), serializedConfig); - configLog.readToEnd().get(READ_TO_END_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - log.error("Failed to write connector configuration to Kafka: ", e); - throw new ConnectException("Error writing connector configuration to Kafka", e); - } - } - /** * Write these task configurations and associated commit messages, unless an inconsistency is found that indicates - * that we would be leaving one of the referenced connectors with an inconsistent state. + * that we would be leaving one of the referenced connectors with an inconsistent state. {@link #claimWritePrivileges()} + * must be successfully invoked before calling this method if the worker is configured to use a fencable producer for + * writes to the config topic. * * @param connector the connector to write task configuration * @param configs list of task configurations for the connector * @throws ConnectException if the task configurations do not resolve inconsistencies found in the existing root * and task configurations. + * @throws ProducerFencedException if write privileges were claimed by another writer before this method was invoked + * @throws IllegalStateException if {@link #claimWritePrivileges()} is required, but was not successfully invoked before + * this method was called */ @Override public void putTaskConfigs(String connector, List> configs) { @@ -425,7 +530,7 @@ public void putTaskConfigs(String connector, List> configs) byte[] serializedConfig = converter.fromConnectData(topic, TASK_CONFIGURATION_V0, connectConfig); log.debug("Writing configuration for connector '{}' task {}", connector, index); ConnectorTaskId connectorTaskId = new ConnectorTaskId(connector, index); - configLog.send(TASK_KEY(connectorTaskId), serializedConfig); + maybeSendFencibly(TASK_KEY(connectorTaskId), serializedConfig); index++; } @@ -441,7 +546,7 @@ public void putTaskConfigs(String connector, List> configs) connectConfig.put("tasks", taskCount); byte[] serializedConfig = converter.fromConnectData(topic, CONNECTOR_TASKS_COMMIT_V0, connectConfig); log.debug("Writing commit for connector '{}' with {} tasks.", connector, taskCount); - configLog.send(COMMIT_TASKS_KEY(connector), serializedConfig); + maybeSendFencibly(COMMIT_TASKS_KEY(connector), serializedConfig); // Read to end to ensure all the commit messages have been written configLog.readToEnd().get(READ_TO_END_TIMEOUT_MS, TimeUnit.MILLISECONDS); @@ -460,6 +565,12 @@ public void refresh(long timeout, TimeUnit unit) throws TimeoutException { } } + /** + * Write a new {@link TargetState} for the connector. Note that {@link #claimWritePrivileges()} does not need to be + * invoked before invoking this method. + * @param connector the name of the connector + * @param state the desired target state for the connector + */ @Override public void putTargetState(String connector, TargetState state) { Struct connectTargetState = new Struct(TARGET_STATE_V0); @@ -469,6 +580,40 @@ public void putTargetState(String connector, TargetState state) { configLog.send(TARGET_STATE_KEY(connector), serializedTargetState); } + /** + * Write a task count record for a connector to persistent storage and wait until it has been acknowledged and read back by + * tailing the Kafka log with a consumer. {@link #claimWritePrivileges()} must be successfully invoked before calling this method + * if the worker is configured to use a fencable producer for writes to the config topic. + * @param connector name of the connector + * @param taskCount number of tasks used by the connector + * @throws ProducerFencedException if write privileges were claimed by another writer before this method was invoked + * @throws IllegalStateException if {@link #claimWritePrivileges()} is required, but was not successfully invoked before + * this method was called + */ + @Override + public void putTaskCountRecord(String connector, int taskCount) { + Struct taskCountRecord = new Struct(TASK_COUNT_RECORD_V0); + taskCountRecord.put("tasks", taskCount); + byte[] serializedTaskCountRecord = converter.fromConnectData(topic, TASK_COUNT_RECORD_V0, taskCountRecord); + log.debug("Writing task count record {} for connector {}", taskCount, connector); + try { + maybeSendFencibly(TASK_COUNT_RECORD_KEY(connector), serializedTaskCountRecord); + configLog.readToEnd().get(); + } catch (InterruptedException | ExecutionException e) { + log.error("Failed to write task count record with {} tasks for connector {} to Kafka: ", taskCount, connector, e); + throw new ConnectException("Error writing task count record to Kafka", e); + } + } + + /** + * Write a session key to persistent storage and wait until it has been acknowledged and read back by tailing the Kafka log + * with a consumer. {@link #claimWritePrivileges()} must be successfully invoked before calling this method if the worker + * is configured to use a fencable producer for writes to the config topic. + * @param sessionKey the session key to distributed + * @throws ProducerFencedException if write privileges were claimed by another writer before this method was invoked + * @throws IllegalStateException if {@link #claimWritePrivileges()} is required, but was not successfully invoked before + * this method was called + */ @Override public void putSessionKey(SessionKey sessionKey) { log.debug("Distributing new session key"); @@ -478,7 +623,7 @@ public void putSessionKey(SessionKey sessionKey) { sessionKeyStruct.put("creation-timestamp", sessionKey.creationTimestamp()); byte[] serializedSessionKey = converter.fromConnectData(topic, SESSION_KEY_V0, sessionKeyStruct); try { - configLog.send(SESSION_KEY_KEY, serializedSessionKey); + maybeSendFencibly(SESSION_KEY_KEY, serializedSessionKey); configLog.readToEnd().get(READ_TO_END_TIMEOUT_MS, TimeUnit.MILLISECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { log.error("Failed to write session key to Kafka: ", e); @@ -486,6 +631,12 @@ public void putSessionKey(SessionKey sessionKey) { } } + /** + * Write a restart request for the connector and optionally its tasks to persistent storage and wait until it has been + * acknowledged and read back by tailing the Kafka log with a consumer. {@link #claimWritePrivileges()} must be successfully + * invoked before calling this method if the worker is configured to use a fencable producer for writes to the config topic. + * @param restartRequest the restart request details + */ @Override public void putRestartRequest(RestartRequest restartRequest) { log.debug("Writing {} to Kafka", restartRequest); @@ -495,7 +646,7 @@ public void putRestartRequest(RestartRequest restartRequest) { value.put(ONLY_FAILED_FIELD_NAME, restartRequest.onlyFailed()); byte[] serializedValue = converter.fromConnectData(topic, value.schema(), value); try { - configLog.send(key, serializedValue); + maybeSendFencibly(key, serializedValue); configLog.readToEnd().get(READ_TO_END_TIMEOUT_MS, TimeUnit.MILLISECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { log.error("Failed to write {} to Kafka: ", restartRequest, e); @@ -505,18 +656,22 @@ public void putRestartRequest(RestartRequest restartRequest) { // package private for testing KafkaBasedLog setupAndCreateKafkaBasedLog(String topic, final WorkerConfig config) { + Map producerProps = new HashMap<>(baseProducerProps); + String clusterId = ConnectUtils.lookupKafkaClusterId(config); Map originals = config.originals(); - Map producerProps = new HashMap<>(originals); - producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); - producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); - producerProps.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, Integer.MAX_VALUE); - ConnectUtils.addMetricsContextProperties(producerProps, config, clusterId); Map consumerProps = new HashMap<>(originals); consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); ConnectUtils.addMetricsContextProperties(consumerProps, config, clusterId); + if (config.exactlyOnceSourceEnabled()) { + ConnectUtils.ensureProperty( + consumerProps, ConsumerConfig.ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT), + "for the worker's config topic consumer when exactly-once source support is enabled", + true + ); + } Map adminProps = new HashMap<>(originals); ConnectUtils.addMetricsContextProperties(adminProps, config, clusterId); @@ -541,6 +696,27 @@ KafkaBasedLog setupAndCreateKafkaBasedLog(String topic, final Wo return createKafkaBasedLog(topic, producerProps, consumerProps, new ConsumeCallback(), topicDescription, adminSupplier); } + private void maybeSendFencibly(String key, byte[] value) { + if (!usesFencableWriter) { + configLog.send(key, value); + return; + } + + if (fencableProducer == null) { + throw new IllegalStateException("Cannot produce to config topic without claiming write privileges first"); + } + + try { + fencableProducer.beginTransaction(); + fencableProducer.send(new ProducerRecord<>(topic, key, value)); + fencableProducer.commitTransaction(); + } catch (ProducerFencedException e) { + log.warn("Failed to write to config topic as producer was fenced out", e); + fencableProducer = null; + throw e; + } + } + private KafkaBasedLog createKafkaBasedLog(String topic, Map producerProps, Map consumerProps, Callback> consumedCallback, @@ -600,7 +776,7 @@ public void onCompletion(Throwable error, ConsumerRecord record) } Object targetState = ((Map) value.value()).get("state"); if (!(targetState instanceof String)) { - log.error("Invalid data for target state for connector '{}': 'state' field should be a Map but is {}", + log.error("Invalid data for target state for connector '{}': 'state' field should be a String but is {}", connectorName, targetState == null ? null : targetState.getClass()); return; } @@ -736,6 +912,7 @@ public void onCompletion(Throwable error, ConsumerRecord record) if (deferred != null) { taskConfigs.putAll(deferred); updatedTasks.addAll(deferred.keySet()); + connectorTaskConfigGenerations.compute(connectorName, (ignored, generation) -> generation != null ? generation + 1 : 0); } inconsistent.remove(connectorName); } @@ -748,6 +925,9 @@ public void onCompletion(Throwable error, ConsumerRecord record) connectorTaskCounts.put(connectorName, newTaskCount); } + // If task configs appear after the latest task count record, the connector needs a new round of zombie fencing + // before it can start tasks with these configs + connectorsPendingFencing.add(connectorName); if (started) updateListener.onTaskConfigUpdate(updatedTasks); } else if (record.key().startsWith(RESTART_PREFIX)) { @@ -756,6 +936,19 @@ public void onCompletion(Throwable error, ConsumerRecord record) if (request != null && started) { updateListener.onRestartRequest(request); } + } else if (record.key().startsWith(TASK_COUNT_RECORD_PREFIX)) { + String connectorName = record.key().substring(TASK_COUNT_RECORD_PREFIX.length()); + if (!(value.value() instanceof Map)) { + log.error("Found task count record ({}) in wrong format: {}", record.key(), value.value().getClass()); + return; + } + int taskCount = intValue(((Map) value.value()).get("tasks")); + + log.debug("Setting task count record for connector '{}' to {}", connectorName, taskCount); + connectorTaskCountRecords.put(connectorName, taskCount); + // If a task count record appears after the latest task configs, the connectors doesn't need a round of zombie + // fencing before it can start tasks with the latest configs + connectorsPendingFencing.remove(connectorName); } else if (record.key().equals(SESSION_KEY_KEY)) { if (value.value() == null) { log.error("Ignoring session key because it is unexpectedly null"); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java index 313baf72c58c0..bc64ace35abfa 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStore.java @@ -17,10 +17,13 @@ package org.apache.kafka.connect.storage; import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.config.TopicConfig; import org.apache.kafka.common.serialization.ByteArrayDeserializer; @@ -41,6 +44,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -63,8 +67,62 @@ public class KafkaOffsetBackingStore implements OffsetBackingStore { private static final Logger log = LoggerFactory.getLogger(KafkaOffsetBackingStore.class); - private KafkaBasedLog offsetLog; - private HashMap data; + /** + * Build a connector-specific offset store with read and write support. + * @param topic the name of the offsets topic to use + * @param producer the producer to use for writing to the offsets topic + * @param consumer the consumer to use for reading from the offsets topic + * @param topicAdmin the topic admin to use for creating and querying metadata for the offsets topic + * @return an offset store backed by the given topic and Kafka clients + */ + public static KafkaOffsetBackingStore forTask(String topic, + Producer producer, + Consumer consumer, + TopicAdmin topicAdmin) { + return new KafkaOffsetBackingStore(() -> topicAdmin) { + @Override + public void configure(final WorkerConfig config) { + offsetLog = KafkaBasedLog.withExistingClients( + topic, + consumer, + producer, + topicAdmin, + consumedCallback, + Time.SYSTEM, + initialize(topic, topicDescription(topic, config)) + ); + } + }; + } + + /** + * Build a connector-specific offset store with read-only support. + * @param topic the name of the offsets topic to use + * @param consumer the consumer to use for reading from the offsets topic + * @param topicAdmin the topic admin to use for creating and querying metadata for the offsets topic + * @return a read-only offset store backed by the given topic and Kafka clients + */ + public static KafkaOffsetBackingStore forConnector(String topic, + Consumer consumer, + TopicAdmin topicAdmin) { + return new KafkaOffsetBackingStore(() -> topicAdmin) { + @Override + public void configure(final WorkerConfig config) { + offsetLog = KafkaBasedLog.withExistingClients( + topic, + consumer, + null, + topicAdmin, + consumedCallback, + Time.SYSTEM, + initialize(topic, topicDescription(topic, config)) + ); + } + }; + } + + protected KafkaBasedLog offsetLog; + private final HashMap data = new HashMap<>(); private final Supplier topicAdminSupplier; private SharedTopicAdmin ownTopicAdmin; @@ -77,6 +135,7 @@ public KafkaOffsetBackingStore(Supplier topicAdmin) { this.topicAdminSupplier = Objects.requireNonNull(topicAdmin); } + @Override public void configure(final WorkerConfig config) { String topic = config.getString(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG); @@ -84,7 +143,6 @@ public void configure(final WorkerConfig config) { throw new ConfigException("Offset storage topic must be specified"); String clusterId = ConnectUtils.lookupKafkaClusterId(config); - data = new HashMap<>(); Map originals = config.originals(); Map producerProps = new HashMap<>(originals); @@ -97,6 +155,13 @@ public void configure(final WorkerConfig config) { consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); ConnectUtils.addMetricsContextProperties(consumerProps, config, clusterId); + if (config.exactlyOnceSourceEnabled()) { + ConnectUtils.ensureProperty( + consumerProps, ConsumerConfig.ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT), + "for the worker offsets topic consumer when exactly-once source support is enabled", + false + ); + } Map adminProps = new HashMap<>(originals); ConnectUtils.addMetricsContextProperties(adminProps, config, clusterId); @@ -108,15 +173,7 @@ public void configure(final WorkerConfig config) { ownTopicAdmin = new SharedTopicAdmin(adminProps); adminSupplier = ownTopicAdmin; } - Map topicSettings = config instanceof DistributedConfig - ? ((DistributedConfig) config).offsetStorageTopicSettings() - : Collections.emptyMap(); - NewTopic topicDescription = TopicAdmin.defineTopic(topic) - .config(topicSettings) // first so that we override user-supplied settings as needed - .compacted() - .partitions(config.getInt(DistributedConfig.OFFSET_STORAGE_PARTITIONS_CONFIG)) - .replicationFactor(config.getShort(DistributedConfig.OFFSET_STORAGE_REPLICATION_FACTOR_CONFIG)) - .build(); + NewTopic topicDescription = topicDescription(topic, config); offsetLog = createKafkaBasedLog(topic, producerProps, consumerProps, consumedCallback, topicDescription, adminSupplier); } @@ -125,7 +182,24 @@ private KafkaBasedLog createKafkaBasedLog(String topic, Map consumerProps, Callback> consumedCallback, final NewTopic topicDescription, Supplier adminSupplier) { - java.util.function.Consumer createTopics = admin -> { + java.util.function.Consumer createTopics = initialize(topic, topicDescription); + return new KafkaBasedLog<>(topic, producerProps, consumerProps, adminSupplier, consumedCallback, Time.SYSTEM, createTopics); + } + + protected NewTopic topicDescription(final String topic, final WorkerConfig config) { + Map topicSettings = config instanceof DistributedConfig + ? ((DistributedConfig) config).offsetStorageTopicSettings() + : Collections.emptyMap(); + return TopicAdmin.defineTopic(topic) + .config(topicSettings) // first so that we override user-supplied settings as needed + .compacted() + .partitions(config.getInt(DistributedConfig.OFFSET_STORAGE_PARTITIONS_CONFIG)) + .replicationFactor(config.getShort(DistributedConfig.OFFSET_STORAGE_REPLICATION_FACTOR_CONFIG)) + .build(); + } + + protected java.util.function.Consumer initialize(final String topic, final NewTopic topicDescription) { + return admin -> { log.debug("Creating admin client to manage Connect internal offset topic"); // Create the topic if it doesn't exist Set newTopics = admin.createTopics(topicDescription); @@ -136,7 +210,6 @@ private KafkaBasedLog createKafkaBasedLog(String topic, Map(topic, producerProps, consumerProps, adminSupplier, consumedCallback, Time.SYSTEM, createTopics); } @Override @@ -191,7 +264,7 @@ public Future set(final Map values, final Callback return producerCallback; } - private final Callback> consumedCallback = new Callback>() { + protected final Callback> consumedCallback = new Callback>() { @Override public void onCompletion(Throwable error, ConsumerRecord record) { ByteBuffer key = record.key() != null ? ByteBuffer.wrap(record.key()) : null; @@ -271,4 +344,4 @@ public synchronized Void get(long timeout, TimeUnit unit) throws InterruptedExce return null; } } -} +} \ No newline at end of file diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryConfigBackingStore.java index 38acf2ee69f12..ff6a357139f0b 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryConfigBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryConfigBackingStore.java @@ -20,7 +20,6 @@ import org.apache.kafka.connect.runtime.SessionKey; import org.apache.kafka.connect.runtime.TargetState; import org.apache.kafka.connect.runtime.WorkerConfigTransformer; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; import org.apache.kafka.connect.util.ConnectorTaskId; import java.util.Collections; @@ -75,8 +74,12 @@ public synchronized ClusterConfigState snapshot() { connectorConfigs, connectorTargetStates, taskConfigs, + Collections.emptyMap(), + Collections.emptyMap(), Collections.emptySet(), - configTransformer); + Collections.emptySet(), + configTransformer + ); } @Override @@ -156,6 +159,11 @@ public void putRestartRequest(RestartRequest restartRequest) { // no-op } + @Override + public void putTaskCountRecord(String connector, int taskCount) { + // no-op + } + @Override public synchronized void setUpdateListener(UpdateListener listener) { this.updateListener = listener; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/OffsetStorageReaderImpl.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/OffsetStorageReaderImpl.java index a1eea43103a39..fb394d60c06e2 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/OffsetStorageReaderImpl.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/OffsetStorageReaderImpl.java @@ -89,9 +89,9 @@ public Map, Map> offsets(Collection Map, Map> offsets(Collection Map, Map> offsets(Collection partition, Map offset) { - data.put(partition, offset); + @SuppressWarnings("unchecked") + public synchronized void offset(Map partition, Map offset) { + data.put((Map) partition, (Map) offset); } private boolean flushing() { @@ -113,12 +114,18 @@ public synchronized boolean beginFlush() { if (data.isEmpty()) return false; - assert !flushing(); toFlush = data; data = new HashMap<>(); return true; } + /** + * @return whether there's anything to flush right now. + */ + public synchronized boolean willFlush() { + return !data.isEmpty(); + } + /** * Flush the current offsets and clear them from this writer. This is non-blocking: it * moves the current set of offsets out of the way, serializes the data, and asynchronously diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkConnector.java b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkConnector.java index 55f95a3448c67..fbe29e13ed414 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkConnector.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSinkConnector.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.utils.AppInfoParser; import org.apache.kafka.connect.connector.Task; -import org.apache.kafka.connect.source.SourceConnector; +import org.apache.kafka.connect.sink.SinkConnector; import java.util.ArrayList; import java.util.HashMap; @@ -29,7 +29,7 @@ /** * @see VerifiableSinkTask */ -public class VerifiableSinkConnector extends SourceConnector { +public class VerifiableSinkConnector extends SinkConnector { private Map config; @Override diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java index 8afbdffa167fd..488ac6926ce63 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/VerifiableSourceTask.java @@ -18,6 +18,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; import org.apache.kafka.tools.ThroughputThrottler; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.connect.data.Schema; @@ -31,6 +33,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Stream; /** * A connector primarily intended for system tests. The connector simply generates as many tasks as requested. The @@ -48,6 +51,7 @@ public class VerifiableSourceTask extends SourceTask { public static final String ID_CONFIG = "id"; public static final String TOPIC_CONFIG = "topic"; public static final String THROUGHPUT_CONFIG = "throughput"; + public static final String COMPLETE_RECORD_DATA_CONFIG = "complete.record.data"; private static final String ID_FIELD = "id"; private static final String SEQNO_FIELD = "seqno"; @@ -61,6 +65,15 @@ public class VerifiableSourceTask extends SourceTask { private long startingSeqno; private long seqno; private ThroughputThrottler throttler; + private boolean completeRecordData; + + private static final Schema COMPLETE_VALUE_SCHEMA = SchemaBuilder.struct() + .field("name", Schema.STRING_SCHEMA) + .field("task", Schema.INT32_SCHEMA) + .field("topic", Schema.STRING_SCHEMA) + .field("time_ms", Schema.INT64_SCHEMA) + .field("seqno", Schema.INT64_SCHEMA) + .build(); @Override public String version() { @@ -87,6 +100,7 @@ public void start(Map props) { seqno = 0; startingSeqno = seqno; throttler = new ThroughputThrottler(throughput, System.currentTimeMillis()); + completeRecordData = "true".equalsIgnoreCase(props.get(COMPLETE_RECORD_DATA_CONFIG)); log.info("Started VerifiableSourceTask {}-{} producing to topic {} resuming from seqno {}", name, id, topic, startingSeqno); } @@ -114,7 +128,9 @@ public List poll() throws InterruptedException { System.out.println(dataJson); Map ccOffset = Collections.singletonMap(SEQNO_FIELD, seqno); - SourceRecord srcRecord = new SourceRecord(partition, ccOffset, topic, Schema.INT32_SCHEMA, id, Schema.INT64_SCHEMA, seqno); + Schema valueSchema = completeRecordData ? COMPLETE_VALUE_SCHEMA : Schema.INT64_SCHEMA; + Object value = completeRecordData ? completeValue(data) : seqno; + SourceRecord srcRecord = new SourceRecord(partition, ccOffset, topic, Schema.INT32_SCHEMA, id, valueSchema, value); List result = Collections.singletonList(srcRecord); seqno++; return result; @@ -143,4 +159,12 @@ public void commitRecord(SourceRecord record, RecordMetadata metadata) throws In public void stop() { throttler.wakeup(); } + + private Object completeValue(Map data) { + Struct result = new Struct(COMPLETE_VALUE_SCHEMA); + Stream.of("name", "task", "topic", "time_ms", "seqno").forEach( + field -> result.put(field, data.get(field)) + ); + return result; + } } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/ConnectUtils.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/ConnectUtils.java index c1b83eb648828..7adbd8f92dfd8 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/ConnectUtils.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/ConnectUtils.java @@ -31,6 +31,8 @@ import org.slf4j.LoggerFactory; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ExecutionException; public final class ConnectUtils { @@ -72,6 +74,73 @@ static String lookupKafkaClusterId(Admin adminClient) { } } + /** + * Ensure that the {@link Map properties} contain an expected value for the given key, inserting the + * expected value into the properties if necessary. + * + *

If there is a pre-existing value for the key in the properties, log a warning to the user + * that this value will be ignored, and the expected value will be used instead. + * + * @param props the configuration properties provided by the user; may not be null + * @param key the name of the property to check on; may not be null + * @param expectedValue the expected value for the property; may not be null + * @param justification the reason the property cannot be overridden. + * Will follow the phrase "The value... for the... property will be ignored as it cannot be overridden ". + * For example, one might supply the message "in connectors with the DLQ feature enabled" for this parameter. + * May be null (in which case, no justification is given to the user in the logged warning message) + * @param caseSensitive whether the value should match case-insensitively + */ + public static void ensureProperty( + Map props, + String key, + String expectedValue, + String justification, + boolean caseSensitive + ) { + ensurePropertyAndGetWarning(props, key, expectedValue, justification, caseSensitive).ifPresent(log::warn); + } + + // Visible for testing + /** + * Ensure that a given key has an expected value in the properties, inserting the expected value into the + * properties if necessary. If a user-supplied value is overridden, return a warning message that can + * be logged to the user notifying them of this fact. + * + * @return an {@link Optional} containing a warning that should be logged to the user if a value they + * supplied in the properties is being overridden, or {@link Optional#empty()} if no such override has + * taken place + */ + static Optional ensurePropertyAndGetWarning( + Map props, + String key, + String expectedValue, + String justification, + boolean caseSensitive) { + if (!props.containsKey(key)) { + // Insert the expected value + props.put(key, expectedValue); + // But don't issue a warning to the user + return Optional.empty(); + } + + String value = Objects.toString(props.get(key)); + boolean matchesExpectedValue = caseSensitive ? expectedValue.equals(value) : expectedValue.equalsIgnoreCase(value); + if (matchesExpectedValue) { + return Optional.empty(); + } + + // Insert the expected value + props.put(key, expectedValue); + + justification = justification != null ? " " + justification : ""; + // And issue a warning to the user + return Optional.of(String.format( + "The value '%s' for the '%s' property will be ignored as it cannot be overridden%s. " + + "The value '%s' will be used instead.", + value, key, justification, expectedValue + )); + } + public static void addMetricsContextProperties(Map prop, WorkerConfig config, String clusterId) { //add all properties predefined with "metrics.context." prop.putAll(config.originalsWithPrefix(CommonClientConfigs.METRICS_CONTEXT_PREFIX, false)); @@ -90,4 +159,5 @@ public static boolean isSinkConnector(Connector connector) { public static boolean isSourceConnector(Connector connector) { return SourceConnector.class.isAssignableFrom(connector.getClass()); } + } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java index b1920d59b20e6..8dc9f5a4f31d2 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/KafkaBasedLog.java @@ -25,6 +25,7 @@ import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; @@ -33,6 +34,7 @@ import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.connect.errors.ConnectException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,10 +42,13 @@ import java.time.Duration; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Queue; import java.util.Set; import java.util.concurrent.Future; @@ -77,21 +82,22 @@ public class KafkaBasedLog { private static final long CREATE_TOPIC_TIMEOUT_NS = TimeUnit.SECONDS.toNanos(30); private static final long MAX_SLEEP_MS = TimeUnit.SECONDS.toMillis(1); - private Time time; + private final Time time; private final String topic; private int partitionCount; private final Map producerConfigs; private final Map consumerConfigs; private final Callback> consumedCallback; private final Supplier topicAdminSupplier; + private final boolean requireAdminForOffsets; private Consumer consumer; - private Producer producer; + private Optional> producer; private TopicAdmin admin; private Thread thread; private boolean stopRequested; - private Queue> readLogEndOffsetCallbacks; - private java.util.function.Consumer initializer; + private final Queue> readLogEndOffsetCallbacks; + private final java.util.function.Consumer initializer; /** * Create a new KafkaBasedLog object. This does not start reading the log and writing is not permitted until @@ -156,6 +162,40 @@ public KafkaBasedLog(String topic, this.readLogEndOffsetCallbacks = new ArrayDeque<>(); this.time = time; this.initializer = initializer != null ? initializer : admin -> { }; + + // If the consumer is configured with isolation.level = read_committed, then its end offsets method cannot be relied on + // as it will not take records from currently-open transactions into account. We want to err on the side of caution in that + // case: when users request a read to the end of the log, we will read up to the point where the latest offsets visible to the + // consumer are at least as high as the (possibly-part-of-a-transaction) end offsets of the topic. + this.requireAdminForOffsets = IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT) + .equals(consumerConfigs.get(ConsumerConfig.ISOLATION_LEVEL_CONFIG)); + } + + public static KafkaBasedLog withExistingClients(String topic, + Consumer consumer, + Producer producer, + TopicAdmin topicAdmin, + Callback> consumedCallback, + Time time, + java.util.function.Consumer initializer) { + return new KafkaBasedLog(topic, + Collections.emptyMap(), + Collections.emptyMap(), + () -> topicAdmin, + consumedCallback, + time, + initializer) { + + @Override + protected Producer createProducer() { + return producer; + } + + @Override + protected Consumer createConsumer() { + return consumer; + } + }; } public void start() { @@ -163,10 +203,19 @@ public void start() { // Create the topic admin client and initialize the topic ... admin = topicAdminSupplier.get(); // may be null + if (admin == null && requireAdminForOffsets) { + throw new ConnectException( + "Must provide a TopicAdmin to KafkaBasedLog when consumer is configured with " + + ConsumerConfig.ISOLATION_LEVEL_CONFIG + " set to " + + IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT) + ); + } initializer.accept(admin); // Then create the producer and consumer - producer = createProducer(); + producer = Optional.ofNullable(createProducer()); + if (!producer.isPresent()) + log.trace("Creating read-only KafkaBasedLog for topic " + topic); consumer = createConsumer(); List partitions = new ArrayList<>(); @@ -210,26 +259,21 @@ public void stop() { synchronized (this) { stopRequested = true; } - consumer.wakeup(); - - try { - thread.join(); - } catch (InterruptedException e) { - throw new ConnectException("Failed to stop KafkaBasedLog. Exiting without cleanly shutting " + - "down it's producer and consumer.", e); + if (consumer != null) { + consumer.wakeup(); } - try { - producer.close(); - } catch (KafkaException e) { - log.error("Failed to stop KafkaBasedLog producer", e); + if (thread != null) { + try { + thread.join(); + } catch (InterruptedException e) { + throw new ConnectException("Failed to stop KafkaBasedLog. Exiting without cleanly shutting " + + "down it's producer and consumer.", e); + } } - try { - consumer.close(); - } catch (KafkaException e) { - log.error("Failed to stop KafkaBasedLog consumer", e); - } + producer.ifPresent(p -> Utils.closeQuietly(p, "KafkaBasedLog producer for topic " + topic)); + Utils.closeQuietly(consumer, "KafkaBasedLog consumer for topic " + topic); // do not close the admin client, since we don't own it admin = null; @@ -239,7 +283,7 @@ public void stop() { /** * Flushes any outstanding writes and then reads to the current end of the log and invokes the specified callback. - * Note that this checks the current, offsets, reads to them, and invokes the callback regardless of whether + * Note that this checks the current offsets, reads to them, and invokes the callback regardless of whether * additional records have been written to the log. If the caller needs to ensure they have truly reached the end * of the log, they must ensure there are no other writers during this period. * @@ -252,7 +296,7 @@ public void stop() { */ public void readToEnd(Callback callback) { log.trace("Starting read to end log for topic {}", topic); - producer.flush(); + flush(); synchronized (this) { readLogEndOffsetCallbacks.add(callback); } @@ -263,7 +307,7 @@ public void readToEnd(Callback callback) { * Flush the underlying producer to ensure that all pending writes have been sent. */ public void flush() { - producer.flush(); + producer.ifPresent(Producer::flush); } /** @@ -281,14 +325,16 @@ public void send(K key, V value) { } public void send(K key, V value, org.apache.kafka.clients.producer.Callback callback) { - producer.send(new ProducerRecord<>(topic, key, value), callback); + producer.orElseThrow(() -> + new IllegalStateException("This KafkaBasedLog was created in read-only mode and does not support write operations") + ).send(new ProducerRecord<>(topic, key, value), callback); } public int partitionCount() { return partitionCount; } - private Producer createProducer() { + protected Producer createProducer() { // Always require producer acks to all to ensure durable writes producerConfigs.put(ProducerConfig.ACKS_CONFIG, "all"); @@ -297,7 +343,7 @@ private Producer createProducer() { return new KafkaProducer<>(producerConfigs); } - private Consumer createConsumer() { + protected Consumer createConsumer() { // Always force reset to the beginning of the log since this class wants to consume all available log data consumerConfigs.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); @@ -345,7 +391,14 @@ private void readToLogEnd() { } // Visible for testing - Map readEndOffsets(Set assignment) { + /** + * Read to the end of the given list of topic partitions + * @param assignment the topic partitions to read to the end of + * @throws UnsupportedVersionException if the log's consumer is using the "read_committed" isolation level (and + * therefore a separate admin client is required to read end offsets for the topic), but the broker does not support + * reading end offsets using an admin client + */ + Map readEndOffsets(Set assignment) throws UnsupportedVersionException { log.trace("Reading to end of offset log"); // Note that we'd prefer to not use the consumer to find the end offsets for the assigned topic partitions. @@ -364,6 +417,10 @@ Map readEndOffsets(Set assignment) { } catch (UnsupportedVersionException e) { // This may happen with really old brokers that don't support the auto topic creation // field in metadata requests + if (requireAdminForOffsets) { + // Should be handled by the caller during log startup + throw e; + } log.debug("Reading to end of log offsets with consumer since admin client is unsupported: {}", e.getMessage()); // Forget the reference to the admin so that we won't even try to use the admin the next time this method is called admin = null; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java index 7b2f152e2eede..e66722f03b921 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/TopicAdmin.java @@ -23,11 +23,13 @@ import org.apache.kafka.clients.admin.CreateTopicsOptions; import org.apache.kafka.clients.admin.DescribeConfigsOptions; import org.apache.kafka.clients.admin.DescribeTopicsOptions; +import org.apache.kafka.clients.admin.ListOffsetsOptions; import org.apache.kafka.clients.admin.ListOffsetsResult; import org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.OffsetSpec; import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.ConfigException; @@ -264,7 +266,7 @@ public static NewTopicBuilder defineTopic(String topicName) { } private static final Logger log = LoggerFactory.getLogger(TopicAdmin.class); - private final Map adminConfig; + private final String bootstrapServers; private final Admin admin; private final boolean logCreation; @@ -274,18 +276,23 @@ public static NewTopicBuilder defineTopic(String topicName) { * @param adminConfig the configuration for the {@link Admin} */ public TopicAdmin(Map adminConfig) { - this(adminConfig, Admin.create(adminConfig)); + this(adminConfig.get(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG), Admin.create(adminConfig)); } - // visible for testing - TopicAdmin(Map adminConfig, Admin adminClient) { - this(adminConfig, adminClient, true); + /** + * Create a new topic admin using the provided {@link Admin} + * + * @param bootstrapServers the Kafka cluster targeted by the admin + * @param adminClient the {@link Admin} to use under the hood + */ + public TopicAdmin(Object bootstrapServers, Admin adminClient) { + this(bootstrapServers, adminClient, true); } // visible for testing - TopicAdmin(Map adminConfig, Admin adminClient, boolean logCreation) { + TopicAdmin(Object bootstrapServers, Admin adminClient, boolean logCreation) { this.admin = adminClient; - this.adminConfig = adminConfig != null ? adminConfig : Collections.emptyMap(); + this.bootstrapServers = bootstrapServers != null ? bootstrapServers.toString() : ""; this.logCreation = logCreation; } @@ -371,7 +378,6 @@ public TopicCreationResponse createOrFindTopics(NewTopic... topics) { } } if (topicsByName.isEmpty()) return EMPTY_CREATION; - String bootstrapServers = bootstrapServers(); String topicNameList = Utils.join(topicsByName.keySet(), "', '"); // Attempt to create any missing topics @@ -448,7 +454,6 @@ public Map describeTopics(String... topics) { if (topics == null) { return Collections.emptyMap(); } - String bootstrapServers = bootstrapServers(); String topicNameList = String.join(", ", topics); Map> newResults = @@ -604,7 +609,6 @@ public Map describeTopicConfigs(String... topicNames) { if (topics.isEmpty()) { return Collections.emptyMap(); } - String bootstrapServers = bootstrapServers(); String topicNameList = topics.stream().collect(Collectors.joining(", ")); Collection resources = topics.stream() .map(t -> new ConfigResource(ConfigResource.Type.TOPIC, t)) @@ -664,7 +668,7 @@ public Map endOffsets(Set partitions) { return Collections.emptyMap(); } Map offsetSpecMap = partitions.stream().collect(Collectors.toMap(Function.identity(), tp -> OffsetSpec.latest())); - ListOffsetsResult resultFuture = admin.listOffsets(offsetSpecMap); + ListOffsetsResult resultFuture = admin.listOffsets(offsetSpecMap, new ListOffsetsOptions(IsolationLevel.READ_UNCOMMITTED)); // Get the individual result for each topic partition so we have better error messages Map result = new HashMap<>(); for (TopicPartition partition : partitions) { @@ -675,28 +679,28 @@ public Map endOffsets(Set partitions) { Throwable cause = e.getCause(); String topic = partition.topic(); if (cause instanceof AuthorizationException) { - String msg = String.format("Not authorized to get the end offsets for topic '%s' on brokers at %s", topic, bootstrapServers()); + String msg = String.format("Not authorized to get the end offsets for topic '%s' on brokers at %s", topic, bootstrapServers); throw new ConnectException(msg, e); } else if (cause instanceof UnsupportedVersionException) { // Should theoretically never happen, because this method is the same as what the consumer uses and therefore // should exist in the broker since before the admin client was added - String msg = String.format("API to get the get the end offsets for topic '%s' is unsupported on brokers at %s", topic, bootstrapServers()); + String msg = String.format("API to get the get the end offsets for topic '%s' is unsupported on brokers at %s", topic, bootstrapServers); throw new UnsupportedVersionException(msg, e); } else if (cause instanceof TimeoutException) { - String msg = String.format("Timed out while waiting to get end offsets for topic '%s' on brokers at %s", topic, bootstrapServers()); + String msg = String.format("Timed out while waiting to get end offsets for topic '%s' on brokers at %s", topic, bootstrapServers); throw new TimeoutException(msg, e); } else if (cause instanceof LeaderNotAvailableException) { - String msg = String.format("Unable to get end offsets during leader election for topic '%s' on brokers at %s", topic, bootstrapServers()); + String msg = String.format("Unable to get end offsets during leader election for topic '%s' on brokers at %s", topic, bootstrapServers); throw new LeaderNotAvailableException(msg, e); } else if (cause instanceof org.apache.kafka.common.errors.RetriableException) { throw (org.apache.kafka.common.errors.RetriableException) cause; } else { - String msg = String.format("Error while getting end offsets for topic '%s' on brokers at %s", topic, bootstrapServers()); + String msg = String.format("Error while getting end offsets for topic '%s' on brokers at %s", topic, bootstrapServers); throw new ConnectException(msg, e); } } catch (InterruptedException e) { Thread.interrupted(); - String msg = String.format("Interrupted while attempting to read end offsets for topic '%s' on brokers at %s", partition.topic(), bootstrapServers()); + String msg = String.format("Interrupted while attempting to read end offsets for topic '%s' on brokers at %s", partition.topic(), bootstrapServers); throw new RetriableException(msg, e); } } @@ -711,9 +715,4 @@ public void close() { public void close(Duration timeout) { admin.close(timeout); } - - private String bootstrapServers() { - Object servers = adminConfig.get(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG); - return servers != null ? servers.toString() : ""; - } } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/connector/policy/BaseConnectorClientConfigOverridePolicyTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/connector/policy/BaseConnectorClientConfigOverridePolicyTest.java index 28fee73a93966..e78b85d651c05 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/connector/policy/BaseConnectorClientConfigOverridePolicyTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/connector/policy/BaseConnectorClientConfigOverridePolicyTest.java @@ -43,7 +43,7 @@ private List configValues(Map clientConfig) { ConnectorClientConfigRequest connectorClientConfigRequest = new ConnectorClientConfigRequest( "test", ConnectorType.SOURCE, - WorkerTest.WorkerTestConnector.class, + WorkerTest.WorkerTestSourceConnector.class, clientConfig, ConnectorClientConfigRequest.ClientType.PRODUCER); return policyToTest().validate(connectorClientConfigRequest); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java index b31455b248483..bed05fa21e41c 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectorHandle.java @@ -112,6 +112,14 @@ public void deleteTask(String taskId) { taskHandles.remove(taskId); } + /** + * Delete all task handles for this connector. + */ + public void clearTasks() { + log.info("Clearing {} existing task handles for connector {}", taskHandles.size(), connectorName); + taskHandles.clear(); + } + /** * Set the number of expected records for this connector. * diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExactlyOnceSourceIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExactlyOnceSourceIntegrationTest.java new file mode 100644 index 0000000000000..a6dea215224d5 --- /dev/null +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ExactlyOnceSourceIntegrationTest.java @@ -0,0 +1,925 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.integration; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.errors.ProducerFencedException; +import org.apache.kafka.common.network.ListenerName; +import org.apache.kafka.common.security.auth.SecurityProtocol; +import org.apache.kafka.connect.connector.Task; +import org.apache.kafka.connect.errors.ConnectException; +import org.apache.kafka.connect.json.JsonConverter; +import org.apache.kafka.connect.json.JsonConverterConfig; +import org.apache.kafka.connect.runtime.Worker; +import org.apache.kafka.connect.runtime.WorkerConfig; +import org.apache.kafka.connect.runtime.distributed.DistributedConfig; +import org.apache.kafka.connect.runtime.rest.entities.ConfigInfo; +import org.apache.kafka.connect.runtime.rest.entities.ConfigInfos; +import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException; +import org.apache.kafka.connect.source.SourceConnector; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.storage.StringConverter; +import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster; +import org.apache.kafka.connect.util.clusters.EmbeddedConnectClusterAssertions; +import org.apache.kafka.connect.util.clusters.EmbeddedKafkaCluster; +import org.apache.kafka.test.IntegrationTest; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.LongStream; + +import static org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.TRANSACTIONAL_ID_CONFIG; +import static org.apache.kafka.connect.integration.MonitorableSourceConnector.TOPIC_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX; +import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX; +import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX; +import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.NAME_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG; +import static org.apache.kafka.connect.runtime.SourceConnectorConfig.OFFSETS_TOPIC_CONFIG; +import static org.apache.kafka.connect.runtime.SourceConnectorConfig.TRANSACTION_BOUNDARY_CONFIG; +import static org.apache.kafka.connect.runtime.SourceConnectorConfig.TRANSACTION_BOUNDARY_INTERVAL_CONFIG; +import static org.apache.kafka.connect.source.SourceTask.TransactionBoundary.CONNECTOR; +import static org.apache.kafka.connect.source.SourceTask.TransactionBoundary.INTERVAL; +import static org.apache.kafka.connect.source.SourceTask.TransactionBoundary.POLL; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +@Category(IntegrationTest.class) +public class ExactlyOnceSourceIntegrationTest { + + private static final Logger log = LoggerFactory.getLogger(ExactlyOnceSourceIntegrationTest.class); + private static final String CLUSTER_GROUP_ID = "exactly-once-source-integration-test"; + private static final String CONNECTOR_NAME = "exactlyOnceQuestionMark"; + + private static final int SOURCE_TASK_PRODUCE_TIMEOUT_MS = 30_000; + + private Properties brokerProps; + private Map workerProps; + private EmbeddedConnectCluster.Builder connectBuilder; + private EmbeddedConnectCluster connect; + private ConnectorHandle connectorHandle; + + @Before + public void setup() { + workerProps = new HashMap<>(); + workerProps.put(DistributedConfig.EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "enabled"); + workerProps.put(DistributedConfig.GROUP_ID_CONFIG, CLUSTER_GROUP_ID); + + brokerProps = new Properties(); + brokerProps.put("transaction.state.log.replication.factor", "1"); + brokerProps.put("transaction.state.log.min.isr", "1"); + + // build a Connect cluster backed by Kafka and Zk + connectBuilder = new EmbeddedConnectCluster.Builder() + .numWorkers(3) + .numBrokers(1) + .workerProps(workerProps) + .brokerProps(brokerProps); + + // get a handle to the connector + connectorHandle = RuntimeHandles.get().connectorHandle(CONNECTOR_NAME); + } + + private void startConnect() { + connect = connectBuilder.build(); + connect.start(); + } + + @After + public void close() { + try { + // stop all Connect, Kafka and Zk threads. + connect.stop(); + } finally { + // Clear the handle for the connector. Fun fact: if you don't do this, your tests become quite flaky. + RuntimeHandles.get().deleteConnector(CONNECTOR_NAME); + } + } + + /** + * A simple test for the pre-flight validation API for connectors to provide their own delivery guarantees. + */ + @Test + public void testPreflightValidation() { + connectBuilder.numWorkers(1); + startConnect(); + + Map props = new HashMap<>(); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getName()); + props.put(TASKS_MAX_CONFIG, "1"); + props.put(TOPIC_CONFIG, "topic"); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(NAME_CONFIG, CONNECTOR_NAME); + + // Test out the "exactly.once.support" property + props.put(EXACTLY_ONCE_SUPPORT_CONFIG, "required"); + + // Connector will return null from SourceConnector::exactlyOnceSupport + props.put("exactly.once.support.level", "null"); + ConfigInfos validation = connect.validateConnectorConfig(MonitorableSourceConnector.class.getSimpleName(), props); + assertEquals("Preflight validation should have exactly one error", 1, validation.errorCount()); + ConfigInfo propertyValidation = findConfigInfo(EXACTLY_ONCE_SUPPORT_CONFIG, validation); + assertFalse("Preflight validation for exactly-once support property should have at least one error message", + propertyValidation.configValue().errors().isEmpty()); + + // Connector will return UNSUPPORTED from SourceConnector::exactlyOnceSupport + props.put("exactly.once.support.level", "unsupported"); + validation = connect.validateConnectorConfig(MonitorableSourceConnector.class.getSimpleName(), props); + assertEquals("Preflight validation should have exactly one error", 1, validation.errorCount()); + propertyValidation = findConfigInfo(EXACTLY_ONCE_SUPPORT_CONFIG, validation); + assertFalse("Preflight validation for exactly-once support property should have at least one error message", + propertyValidation.configValue().errors().isEmpty()); + + // Connector will throw an exception from SourceConnector::exactlyOnceSupport + props.put("exactly.once.support.level", "fail"); + validation = connect.validateConnectorConfig(MonitorableSourceConnector.class.getSimpleName(), props); + assertEquals("Preflight validation should have exactly one error", 1, validation.errorCount()); + propertyValidation = findConfigInfo(EXACTLY_ONCE_SUPPORT_CONFIG, validation); + assertFalse("Preflight validation for exactly-once support property should have at least one error message", + propertyValidation.configValue().errors().isEmpty()); + + // Connector will return SUPPORTED from SourceConnector::exactlyOnceSupport + props.put("exactly.once.support.level", "supported"); + validation = connect.validateConnectorConfig(MonitorableSourceConnector.class.getSimpleName(), props); + assertEquals("Preflight validation should have zero errors", 0, validation.errorCount()); + + // Test out the transaction boundary definition property + props.put(TRANSACTION_BOUNDARY_CONFIG, CONNECTOR.toString()); + + // Connector will return null from SourceConnector::canDefineTransactionBoundaries + props.put("custom.transaction.boundary.support", "null"); + validation = connect.validateConnectorConfig(MonitorableSourceConnector.class.getSimpleName(), props); + assertEquals("Preflight validation should have exactly one error", 1, validation.errorCount()); + propertyValidation = findConfigInfo(TRANSACTION_BOUNDARY_CONFIG, validation); + assertFalse("Preflight validation for transaction boundary property should have at least one error message", + propertyValidation.configValue().errors().isEmpty()); + + // Connector will return UNSUPPORTED from SourceConnector::canDefineTransactionBoundaries + props.put("custom.transaction.boundary.support", "unsupported"); + validation = connect.validateConnectorConfig(MonitorableSourceConnector.class.getSimpleName(), props); + assertEquals("Preflight validation should have exactly one error", 1, validation.errorCount()); + propertyValidation = findConfigInfo(TRANSACTION_BOUNDARY_CONFIG, validation); + assertFalse("Preflight validation for transaction boundary property should have at least one error message", + propertyValidation.configValue().errors().isEmpty()); + + // Connector will throw an exception from SourceConnector::canDefineTransactionBoundaries + props.put("custom.transaction.boundary.support", "null"); + validation = connect.validateConnectorConfig(MonitorableSourceConnector.class.getSimpleName(), props); + assertEquals("Preflight validation should have exactly one error", 1, validation.errorCount()); + propertyValidation = findConfigInfo(TRANSACTION_BOUNDARY_CONFIG, validation); + assertFalse("Preflight validation for transaction boundary property should have at least one error message", + propertyValidation.configValue().errors().isEmpty()); + + // Connector will return SUPPORTED from SourceConnector::canDefineTransactionBoundaries + props.put("custom.transaction.boundary.support", "supported"); + validation = connect.validateConnectorConfig(MonitorableSourceConnector.class.getSimpleName(), props); + assertEquals("Preflight validation should have zero errors", 0, validation.errorCount()); + } + + /** + * A simple green-path test that ensures the worker can start up a source task with exactly-once support enabled + * and write some records to Kafka that will be visible to a downstream consumer using the "READ_COMMITTED" + * isolation level. The "poll" transaction boundary is used. + */ + @Test + public void testPollBoundary() throws Exception { + // Much slower offset commit interval; should never be triggered during this test + workerProps.put(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG, "600000"); + connectBuilder.numWorkers(1); + startConnect(); + + String topic = "test-topic"; + connect.kafka().createTopic(topic, 3); + + int recordsProduced = 100; + + Map props = new HashMap<>(); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getName()); + props.put(TASKS_MAX_CONFIG, "1"); + props.put(TOPIC_CONFIG, topic); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(NAME_CONFIG, CONNECTOR_NAME); + props.put(TRANSACTION_BOUNDARY_CONFIG, POLL.toString()); + props.put("messages.per.poll", Integer.toString(recordsProduced)); + + // expect all records to be consumed and committed by the connector + connectorHandle.expectedRecords(recordsProduced); + connectorHandle.expectedCommits(recordsProduced); + + // start a source connector + connect.configureConnector(CONNECTOR_NAME, props); + + log.info("Waiting for records to be provided to worker by task"); + // wait for the connector tasks to produce enough records + connectorHandle.awaitRecords(SOURCE_TASK_PRODUCE_TIMEOUT_MS); + + log.info("Waiting for records to be committed to Kafka by worker"); + // wait for the connector tasks to commit enough records + connectorHandle.awaitCommits(TimeUnit.MINUTES.toMillis(1)); + + // consume all records from the source topic or fail, to ensure that they were correctly produced + int recordNum = connect.kafka() + .consume( + recordsProduced, + TimeUnit.MINUTES.toMillis(1), + Collections.singletonMap(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"), + "test-topic") + .count(); + assertTrue("Not enough records produced by source connector. Expected at least: " + recordsProduced + " + but got " + recordNum, + recordNum >= recordsProduced); + } + + /** + * A simple green-path test that ensures the worker can start up a source task with exactly-once support enabled + * and write some records to Kafka that will be visible to a downstream consumer using the "READ_COMMITTED" + * isolation level. The "interval" transaction boundary is used with a connector-specific override. + */ + @Test + public void testIntervalBoundary() throws Exception { + // Much slower offset commit interval; should never be triggered during this test + workerProps.put(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG, "600000"); + connectBuilder.numWorkers(1); + startConnect(); + + String topic = "test-topic"; + connect.kafka().createTopic(topic, 3); + + int recordsProduced = 100; + + Map props = new HashMap<>(); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getName()); + props.put(TASKS_MAX_CONFIG, "1"); + props.put(TOPIC_CONFIG, topic); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(NAME_CONFIG, CONNECTOR_NAME); + props.put(TRANSACTION_BOUNDARY_CONFIG, INTERVAL.toString()); + props.put(TRANSACTION_BOUNDARY_INTERVAL_CONFIG, "10000"); + props.put("messages.per.poll", Integer.toString(recordsProduced)); + + // expect all records to be consumed and committed by the connector + connectorHandle.expectedRecords(recordsProduced); + connectorHandle.expectedCommits(recordsProduced); + + // start a source connector + connect.configureConnector(CONNECTOR_NAME, props); + + log.info("Waiting for records to be provided to worker by task"); + // wait for the connector tasks to produce enough records + connectorHandle.awaitRecords(SOURCE_TASK_PRODUCE_TIMEOUT_MS); + + log.info("Waiting for records to be committed to Kafka by worker"); + // wait for the connector tasks to commit enough records + connectorHandle.awaitCommits(TimeUnit.MINUTES.toMillis(1)); + + // consume all records from the source topic or fail, to ensure that they were correctly produced + int recordNum = connect.kafka() + .consume( + recordsProduced, + TimeUnit.MINUTES.toMillis(1), + Collections.singletonMap(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"), + "test-topic") + .count(); + assertTrue("Not enough records produced by source connector. Expected at least: " + recordsProduced + " + but got " + recordNum, + recordNum >= recordsProduced); + } + + /** + * A simple green-path test that ensures the worker can start up a source task with exactly-once support enabled + * and write some records to Kafka that will be visible to a downstream consumer using the "READ_COMMITTED" + * isolation level. The "connector" transaction boundary is used with a connector that defines transactions whose + * size correspond to successive elements of the Fibonacci sequence, where transactions with an even number of + * records are aborted, and those with an odd number of records are committed. + */ + @Test + public void testConnectorBoundary() throws Exception { + String offsetsTopic = "exactly-once-source-cluster-offsets"; + workerProps.put(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG, offsetsTopic); + connectBuilder.numWorkers(1); + startConnect(); + + String topic = "test-topic"; + connect.kafka().createTopic(topic, 3); + + int recordsProduced = 100; + + Map props = new HashMap<>(); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getName()); + props.put(TASKS_MAX_CONFIG, "1"); + props.put(TOPIC_CONFIG, topic); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(NAME_CONFIG, CONNECTOR_NAME); + props.put(TRANSACTION_BOUNDARY_CONFIG, CONNECTOR.toString()); + props.put("custom.transaction.boundary.support", "supported"); + props.put("messages.per.poll", Integer.toString(recordsProduced)); + + // expect all records to be consumed and committed by the connector + connectorHandle.expectedRecords(recordsProduced); + connectorHandle.expectedCommits(recordsProduced); + + // start a source connector + connect.configureConnector(CONNECTOR_NAME, props); + + log.info("Waiting for records to be provided to worker by task"); + // wait for the connector tasks to produce enough records + connectorHandle.awaitRecords(SOURCE_TASK_PRODUCE_TIMEOUT_MS); + + log.info("Waiting for records to be committed to Kafka by worker"); + // wait for the connector tasks to commit enough records + connectorHandle.awaitCommits(TimeUnit.MINUTES.toMillis(1)); + + Map consumerProps = new HashMap<>(); + consumerProps.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"); + consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + // consume all records from the source topic or fail, to ensure that they were correctly produced + ConsumerRecords sourceRecords = connect.kafka() + .consume( + recordsProduced, + TimeUnit.MINUTES.toMillis(1), + consumerProps, + "test-topic"); + assertTrue("Not enough records produced by source connector. Expected at least: " + recordsProduced + " + but got " + sourceRecords.count(), + sourceRecords.count() >= recordsProduced); + + // also consume from the cluster's offsets topic to verify that the expected offsets (which should correspond to the connector's + // custom transaction boundaries) were committed + List expectedOffsetSeqnos = new ArrayList<>(); + long lastExpectedOffsetSeqno = 1; + long nextExpectedOffsetSeqno = 1; + while (nextExpectedOffsetSeqno <= recordsProduced) { + expectedOffsetSeqnos.add(nextExpectedOffsetSeqno); + nextExpectedOffsetSeqno += lastExpectedOffsetSeqno; + lastExpectedOffsetSeqno = nextExpectedOffsetSeqno - lastExpectedOffsetSeqno; + } + ConsumerRecords offsetRecords = connect.kafka() + .consume( + expectedOffsetSeqnos.size(), + TimeUnit.MINUTES.toMillis(1), + consumerProps, + offsetsTopic + ); + + List actualOffsetSeqnos = new ArrayList<>(); + offsetRecords.forEach(record -> actualOffsetSeqnos.add(parseAndAssertOffsetForSingleTask(record))); + + assertEquals("Committed offsets should match connector-defined transaction boundaries", + expectedOffsetSeqnos, actualOffsetSeqnos.subList(0, expectedOffsetSeqnos.size())); + + List expectedRecordSeqnos = LongStream.range(1, recordsProduced + 1).boxed().collect(Collectors.toList()); + long priorBoundary = 1; + long nextBoundary = 2; + while (priorBoundary < expectedRecordSeqnos.get(expectedRecordSeqnos.size() - 1)) { + if (nextBoundary % 2 == 0) { + for (long i = priorBoundary + 1; i < nextBoundary + 1; i++) { + expectedRecordSeqnos.remove(i); + } + } + nextBoundary += priorBoundary; + priorBoundary = nextBoundary - priorBoundary; + } + List actualRecordSeqnos = new ArrayList<>(); + sourceRecords.forEach(record -> actualRecordSeqnos.add(parseAndAssertValueForSingleTask(record))); + Collections.sort(actualRecordSeqnos); + assertEquals("Committed records should exclude connector-aborted transactions", + expectedRecordSeqnos, actualRecordSeqnos.subList(0, expectedRecordSeqnos.size())); + } + + /** + * Brings up a one-node cluster, then intentionally fences out the transactional producer used by the leader + * for writes to the config topic to simulate a zombie leader being active in the cluster. The leader should + * automatically recover, verify that it is still the leader, and then succeed to create a connector when the + * user resends the request. + */ + @Test + public void testFencedLeaderRecovery() throws Exception { + // Much slower offset commit interval; should never be triggered during this test + workerProps.put(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG, "600000"); + startConnect(); + + String topic = "test-topic"; + connect.kafka().createTopic(topic, 3); + + int recordsProduced = 100; + + Map props = new HashMap<>(); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getName()); + props.put(TASKS_MAX_CONFIG, "1"); + props.put(TOPIC_CONFIG, topic); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(NAME_CONFIG, CONNECTOR_NAME); + props.put(TRANSACTION_BOUNDARY_CONFIG, POLL.toString()); + props.put("messages.per.poll", Integer.toString(recordsProduced)); + + // expect all records to be consumed and committed by the connector + connectorHandle.expectedRecords(recordsProduced); + connectorHandle.expectedCommits(recordsProduced); + + // make sure the worker is actually up (otherwise, it may fence out our simulated zombie leader, instead of the other way around) + assertEquals(404, connect.requestGet(connect.endpointForResource("connectors/nonexistent")).getStatus()); + + // fence out the leader of the cluster + transactionalProducer(DistributedConfig.transactionalProducerId(CLUSTER_GROUP_ID)).initTransactions(); + + // start a source connector--should fail the first time + assertThrows(ConnectRestException.class, () -> connect.configureConnector(CONNECTOR_NAME, props)); + + // if at first you don't succeed, then spam the worker with rest requests until it gives in to your demands + connect.configureConnector(CONNECTOR_NAME, props); + + log.info("Waiting for records to be provided to worker by task"); + // wait for the connector tasks to produce enough records + connectorHandle.awaitRecords(SOURCE_TASK_PRODUCE_TIMEOUT_MS); + + log.info("Waiting for records to be committed to Kafka by worker"); + // wait for the connector tasks to commit enough records + connectorHandle.awaitCommits(TimeUnit.MINUTES.toMillis(1)); + + // consume all records from the source topic or fail, to ensure that they were correctly produced + int recordNum = connect.kafka() + .consume( + recordsProduced, + TimeUnit.MINUTES.toMillis(1), + Collections.singletonMap(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"), + "test-topic") + .count(); + assertTrue("Not enough records produced by source connector. Expected at least: " + recordsProduced + " + but got " + recordNum, + recordNum >= recordsProduced); + } + + /** + * A moderately-complex green-path test that ensures the worker can start up and run tasks for a source + * connector that gets reconfigured, and will fence out potential zombie tasks for older generations before + * bringing up new task instances. + */ + @Test + public void testConnectorReconfiguration() throws Exception { + // Much slower offset commit interval; should never be triggered during this test + workerProps.put(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG, "600000"); + startConnect(); + + String topic = "test-topic"; + connect.kafka().createTopic(topic, 3); + + int recordsProduced = 100; + + Map props = new HashMap<>(); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getName()); + props.put(TOPIC_CONFIG, topic); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(NAME_CONFIG, CONNECTOR_NAME); + props.put("messages.per.poll", Integer.toString(recordsProduced)); + + // expect all records to be consumed and committed by the connector + connectorHandle.expectedRecords(recordsProduced); + connectorHandle.expectedCommits(recordsProduced); + + StartAndStopLatch connectorStart = connectorAndTaskStart(3); + props.put(TASKS_MAX_CONFIG, "3"); + // start a source connector + connect.configureConnector(CONNECTOR_NAME, props); + assertConnectorStarted(connectorStart); + + assertProducersAreFencedOnReconfiguration(3, 5, topic, props); + assertProducersAreFencedOnReconfiguration(5, 1, topic, props); + assertProducersAreFencedOnReconfiguration(1, 5, topic, props); + assertProducersAreFencedOnReconfiguration(5, 3, topic, props); + + // Do a final sanity check to make sure that the final generation of tasks is able to run + log.info("Waiting for records to be provided to worker by task"); + // wait for the connector tasks to produce enough records + connectorHandle.awaitRecords(SOURCE_TASK_PRODUCE_TIMEOUT_MS); + + log.info("Waiting for records to be committed to Kafka by worker"); + // wait for the connector tasks to commit enough records + connectorHandle.awaitCommits(TimeUnit.MINUTES.toMillis(1)); + + // consume all records from the source topic or fail, to ensure that they were correctly produced + int recordNum = connect.kafka() + .consume( + recordsProduced, + TimeUnit.MINUTES.toMillis(1), + Collections.singletonMap(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"), + "test-topic") + .count(); + assertTrue("Not enough records produced by source connector. Expected at least: " + recordsProduced + " + but got " + recordNum, + recordNum >= recordsProduced); + } + + @Test + public void testConnectorFailsOnInabilityToFence() throws Exception { + brokerProps.put("authorizer.class.name", "kafka.security.authorizer.AclAuthorizer"); + brokerProps.put("sasl.enabled.mechanisms", "PLAIN"); + brokerProps.put("sasl.mechanism.inter.broker.protocol", "PLAIN"); + brokerProps.put("security.inter.broker.protocol", "SASL_PLAINTEXT"); + brokerProps.put("listeners", "PLAINTEXT://localhost:0,SASL_PLAINTEXT://localhost:0"); + brokerProps.put("listener.name.sasl_plaintext.plain.sasl.jaas.config", + "org.apache.kafka.common.security.plain.PlainLoginModule required " + + "username=\"super\" " + + "password=\"super_pwd\" " + + "user_super=\"super_pwd\";"); + brokerProps.put("super.users", "User:super"); + + Map superUserClientConfig = new HashMap<>(); + superUserClientConfig.put("sasl.mechanism", "PLAIN"); + superUserClientConfig.put("security.protocol", "SASL_PLAINTEXT"); + superUserClientConfig.put("sasl.jaas.config", + "org.apache.kafka.common.security.plain.PlainLoginModule required " + + "username=\"super\" " + + "password=\"super_pwd\";"); + workerProps.putAll(superUserClientConfig); + + connectBuilder.brokerListener(ListenerName.forSecurityProtocol(SecurityProtocol.SASL_PLAINTEXT)); + startConnect(); + + String topic = "test-topic"; + Properties adminClientProperties = new Properties(); + superUserClientConfig.forEach(adminClientProperties::put); + adminClientProperties.put(BOOTSTRAP_SERVERS_CONFIG, + connect.kafka().bootstrapServers(ListenerName.forSecurityProtocol(SecurityProtocol.SASL_PLAINTEXT))); + connect.kafka().createTopic(topic, 3, 1, Collections.emptyMap(), adminClientProperties); + + Map props = new HashMap<>(); + int tasksMax = 2; // Use two tasks since single-task connectors don't require zombie fencing + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getName()); + props.put(TOPIC_CONFIG, topic); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(NAME_CONFIG, CONNECTOR_NAME); + props.put(TASKS_MAX_CONFIG, Integer.toString(tasksMax)); + superUserClientConfig.forEach((property, value) -> { + props.put(CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX + property, value); + props.put(CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX + property, value); + props.put(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX + property, value); + }); + + StartAndStopLatch connectorStart = connectorAndTaskStart(tasksMax); + + log.info("Bringing up connector with valid admin client credentials"); + connect.configureConnector(CONNECTOR_NAME, props); + assertConnectorStarted(connectorStart); + // Verify that the task has been able to start successfully + connect.assertions().assertConnectorAndExactlyNumTasksAreRunning(CONNECTOR_NAME, tasksMax, "Connector and task should have started successfully"); + + // Reconfigure the connector to point towards a broker port that won't be able to provide a principal for it, + // which should lead to an authorization exception since the User:ANONYMOUS principal can't do anything + superUserClientConfig.keySet().forEach(property -> + props.remove(CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX + property) + ); + props.put(CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX + "security.protocol", "PLAINTEXT"); + props.put(CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, + connect.kafka().bootstrapServers(ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT))); + log.info("Bringing up connector with invalid admin client credentials"); + connect.configureConnector(CONNECTOR_NAME, props); + + // Verify that the task has failed, and that the failure is visible to users via the REST API + connect.assertions().assertConnectorIsRunningAndTasksHaveFailed(CONNECTOR_NAME, tasksMax, "Task should have failed on startup"); + } + + @Test + public void testSeparateOffsetsTopic() throws Exception { + String globalOffsetsTopic = "cluster-global-offsets-topic"; + workerProps.put(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG, globalOffsetsTopic); + startConnect(); + EmbeddedKafkaCluster connectorTargetedCluster = new EmbeddedKafkaCluster(1, brokerProps); + try (Closeable clusterShutdown = connectorTargetedCluster::stop) { + connectorTargetedCluster.start(); + String topic = "test-topic"; + connectorTargetedCluster.createTopic(topic, 3); + + int recordsProduced = 100; + + Map props = new HashMap<>(); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getName()); + props.put(TASKS_MAX_CONFIG, "1"); + props.put(TOPIC_CONFIG, topic); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(NAME_CONFIG, CONNECTOR_NAME); + props.put(TRANSACTION_BOUNDARY_CONFIG, POLL.toString()); + props.put("messages.per.poll", Integer.toString(recordsProduced)); + props.put(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, connectorTargetedCluster.bootstrapServers()); + props.put(CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, connectorTargetedCluster.bootstrapServers()); + props.put(CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, connectorTargetedCluster.bootstrapServers()); + String offsetsTopic = CONNECTOR_NAME + "-offsets"; + props.put(OFFSETS_TOPIC_CONFIG, offsetsTopic); + + // expect all records to be consumed and committed by the connector + connectorHandle.expectedRecords(recordsProduced); + connectorHandle.expectedCommits(recordsProduced); + + // start a source connector + connect.configureConnector(CONNECTOR_NAME, props); + + log.info("Waiting for records to be provided to worker by task"); + // wait for the connector tasks to produce enough records + connectorHandle.awaitRecords(SOURCE_TASK_PRODUCE_TIMEOUT_MS); + + log.info("Waiting for records to be committed to Kafka by worker"); + // wait for the connector tasks to commit enough records + connectorHandle.awaitCommits(TimeUnit.MINUTES.toMillis(1)); + + // consume all records from the source topic or fail, to ensure that they were correctly produced + int recordNum = connectorTargetedCluster + .consume( + recordsProduced, + TimeUnit.MINUTES.toMillis(1), + Collections.singletonMap(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"), + "test-topic") + .count(); + assertTrue("Not enough records produced by source connector. Expected at least: " + recordsProduced + " + but got " + recordNum, + recordNum >= recordsProduced); + + // also consume from the connector's dedicated offsets topic; just need to read one offset record + ConsumerRecord offsetRecord = connectorTargetedCluster + .consume( + 1, + TimeUnit.MINUTES.toMillis(1), + Collections.singletonMap(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"), + offsetsTopic + ).iterator().next(); + long seqno = parseAndAssertOffsetForSingleTask(offsetRecord); + assertEquals("Offset commits should occur on connector-defined poll boundaries, which happen every " + recordsProduced + " records", + 0, seqno % recordsProduced); + + // also consume from the cluster's global offsets topic; again, just need to read one offset record + offsetRecord = connect.kafka() + .consume( + 1, + TimeUnit.MINUTES.toMillis(1), + globalOffsetsTopic + ).iterator().next(); + seqno = parseAndAssertOffsetForSingleTask(offsetRecord); + assertEquals("Offset commits should occur on connector-defined poll boundaries, which happen every " + recordsProduced + " records", + 0, seqno % recordsProduced); + + // Delete the connector before shutting down the Kafka cluster it's targeting + connect.deleteConnector(CONNECTOR_NAME); + } + } + + @Test + public void testPotentialDeadlockWhenProducingToOffsetsTopic() throws Exception { + connectBuilder.numWorkers(1); + startConnect(); + + String topic = "test-topic"; + connect.kafka().createTopic(topic, 3); + + int recordsProduced = 100; + + Map props = new HashMap<>(); + // See below; this connector does nothing except request offsets from the worker in SourceTask::poll + // and then return a single record targeted at its offsets topic + props.put(CONNECTOR_CLASS_CONFIG, NaughtyConnector.class.getName()); + props.put(TASKS_MAX_CONFIG, "1"); + props.put(NAME_CONFIG, CONNECTOR_NAME); + props.put(TRANSACTION_BOUNDARY_CONFIG, INTERVAL.toString()); + props.put("messages.per.poll", Integer.toString(recordsProduced)); + props.put(OFFSETS_TOPIC_CONFIG, "whoops"); + + // start a source connector + connect.configureConnector(CONNECTOR_NAME, props); + + connect.assertions().assertConnectorIsRunningAndTasksHaveFailed( + CONNECTOR_NAME, 1, "Task should have failed after trying to produce to its own offsets topic"); + } + + private ConfigInfo findConfigInfo(String property, ConfigInfos validationResult) { + return validationResult.values().stream() + .filter(info -> property.equals(info.configKey().name())) + .findAny() + .orElseThrow(() -> new AssertionError("Failed to find configuration validation result for property '" + property + "'")); + } + + @SuppressWarnings("unchecked") + private long parseAndAssertOffsetForSingleTask(ConsumerRecord offsetRecord) { + JsonConverter offsetsConverter = new JsonConverter(); + // The JSON converter behaves identically for keys and values. If that ever changes, we may need to update this test to use + // separate converter instances. + + offsetsConverter.configure(Collections.singletonMap(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, "false"), false); + Object keyObject = offsetsConverter.toConnectData("topic name is not used by converter", offsetRecord.key()).value(); + Object valueObject = offsetsConverter.toConnectData("topic name is not used by converter", offsetRecord.value()).value(); + + assertNotNull("Offset value should not be null", valueObject); + + assertEquals("Serialized source partition should match expected format", + Arrays.asList(CONNECTOR_NAME, MonitorableSourceConnector.sourcePartition(MonitorableSourceConnector.taskId(CONNECTOR_NAME, 0))), + keyObject); + + Map value = assertAndCast(valueObject, Map.class, "Value"); + + Object seqnoObject = value.get("saved"); + assertNotNull("Serialized source offset should contain 'seqno' field from MonitorableSourceConnector", seqnoObject); + return assertAndCast(seqnoObject, Long.class, "Seqno offset field"); + } + + private long parseAndAssertValueForSingleTask(ConsumerRecord sourceRecord) { + assertNotNull("Record key should not be null", sourceRecord.key()); + assertNotNull("Record value should not be null", sourceRecord.value()); + + String key = new String(sourceRecord.key()); + String value = new String(sourceRecord.value()); + + String taskId = MonitorableSourceConnector.taskId(CONNECTOR_NAME, 0); + String keyPrefix = "key-" + taskId + "-"; + String valuePrefix = "value-" + taskId + "-"; + + assertTrue("Key should start with \"" + keyPrefix + "\"", key.startsWith(keyPrefix)); + assertTrue("Value should start with \"" + valuePrefix + "\"", value.startsWith(valuePrefix)); + + String keySeqno = key.substring(keyPrefix.length()); + String valueSeqno = value.substring(valuePrefix.length()); + + assertEquals("Seqnos for key and value should match", keySeqno, valueSeqno); + + return Long.parseLong(keySeqno); + } + + @SuppressWarnings("unchecked") + private static T assertAndCast(Object o, Class klass, String objectDescription) { + String className = o == null ? "null" : o.getClass().getName(); + assertTrue(objectDescription + " should be " + klass.getName() + "; was " + className + " instead", klass.isInstance(o)); + return (T) o; + } + + /** + * Clear all existing task handles for the connector, then preemptively create {@code numTasks} many task handles for it, + * and return a {@link StartAndStopLatch} that can be used to {@link StartAndStopLatch#await(long, TimeUnit) await} + * the startup of that connector and the expected number of tasks. + * @param numTasks the number of tasks that should be started + * @return a {@link StartAndStopLatch} that will block until the connector and the expected number of tasks have started + */ + private StartAndStopLatch connectorAndTaskStart(int numTasks) { + connectorHandle.clearTasks(); + IntStream.range(0, numTasks) + .mapToObj(i -> MonitorableSourceConnector.taskId(CONNECTOR_NAME, i)) + .forEach(connectorHandle::taskHandle); + return connectorHandle.expectedStarts(1, true); + } + + private void assertConnectorStarted(StartAndStopLatch connectorStart) throws InterruptedException { + assertTrue("Connector and tasks did not finish startup in time", + connectorStart.await( + EmbeddedConnectClusterAssertions.CONNECTOR_SETUP_DURATION_MS, + TimeUnit.MILLISECONDS + ) + ); + } + + private void assertProducersAreFencedOnReconfiguration( + int currentNumTasks, + int newNumTasks, + String topic, + Map baseConnectorProps) throws InterruptedException { + + // create a collection of producers that simulate the producers used for the existing tasks + List> producers = IntStream.range(0, currentNumTasks) + .mapToObj(i -> Worker.transactionalId(CLUSTER_GROUP_ID, CONNECTOR_NAME, i)) + .map(this::transactionalProducer) + .collect(Collectors.toList()); + + producers.forEach(KafkaProducer::initTransactions); + + // reconfigure the connector with a new number of tasks + StartAndStopLatch connectorStart = connectorAndTaskStart(newNumTasks); + baseConnectorProps.put(TASKS_MAX_CONFIG, Integer.toString(newNumTasks)); + log.info("Reconfiguring connector from {} tasks to {}", currentNumTasks, newNumTasks); + connect.configureConnector(CONNECTOR_NAME, baseConnectorProps); + assertConnectorStarted(connectorStart); + + // validate that the old producers were fenced out + producers.forEach(producer -> assertTransactionalProducerIsFenced(producer, topic)); + } + + private KafkaProducer transactionalProducer(String transactionalId) { + Map transactionalProducerProps = new HashMap<>(); + transactionalProducerProps.put(ENABLE_IDEMPOTENCE_CONFIG, true); + transactionalProducerProps.put(TRANSACTIONAL_ID_CONFIG, transactionalId); + return connect.kafka().createProducer(transactionalProducerProps); + } + + private void assertTransactionalProducerIsFenced(KafkaProducer producer, String topic) { + producer.beginTransaction(); + assertThrows("Producer should be fenced out", + ProducerFencedException.class, + () -> { + producer.send(new ProducerRecord<>(topic, new byte[] {69}, new byte[] {96})); + producer.commitTransaction(); + } + ); + producer.close(Duration.ZERO); + } + + public static class NaughtyConnector extends SourceConnector { + private Map props; + + @Override + public void start(Map props) { + this.props = props; + } + + @Override + public Class taskClass() { + return NaughtyTask.class; + } + + @Override + public List> taskConfigs(int maxTasks) { + return IntStream.range(0, maxTasks).mapToObj(i -> props).collect(Collectors.toList()); + } + + @Override + public void stop() { + } + + @Override + public ConfigDef config() { + return new ConfigDef(); + } + + @Override + public String version() { + return "none"; + } + } + + public static class NaughtyTask extends SourceTask { + private String topic; + + @Override + public void start(Map props) { + if (!props.containsKey(OFFSETS_TOPIC_CONFIG)) { + throw new ConnectException("No offsets topic"); + } + this.topic = props.get(OFFSETS_TOPIC_CONFIG); + } + + @Override + public List poll() { + // Request a read to the end of the offsets topic + context.offsetStorageReader().offset(Collections.singletonMap("", null)); + // Produce a record to the offsets topic + return Collections.singletonList(new SourceRecord(null, null, topic, null, "", null, null)); + } + + @Override + public void stop() { + } + + @Override + public String version() { + return "none"; + } + } +} diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java index afd93257e584b..ce334a357ce3f 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/MonitorableSourceConnector.java @@ -20,8 +20,11 @@ import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.connect.connector.Task; import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.header.ConnectHeaders; import org.apache.kafka.connect.runtime.TestSourceConnector; +import org.apache.kafka.connect.source.ConnectorTransactionBoundaries; +import org.apache.kafka.connect.source.ExactlyOnceSupport; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; import org.apache.kafka.tools.ThroughputThrottler; @@ -32,6 +35,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -74,7 +78,7 @@ public List> taskConfigs(int maxTasks) { for (int i = 0; i < maxTasks; i++) { Map config = new HashMap<>(commonConfigs); config.put("connector.name", connectorName); - config.put("task.id", connectorName + "-" + i); + config.put("task.id", taskId(connectorName, i)); configs.add(config); } return configs; @@ -92,18 +96,55 @@ public ConfigDef config() { return new ConfigDef(); } + @Override + public ExactlyOnceSupport exactlyOnceSupport(Map connectorConfig) { + String supportLevel = connectorConfig.getOrDefault("exactly.once.support.level", "null").toLowerCase(Locale.ROOT); + switch (supportLevel) { + case "supported": + return ExactlyOnceSupport.SUPPORTED; + case "unsupported": + return ExactlyOnceSupport.UNSUPPORTED; + case "fail": + throw new ConnectException("\uD83D\uDE0E"); + default: + case "null": + return null; + } + } + + @Override + public ConnectorTransactionBoundaries canDefineTransactionBoundaries(Map connectorConfig) { + String supportLevel = connectorConfig.getOrDefault("custom.transaction.boundary.support", "null").toLowerCase(Locale.ROOT); + switch (supportLevel) { + case "supported": + return ConnectorTransactionBoundaries.SUPPORTED; + case "unsupported": + return ConnectorTransactionBoundaries.UNSUPPORTED; + case "fail": + throw new ConnectException("\uD83D\uDC83"); + default: + case "null": + return null; + } + } + + public static String taskId(String connectorName, int taskId) { + return connectorName + "-" + taskId; + } + public static class MonitorableSourceTask extends SourceTask { - private String connectorName; private String taskId; private String topicName; private TaskHandle taskHandle; private volatile boolean stopped; private long startingSeqno; private long seqno; - private long throughput; private int batchSize; private ThroughputThrottler throttler; + private long priorTransactionBoundary; + private long nextTransactionBoundary; + @Override public String version() { return "unknown"; @@ -112,21 +153,23 @@ public String version() { @Override public void start(Map props) { taskId = props.get("task.id"); - connectorName = props.get("connector.name"); topicName = props.getOrDefault(TOPIC_CONFIG, "sequential-topic"); - throughput = Long.valueOf(props.getOrDefault("throughput", "-1")); - batchSize = Integer.valueOf(props.getOrDefault("messages.per.poll", "1")); - taskHandle = RuntimeHandles.get().connectorHandle(connectorName).taskHandle(taskId); + batchSize = Integer.parseInt(props.getOrDefault("messages.per.poll", "1")); + taskHandle = RuntimeHandles.get().connectorHandle(props.get("connector.name")).taskHandle(taskId); Map offset = Optional.ofNullable( context.offsetStorageReader().offset(Collections.singletonMap("task.id", taskId))) .orElse(Collections.emptyMap()); startingSeqno = Optional.ofNullable((Long) offset.get("saved")).orElse(0L); + seqno = startingSeqno; log.info("Started {} task {} with properties {}", this.getClass().getSimpleName(), taskId, props); - throttler = new ThroughputThrottler(throughput, System.currentTimeMillis()); + throttler = new ThroughputThrottler(Long.parseLong(props.getOrDefault("throughput", "-1")), System.currentTimeMillis()); taskHandle.recordTaskStart(); + priorTransactionBoundary = 0; + nextTransactionBoundary = 1; if (Boolean.parseBoolean(props.getOrDefault("task-" + taskId + ".start.inject.error", "false"))) { throw new RuntimeException("Injecting errors during task start"); } + calculateNextBoundary(); } @Override @@ -136,19 +179,24 @@ public List poll() { throttler.throttle(); } taskHandle.record(batchSize); - log.info("Returning batch of {} records", batchSize); + log.trace("Returning batch of {} records", batchSize); return LongStream.range(0, batchSize) - .mapToObj(i -> new SourceRecord( - Collections.singletonMap("task.id", taskId), - Collections.singletonMap("saved", ++seqno), - topicName, - null, - Schema.STRING_SCHEMA, - "key-" + taskId + "-" + seqno, - Schema.STRING_SCHEMA, - "value-" + taskId + "-" + seqno, - null, - new ConnectHeaders().addLong("header-" + seqno, seqno))) + .mapToObj(i -> { + seqno++; + SourceRecord record = new SourceRecord( + sourcePartition(taskId), + sourceOffset(seqno), + topicName, + null, + Schema.STRING_SCHEMA, + "key-" + taskId + "-" + seqno, + Schema.STRING_SCHEMA, + "value-" + taskId + "-" + seqno, + null, + new ConnectHeaders().addLong("header-" + seqno, seqno)); + maybeDefineTransactionBoundary(record); + return record; + }) .collect(Collectors.toList()); } return null; @@ -172,5 +220,35 @@ public void stop() { stopped = true; taskHandle.recordTaskStop(); } + + private void calculateNextBoundary() { + while (nextTransactionBoundary <= seqno) { + nextTransactionBoundary += priorTransactionBoundary; + priorTransactionBoundary = nextTransactionBoundary - priorTransactionBoundary; + } + } + + private void maybeDefineTransactionBoundary(SourceRecord record) { + if (context.transactionContext() == null || seqno != nextTransactionBoundary) { + return; + } + // If the transaction boundary ends on an even-numbered offset, abort it + // Otherwise, commit + boolean abort = nextTransactionBoundary % 2 == 0; + calculateNextBoundary(); + if (abort) { + context.transactionContext().abortTransaction(record); + } else { + context.transactionContext().commitTransaction(record); + } + } + } + + public static Map sourcePartition(String taskId) { + return Collections.singletonMap("task.id", taskId); + } + + public static Map sourceOffset(long seqno) { + return Collections.singletonMap("saved", seqno); } } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java index 360c4ae6a255b..a34460afe3315 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractHerderTest.java @@ -30,7 +30,6 @@ import org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy; import org.apache.kafka.connect.connector.policy.NoneConnectorClientConfigOverridePolicy; import org.apache.kafka.connect.connector.policy.PrincipalConnectorClientConfigOverridePolicy; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; import org.apache.kafka.connect.runtime.isolation.PluginDesc; import org.apache.kafka.connect.runtime.isolation.Plugins; import org.apache.kafka.connect.runtime.rest.entities.ConfigInfo; @@ -41,6 +40,7 @@ import org.apache.kafka.connect.runtime.rest.errors.BadRequestException; import org.apache.kafka.connect.source.SourceConnector; import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.storage.ConfigBackingStore; import org.apache.kafka.connect.storage.StatusBackingStore; import org.apache.kafka.connect.transforms.Transformation; @@ -51,6 +51,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.easymock.PowerMock; +import org.powermock.api.easymock.annotation.Mock; import org.powermock.api.easymock.annotation.MockStrict; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @@ -126,10 +127,10 @@ public class AbstractHerderTest { } private static final ClusterConfigState SNAPSHOT = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), - TASK_CONFIGS_MAP, Collections.emptySet()); + TASK_CONFIGS_MAP, Collections.emptyMap(), Collections.emptyMap(), Collections.emptySet(), Collections.emptySet()); private static final ClusterConfigState SNAPSHOT_NO_TASKS = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), - Collections.emptyMap(), Collections.emptySet()); + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptySet(), Collections.emptySet()); private final String workerId = "workerId"; private final String kafkaClusterId = "I4ZmrWqfT2e-upky_4fdPA"; @@ -139,7 +140,7 @@ public class AbstractHerderTest { @MockStrict private Worker worker; @MockStrict private WorkerConfigTransformer transformer; - @MockStrict private Plugins plugins; + @Mock private Plugins plugins; @MockStrict private ClassLoader classLoader; @MockStrict private ConfigBackingStore configStore; @MockStrict private StatusBackingStore statusStore; @@ -433,13 +434,18 @@ public void testConfigValidationMissingName() throws Throwable { // We expect there to be errors due to the missing name and .... Note that these assertions depend heavily on // the config fields for SourceConnectorConfig, but we expect these to change rarely. assertEquals(TestSourceConnector.class.getName(), result.name()); - assertEquals(Arrays.asList(ConnectorConfig.COMMON_GROUP, ConnectorConfig.TRANSFORMS_GROUP, - ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, SourceConnectorConfig.TOPIC_CREATION_GROUP), result.groups()); + assertEquals( + Arrays.asList( + ConnectorConfig.COMMON_GROUP, ConnectorConfig.TRANSFORMS_GROUP, + ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, + SourceConnectorConfig.TOPIC_CREATION_GROUP, SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_GROUP, + SourceConnectorConfig.OFFSETS_TOPIC_GROUP), + result.groups()); assertEquals(2, result.errorCount()); Map infos = result.values().stream() .collect(Collectors.toMap(info -> info.configKey().name(), Function.identity())); - // Base connector config has 14 fields, connector's configs add 2 - assertEquals(17, infos.size()); + // Base connector config has 14 fields, connector's configs add 7 + assertEquals(21, infos.size()); // Missing name should generate an error assertEquals(ConnectorConfig.NAME_CONFIG, infos.get(ConnectorConfig.NAME_CONFIG).configValue().name()); @@ -529,6 +535,8 @@ public void testConfigValidationTransformsExtendResults() throws Throwable { ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, SourceConnectorConfig.TOPIC_CREATION_GROUP, + SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_GROUP, + SourceConnectorConfig.OFFSETS_TOPIC_GROUP, "Transforms: xformA", "Transforms: xformB" ); @@ -536,7 +544,7 @@ public void testConfigValidationTransformsExtendResults() throws Throwable { assertEquals(2, result.errorCount()); Map infos = result.values().stream() .collect(Collectors.toMap(info -> info.configKey().name(), Function.identity())); - assertEquals(22, infos.size()); + assertEquals(26, infos.size()); // Should get 2 type fields from the transforms, first adds its own config since it has a valid class assertEquals("transforms.xformA.type", infos.get("transforms.xformA.type").configValue().name()); @@ -588,6 +596,8 @@ public void testConfigValidationPredicatesExtendResults() { ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, SourceConnectorConfig.TOPIC_CREATION_GROUP, + SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_GROUP, + SourceConnectorConfig.OFFSETS_TOPIC_GROUP, "Transforms: xformA", "Predicates: predX", "Predicates: predY" @@ -596,7 +606,7 @@ public void testConfigValidationPredicatesExtendResults() { assertEquals(2, result.errorCount()); Map infos = result.values().stream() .collect(Collectors.toMap(info -> info.configKey().name(), Function.identity())); - assertEquals(24, infos.size()); + assertEquals(28, infos.size()); // Should get 2 type fields from the transforms, first adds its own config since it has a valid class assertEquals("transforms.xformA.type", infos.get("transforms.xformA.type").configValue().name()); @@ -657,12 +667,14 @@ public void testConfigValidationPrincipalOnlyOverride() throws Throwable { ConnectorConfig.TRANSFORMS_GROUP, ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, - SourceConnectorConfig.TOPIC_CREATION_GROUP + SourceConnectorConfig.TOPIC_CREATION_GROUP, + SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_GROUP, + SourceConnectorConfig.OFFSETS_TOPIC_GROUP ); assertEquals(expectedGroups, result.groups()); assertEquals(1, result.errorCount()); - // Base connector config has 14 fields, connector's configs add 2, and 2 producer overrides - assertEquals(19, result.values().size()); + // Base connector config has 14 fields, connector's configs add 7, and 2 producer overrides + assertEquals(23, result.values().size()); assertTrue(result.values().stream().anyMatch( configInfo -> ackConfigKey.equals(configInfo.configValue().name()) && !configInfo.configValue().errors().isEmpty())); assertTrue(result.values().stream().anyMatch( diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTaskTest.java new file mode 100644 index 0000000000000..59868cedfa1a1 --- /dev/null +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTaskTest.java @@ -0,0 +1,842 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.runtime; + +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.InvalidRecordException; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.TopicPartitionInfo; +import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaAndValue; +import org.apache.kafka.connect.errors.ConnectException; +import org.apache.kafka.connect.errors.RetriableException; +import org.apache.kafka.connect.header.ConnectHeaders; +import org.apache.kafka.connect.integration.MonitorableSourceConnector; +import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperatorTest; +import org.apache.kafka.connect.runtime.isolation.Plugins; +import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.storage.CloseableOffsetStorageReader; +import org.apache.kafka.connect.storage.ConnectorOffsetBackingStore; +import org.apache.kafka.connect.storage.Converter; +import org.apache.kafka.connect.storage.HeaderConverter; +import org.apache.kafka.connect.storage.OffsetStorageWriter; +import org.apache.kafka.connect.storage.StatusBackingStore; +import org.apache.kafka.connect.storage.StringConverter; +import org.apache.kafka.connect.util.ConnectorTaskId; +import org.apache.kafka.connect.util.TopicAdmin; +import org.apache.kafka.connect.util.TopicCreationGroup; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.easymock.IAnswer; +import org.easymock.IExpectationSetters; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.api.easymock.annotation.Mock; +import org.powermock.api.easymock.annotation.MockStrict; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Future; +import java.util.concurrent.TimeoutException; + +import static org.apache.kafka.connect.integration.MonitorableSourceConnector.TOPIC_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.SourceConnectorConfig.TOPIC_CREATION_GROUPS_CONFIG; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.DEFAULT_TOPIC_CREATION_PREFIX; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.EXCLUDE_REGEX_CONFIG; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.INCLUDE_REGEX_CONFIG; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.PARTITIONS_CONFIG; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.REPLICATION_FACTOR_CONFIG; +import static org.apache.kafka.connect.runtime.WorkerConfig.TOPIC_CREATION_ENABLE_CONFIG; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +@PowerMockIgnore({"javax.management.*", + "org.apache.log4j.*"}) +@RunWith(PowerMockRunner.class) +public class AbstractWorkerSourceTaskTest { + + private static final String TOPIC = "topic"; + private static final String OTHER_TOPIC = "other-topic"; + private static final Map PARTITION = Collections.singletonMap("key", "partition".getBytes()); + private static final Map OFFSET = Collections.singletonMap("key", 12); + + // Connect-format data + private static final Schema KEY_SCHEMA = Schema.INT32_SCHEMA; + private static final Integer KEY = -1; + private static final Schema RECORD_SCHEMA = Schema.INT64_SCHEMA; + private static final Long RECORD = 12L; + // Serialized data. The actual format of this data doesn't matter -- we just want to see that the right version + // is used in the right place. + private static final byte[] SERIALIZED_KEY = "converted-key".getBytes(); + private static final byte[] SERIALIZED_RECORD = "converted-record".getBytes(); + + @Mock private SourceTask sourceTask; + @Mock private TopicAdmin admin; + @Mock private KafkaProducer producer; + @Mock private Converter keyConverter; + @Mock private Converter valueConverter; + @Mock private HeaderConverter headerConverter; + @Mock private TransformationChain transformationChain; + @Mock private CloseableOffsetStorageReader offsetReader; + @Mock private OffsetStorageWriter offsetWriter; + @Mock private ConnectorOffsetBackingStore offsetStore; + @Mock private StatusBackingStore statusBackingStore; + @Mock private WorkerSourceTaskContext sourceTaskContext; + @MockStrict private TaskStatus.Listener statusListener; + + private final ConnectorTaskId taskId = new ConnectorTaskId("job", 0); + private final ConnectorTaskId taskId1 = new ConnectorTaskId("job", 1); + + private Plugins plugins; + private WorkerConfig config; + private SourceConnectorConfig sourceConfig; + private MockConnectMetrics metrics = new MockConnectMetrics(); + private Capture producerCallbacks; + + private AbstractWorkerSourceTask workerTask; + + @Before + public void setup() { + Map workerProps = workerProps(); + plugins = new Plugins(workerProps); + config = new StandaloneConfig(workerProps); + sourceConfig = new SourceConnectorConfig(plugins, sourceConnectorPropsWithGroups(TOPIC), true); + producerCallbacks = EasyMock.newCapture(); + metrics = new MockConnectMetrics(); + } + + private Map workerProps() { + Map props = new HashMap<>(); + props.put("key.converter", "org.apache.kafka.connect.json.JsonConverter"); + props.put("value.converter", "org.apache.kafka.connect.json.JsonConverter"); + props.put("offset.storage.file.filename", "/tmp/connect.offsets"); + props.put(TOPIC_CREATION_ENABLE_CONFIG, "true"); + return props; + } + + private Map sourceConnectorPropsWithGroups(String topic) { + // setup up props for the source connector + Map props = new HashMap<>(); + props.put("name", "foo-connector"); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getSimpleName()); + props.put(TASKS_MAX_CONFIG, String.valueOf(1)); + props.put(TOPIC_CONFIG, topic); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(TOPIC_CREATION_GROUPS_CONFIG, String.join(",", "foo", "bar")); + props.put(DEFAULT_TOPIC_CREATION_PREFIX + REPLICATION_FACTOR_CONFIG, String.valueOf(1)); + props.put(DEFAULT_TOPIC_CREATION_PREFIX + PARTITIONS_CONFIG, String.valueOf(1)); + props.put(SourceConnectorConfig.TOPIC_CREATION_PREFIX + "foo" + "." + INCLUDE_REGEX_CONFIG, topic); + props.put(SourceConnectorConfig.TOPIC_CREATION_PREFIX + "bar" + "." + INCLUDE_REGEX_CONFIG, ".*"); + props.put(SourceConnectorConfig.TOPIC_CREATION_PREFIX + "bar" + "." + EXCLUDE_REGEX_CONFIG, topic); + return props; + } + + @After + public void tearDown() { + if (metrics != null) metrics.stop(); + } + + @Test + public void testMetricsGroup() { + AbstractWorkerSourceTask.SourceTaskMetricsGroup group = new AbstractWorkerSourceTask.SourceTaskMetricsGroup(taskId, metrics); + AbstractWorkerSourceTask.SourceTaskMetricsGroup group1 = new AbstractWorkerSourceTask.SourceTaskMetricsGroup(taskId1, metrics); + for (int i = 0; i != 10; ++i) { + group.recordPoll(100, 1000 + i * 100); + group.recordWrite(10); + } + for (int i = 0; i != 20; ++i) { + group1.recordPoll(100, 1000 + i * 100); + group1.recordWrite(10); + } + assertEquals(1900.0, metrics.currentMetricValueAsDouble(group.metricGroup(), "poll-batch-max-time-ms"), 0.001d); + assertEquals(1450.0, metrics.currentMetricValueAsDouble(group.metricGroup(), "poll-batch-avg-time-ms"), 0.001d); + assertEquals(33.333, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-poll-rate"), 0.001d); + assertEquals(1000, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-poll-total"), 0.001d); + assertEquals(3.3333, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-write-rate"), 0.001d); + assertEquals(100, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-write-total"), 0.001d); + assertEquals(900.0, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-active-count"), 0.001d); + + // Close the group + group.close(); + + for (MetricName metricName : group.metricGroup().metrics().metrics().keySet()) { + // Metrics for this group should no longer exist + assertFalse(group.metricGroup().groupId().includes(metricName)); + } + // Sensors for this group should no longer exist + assertNull(group.metricGroup().metrics().getSensor("sink-record-read")); + assertNull(group.metricGroup().metrics().getSensor("sink-record-send")); + assertNull(group.metricGroup().metrics().getSensor("sink-record-active-count")); + assertNull(group.metricGroup().metrics().getSensor("partition-count")); + assertNull(group.metricGroup().metrics().getSensor("offset-seq-number")); + assertNull(group.metricGroup().metrics().getSensor("offset-commit-completion")); + assertNull(group.metricGroup().metrics().getSensor("offset-commit-completion-skip")); + assertNull(group.metricGroup().metrics().getSensor("put-batch-time")); + + assertEquals(2900.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "poll-batch-max-time-ms"), 0.001d); + assertEquals(1950.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "poll-batch-avg-time-ms"), 0.001d); + assertEquals(66.667, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-poll-rate"), 0.001d); + assertEquals(2000, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-poll-total"), 0.001d); + assertEquals(6.667, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-write-rate"), 0.001d); + assertEquals(200, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-write-total"), 0.001d); + assertEquals(1800.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-active-count"), 0.001d); + } + + @Test + public void testSendRecordsConvertsData() throws Exception { + createWorkerTask(); + + List records = new ArrayList<>(); + // Can just use the same record for key and value + records.add(new SourceRecord(PARTITION, OFFSET, "topic", null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD)); + + Capture> sent = expectSendRecordAnyTimes(); + + expectTopicCreation(TOPIC); + + PowerMock.replayAll(); + + workerTask.toSend = records; + workerTask.sendRecords(); + assertEquals(SERIALIZED_KEY, sent.getValue().key()); + assertEquals(SERIALIZED_RECORD, sent.getValue().value()); + + PowerMock.verifyAll(); + } + + @Test + public void testSendRecordsPropagatesTimestamp() throws Exception { + final Long timestamp = System.currentTimeMillis(); + + createWorkerTask(); + + List records = Collections.singletonList( + new SourceRecord(PARTITION, OFFSET, "topic", null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD, timestamp) + ); + + Capture> sent = expectSendRecordAnyTimes(); + + expectTopicCreation(TOPIC); + + PowerMock.replayAll(); + + workerTask.toSend = records; + workerTask.sendRecords(); + assertEquals(timestamp, sent.getValue().timestamp()); + + PowerMock.verifyAll(); + } + + @Test + public void testSendRecordsCorruptTimestamp() throws Exception { + final Long timestamp = -3L; + createWorkerTask(); + + List records = Collections.singletonList( + new SourceRecord(PARTITION, OFFSET, "topic", null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD, timestamp) + ); + + Capture> sent = expectSendRecordAnyTimes(); + + PowerMock.replayAll(); + + workerTask.toSend = records; + assertThrows(InvalidRecordException.class, workerTask::sendRecords); + assertFalse(sent.hasCaptured()); + + PowerMock.verifyAll(); + } + + @Test + public void testSendRecordsNoTimestamp() throws Exception { + final Long timestamp = -1L; + createWorkerTask(); + + List records = Collections.singletonList( + new SourceRecord(PARTITION, OFFSET, "topic", null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD, timestamp) + ); + + Capture> sent = expectSendRecordAnyTimes(); + + expectTopicCreation(TOPIC); + + PowerMock.replayAll(); + + workerTask.toSend = records; + workerTask.sendRecords(); + assertNull(sent.getValue().timestamp()); + + PowerMock.verifyAll(); + } + + @Test + public void testHeaders() throws Exception { + Headers headers = new RecordHeaders(); + headers.add("header_key", "header_value".getBytes()); + + org.apache.kafka.connect.header.Headers connectHeaders = new ConnectHeaders(); + connectHeaders.add("header_key", new SchemaAndValue(Schema.STRING_SCHEMA, "header_value")); + + createWorkerTask(); + + List records = new ArrayList<>(); + records.add(new SourceRecord(PARTITION, OFFSET, TOPIC, null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD, null, connectHeaders)); + + expectTopicCreation(TOPIC); + + Capture> sent = expectSendRecord(TOPIC, true, headers); + + PowerMock.replayAll(); + + workerTask.toSend = records; + workerTask.sendRecords(); + assertEquals(SERIALIZED_KEY, sent.getValue().key()); + assertEquals(SERIALIZED_RECORD, sent.getValue().value()); + assertEquals(headers, sent.getValue().headers()); + + PowerMock.verifyAll(); + } + + @Test + public void testHeadersWithCustomConverter() throws Exception { + StringConverter stringConverter = new StringConverter(); + TestConverterWithHeaders testConverter = new TestConverterWithHeaders(); + + createWorkerTask(stringConverter, testConverter, stringConverter); + + List records = new ArrayList<>(); + + String stringA = "Árvíztűrő tükörfúrógép"; + org.apache.kafka.connect.header.Headers headersA = new ConnectHeaders(); + String encodingA = "latin2"; + headersA.addString("encoding", encodingA); + + records.add(new SourceRecord(PARTITION, OFFSET, "topic", null, Schema.STRING_SCHEMA, "a", Schema.STRING_SCHEMA, stringA, null, headersA)); + + String stringB = "Тестовое сообщение"; + org.apache.kafka.connect.header.Headers headersB = new ConnectHeaders(); + String encodingB = "koi8_r"; + headersB.addString("encoding", encodingB); + + records.add(new SourceRecord(PARTITION, OFFSET, "topic", null, Schema.STRING_SCHEMA, "b", Schema.STRING_SCHEMA, stringB, null, headersB)); + + expectTopicCreation(TOPIC); + + Capture> sentRecordA = expectSendRecord(TOPIC, false, null); + Capture> sentRecordB = expectSendRecord(TOPIC, false, null); + + PowerMock.replayAll(); + + workerTask.toSend = records; + workerTask.sendRecords(); + + assertEquals(ByteBuffer.wrap("a".getBytes()), ByteBuffer.wrap(sentRecordA.getValue().key())); + assertEquals( + ByteBuffer.wrap(stringA.getBytes(encodingA)), + ByteBuffer.wrap(sentRecordA.getValue().value()) + ); + assertEquals(encodingA, new String(sentRecordA.getValue().headers().lastHeader("encoding").value())); + + assertEquals(ByteBuffer.wrap("b".getBytes()), ByteBuffer.wrap(sentRecordB.getValue().key())); + assertEquals( + ByteBuffer.wrap(stringB.getBytes(encodingB)), + ByteBuffer.wrap(sentRecordB.getValue().value()) + ); + assertEquals(encodingB, new String(sentRecordB.getValue().headers().lastHeader("encoding").value())); + + PowerMock.verifyAll(); + } + + @Test + public void testTopicCreateWhenTopicExists() throws Exception { + createWorkerTask(); + + SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + expectPreliminaryCalls(); + TopicPartitionInfo topicPartitionInfo = new TopicPartitionInfo(0, null, Collections.emptyList(), Collections.emptyList()); + TopicDescription topicDesc = new TopicDescription(TOPIC, false, Collections.singletonList(topicPartitionInfo)); + EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.singletonMap(TOPIC, topicDesc)); + + expectSendRecord(); + expectSendRecord(); + + PowerMock.replayAll(); + + workerTask.toSend = Arrays.asList(record1, record2); + workerTask.sendRecords(); + } + + @Test + public void testSendRecordsTopicDescribeRetries() throws Exception { + createWorkerTask(); + + SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + expectPreliminaryCalls(); + // First round - call to describe the topic times out + EasyMock.expect(admin.describeTopics(TOPIC)) + .andThrow(new RetriableException(new TimeoutException("timeout"))); + + // Second round - calls to describe and create succeed + expectTopicCreation(TOPIC); + // Exactly two records are sent + expectSendRecord(); + expectSendRecord(); + + PowerMock.replayAll(); + + workerTask.toSend = Arrays.asList(record1, record2); + workerTask.sendRecords(); + assertEquals(Arrays.asList(record1, record2), workerTask.toSend); + + // Next they all succeed + workerTask.sendRecords(); + assertNull(workerTask.toSend); + } + + @Test + public void testSendRecordsTopicCreateRetries() throws Exception { + createWorkerTask(); + + SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + // First call to describe the topic times out + expectPreliminaryCalls(); + EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap()); + Capture newTopicCapture = EasyMock.newCapture(); + EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))) + .andThrow(new RetriableException(new TimeoutException("timeout"))); + + // Second round + expectTopicCreation(TOPIC); + expectSendRecord(); + expectSendRecord(); + + PowerMock.replayAll(); + + workerTask.toSend = Arrays.asList(record1, record2); + workerTask.sendRecords(); + assertEquals(Arrays.asList(record1, record2), workerTask.toSend); + + // Next they all succeed + workerTask.sendRecords(); + assertNull(workerTask.toSend); + } + + @Test + public void testSendRecordsTopicDescribeRetriesMidway() throws Exception { + createWorkerTask(); + + // Differentiate only by Kafka partition so we can reuse conversion expectations + SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record3 = new SourceRecord(PARTITION, OFFSET, OTHER_TOPIC, 3, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + // First round + expectPreliminaryCalls(OTHER_TOPIC); + expectTopicCreation(TOPIC); + expectSendRecord(); + expectSendRecord(); + + // First call to describe the topic times out + EasyMock.expect(admin.describeTopics(OTHER_TOPIC)) + .andThrow(new RetriableException(new TimeoutException("timeout"))); + + // Second round + expectTopicCreation(OTHER_TOPIC); + expectSendRecord(OTHER_TOPIC, false, emptyHeaders()); + + PowerMock.replayAll(); + + // Try to send 3, make first pass, second fail. Should save last two + workerTask.toSend = Arrays.asList(record1, record2, record3); + workerTask.sendRecords(); + assertEquals(Arrays.asList(record3), workerTask.toSend); + + // Next they all succeed + workerTask.sendRecords(); + assertNull(workerTask.toSend); + + PowerMock.verifyAll(); + } + + @Test + public void testSendRecordsTopicCreateRetriesMidway() throws Exception { + createWorkerTask(); + + // Differentiate only by Kafka partition so we can reuse conversion expectations + SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record3 = new SourceRecord(PARTITION, OFFSET, OTHER_TOPIC, 3, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + // First round + expectPreliminaryCalls(OTHER_TOPIC); + expectTopicCreation(TOPIC); + expectSendRecord(); + expectSendRecord(); + + EasyMock.expect(admin.describeTopics(OTHER_TOPIC)).andReturn(Collections.emptyMap()); + // First call to create the topic times out + Capture newTopicCapture = EasyMock.newCapture(); + EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))) + .andThrow(new RetriableException(new TimeoutException("timeout"))); + + // Second round + expectTopicCreation(OTHER_TOPIC); + expectSendRecord(OTHER_TOPIC, false, emptyHeaders()); + + PowerMock.replayAll(); + + // Try to send 3, make first pass, second fail. Should save last two + workerTask.toSend = Arrays.asList(record1, record2, record3); + workerTask.sendRecords(); + assertEquals(Arrays.asList(record3), workerTask.toSend); + + // Next they all succeed + workerTask.sendRecords(); + assertNull(workerTask.toSend); + + PowerMock.verifyAll(); + } + + @Test + public void testTopicDescribeFails() { + createWorkerTask(); + + SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + expectPreliminaryCalls(); + EasyMock.expect(admin.describeTopics(TOPIC)) + .andThrow(new ConnectException(new TopicAuthorizationException("unauthorized"))); + + PowerMock.replayAll(); + + workerTask.toSend = Arrays.asList(record1, record2); + assertThrows(ConnectException.class, workerTask::sendRecords); + } + + @Test + public void testTopicCreateFails() throws Exception { + createWorkerTask(); + + SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + expectPreliminaryCalls(); + EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap()); + + Capture newTopicCapture = EasyMock.newCapture(); + EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))) + .andThrow(new ConnectException(new TopicAuthorizationException("unauthorized"))); + + PowerMock.replayAll(); + + workerTask.toSend = Arrays.asList(record1, record2); + assertThrows(ConnectException.class, workerTask::sendRecords); + assertTrue(newTopicCapture.hasCaptured()); + } + + @Test + public void testTopicCreateFailsWithExceptionWhenCreateReturnsTopicNotCreatedOrFound() throws Exception { + createWorkerTask(); + + SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + expectPreliminaryCalls(); + EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap()); + + Capture newTopicCapture = EasyMock.newCapture(); + EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andReturn(TopicAdmin.EMPTY_CREATION); + + PowerMock.replayAll(); + + workerTask.toSend = Arrays.asList(record1, record2); + assertThrows(ConnectException.class, workerTask::sendRecords); + assertTrue(newTopicCapture.hasCaptured()); + } + + @Test + public void testTopicCreateSucceedsWhenCreateReturnsExistingTopicFound() throws Exception { + createWorkerTask(); + + SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + expectPreliminaryCalls(); + EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap()); + + Capture newTopicCapture = EasyMock.newCapture(); + EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andReturn(foundTopic(TOPIC)); + + expectSendRecord(); + expectSendRecord(); + + PowerMock.replayAll(); + + workerTask.toSend = Arrays.asList(record1, record2); + workerTask.sendRecords(); + } + + @Test + public void testTopicCreateSucceedsWhenCreateReturnsNewTopicFound() throws Exception { + createWorkerTask(); + + SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + expectPreliminaryCalls(); + EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap()); + + Capture newTopicCapture = EasyMock.newCapture(); + EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andReturn(createdTopic(TOPIC)); + + expectSendRecord(); + expectSendRecord(); + + PowerMock.replayAll(); + + workerTask.toSend = Arrays.asList(record1, record2); + workerTask.sendRecords(); + } + + private Capture> expectSendRecord( + String topic, + boolean anyTimes, + Headers headers + ) { + if (headers != null) + expectConvertHeadersAndKeyValue(topic, anyTimes, headers); + + expectApplyTransformationChain(anyTimes); + + Capture> sent = EasyMock.newCapture(); + + IExpectationSetters> expect = EasyMock.expect( + producer.send(EasyMock.capture(sent), EasyMock.capture(producerCallbacks))); + + IAnswer> expectResponse = () -> { + synchronized (producerCallbacks) { + for (Callback cb : producerCallbacks.getValues()) { + cb.onCompletion(new RecordMetadata(new TopicPartition("foo", 0), 0, 0, 0L, 0, 0), null); + } + producerCallbacks.reset(); + } + return null; + }; + + if (anyTimes) + expect.andStubAnswer(expectResponse); + else + expect.andAnswer(expectResponse); + + expectTaskGetTopic(anyTimes); + + return sent; + } + + private Capture> expectSendRecordAnyTimes() { + return expectSendRecord(TOPIC, true, emptyHeaders()); + } + + private Capture> expectSendRecord() { + return expectSendRecord(TOPIC, false, emptyHeaders()); + } + + private void expectTaskGetTopic(boolean anyTimes) { + final Capture connectorCapture = EasyMock.newCapture(); + final Capture topicCapture = EasyMock.newCapture(); + IExpectationSetters expect = EasyMock.expect(statusBackingStore.getTopic( + EasyMock.capture(connectorCapture), + EasyMock.capture(topicCapture))); + if (anyTimes) { + expect.andStubAnswer(() -> new TopicStatus( + topicCapture.getValue(), + new ConnectorTaskId(connectorCapture.getValue(), 0), + Time.SYSTEM.milliseconds())); + } else { + expect.andAnswer(() -> new TopicStatus( + topicCapture.getValue(), + new ConnectorTaskId(connectorCapture.getValue(), 0), + Time.SYSTEM.milliseconds())); + } + if (connectorCapture.hasCaptured() && topicCapture.hasCaptured()) { + assertEquals("job", connectorCapture.getValue()); + assertEquals(TOPIC, topicCapture.getValue()); + } + } + + private void expectTopicCreation(String topic) { + if (config.topicCreationEnable()) { + EasyMock.expect(admin.describeTopics(topic)).andReturn(Collections.emptyMap()); + Capture newTopicCapture = EasyMock.newCapture(); + EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andReturn(createdTopic(topic)); + } + } + + private TopicAdmin.TopicCreationResponse createdTopic(String topic) { + Set created = Collections.singleton(topic); + Set existing = Collections.emptySet(); + return new TopicAdmin.TopicCreationResponse(created, existing); + } + + private TopicAdmin.TopicCreationResponse foundTopic(String topic) { + Set created = Collections.emptySet(); + Set existing = Collections.singleton(topic); + return new TopicAdmin.TopicCreationResponse(created, existing); + } + + private void expectPreliminaryCalls() { + expectPreliminaryCalls(TOPIC); + } + + private void expectPreliminaryCalls(String topic) { + expectConvertHeadersAndKeyValue(topic, true, emptyHeaders()); + expectApplyTransformationChain(false); + PowerMock.expectLastCall(); + } + + private void expectConvertHeadersAndKeyValue(String topic, boolean anyTimes, Headers headers) { + for (Header header : headers) { + IExpectationSetters convertHeaderExpect = EasyMock.expect(headerConverter.fromConnectHeader(topic, header.key(), Schema.STRING_SCHEMA, new String(header.value()))); + if (anyTimes) + convertHeaderExpect.andStubReturn(header.value()); + else + convertHeaderExpect.andReturn(header.value()); + } + IExpectationSetters convertKeyExpect = EasyMock.expect(keyConverter.fromConnectData(topic, headers, KEY_SCHEMA, KEY)); + if (anyTimes) + convertKeyExpect.andStubReturn(SERIALIZED_KEY); + else + convertKeyExpect.andReturn(SERIALIZED_KEY); + IExpectationSetters convertValueExpect = EasyMock.expect(valueConverter.fromConnectData(topic, headers, RECORD_SCHEMA, RECORD)); + if (anyTimes) + convertValueExpect.andStubReturn(SERIALIZED_RECORD); + else + convertValueExpect.andReturn(SERIALIZED_RECORD); + } + + private void expectApplyTransformationChain(boolean anyTimes) { + final Capture recordCapture = EasyMock.newCapture(); + IExpectationSetters convertKeyExpect = EasyMock.expect(transformationChain.apply(EasyMock.capture(recordCapture))); + if (anyTimes) + convertKeyExpect.andStubAnswer(recordCapture::getValue); + else + convertKeyExpect.andAnswer(recordCapture::getValue); + } + + private RecordHeaders emptyHeaders() { + return new RecordHeaders(); + } + + private void createWorkerTask() { + createWorkerTask(keyConverter, valueConverter, headerConverter); + } + + private void createWorkerTask(Converter keyConverter, Converter valueConverter, HeaderConverter headerConverter) { + workerTask = new AbstractWorkerSourceTask( + taskId, sourceTask, statusListener, TargetState.STARTED, keyConverter, valueConverter, headerConverter, transformationChain, + sourceTaskContext, producer, admin, TopicCreationGroup.configuredGroups(sourceConfig), offsetReader, offsetWriter, offsetStore, + config, metrics, plugins.delegatingLoader(), Time.SYSTEM, RetryWithToleranceOperatorTest.NOOP_OPERATOR, + statusBackingStore, Runnable::run) { + @Override + protected void prepareToInitializeTask() { + } + + @Override + protected void prepareToEnterSendLoop() { + } + + @Override + protected void beginSendIteration() { + } + + @Override + protected void prepareToPollTask() { + } + + @Override + protected void recordDropped(SourceRecord record) { + } + + @Override + protected Optional prepareToSendRecord(SourceRecord sourceRecord, ProducerRecord producerRecord) { + return Optional.empty(); + } + + @Override + protected void recordDispatched(SourceRecord record) { + } + + @Override + protected void batchDispatched() { + } + + @Override + protected void recordSent(SourceRecord sourceRecord, ProducerRecord producerRecord, RecordMetadata recordMetadata) { + } + + @Override + protected void producerSendFailed(boolean synchronous, ProducerRecord producerRecord, SourceRecord preTransformRecord, Exception e) { + } + + @Override + protected void finalOffsetCommit(boolean failed) { + } + }; + + } + +} diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java index 4743dced429ac..4900205b8a38b 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java @@ -33,7 +33,7 @@ import org.apache.kafka.connect.errors.RetriableException; import org.apache.kafka.connect.integration.MonitorableSourceConnector; import org.apache.kafka.connect.json.JsonConverter; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.runtime.errors.ErrorHandlingMetrics; import org.apache.kafka.connect.runtime.errors.ErrorReporter; import org.apache.kafka.connect.runtime.errors.LogReporter; @@ -48,6 +48,7 @@ import org.apache.kafka.connect.sink.SinkTask; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.storage.ConnectorOffsetBackingStore; import org.apache.kafka.connect.storage.Converter; import org.apache.kafka.connect.storage.HeaderConverter; import org.apache.kafka.connect.storage.OffsetStorageReaderImpl; @@ -160,6 +161,8 @@ public class ErrorHandlingTaskTest { OffsetStorageReaderImpl offsetReader; @Mock OffsetStorageWriter offsetWriter; + @Mock + private ConnectorOffsetBackingStore offsetStore; private Capture rebalanceListener = EasyMock.newCapture(); @SuppressWarnings("unused") @@ -530,6 +533,12 @@ private void expectClose() { admin.close(EasyMock.anyObject(Duration.class)); EasyMock.expectLastCall(); + + offsetReader.close(); + EasyMock.expectLastCall(); + + offsetStore.stop(); + EasyMock.expectLastCall(); } private void expectTopicCreation(String topic) { @@ -590,7 +599,7 @@ private void createSourceTask(TargetState initialState, RetryWithToleranceOperat WorkerSourceTask.class, new String[]{"commitOffsets", "isStopping"}, taskId, sourceTask, statusListener, initialState, converter, converter, headerConverter, sourceTransforms, producer, admin, TopicCreationGroup.configuredGroups(sourceConfig), - offsetReader, offsetWriter, workerConfig, + offsetReader, offsetWriter, offsetStore, workerConfig, ClusterConfigState.EMPTY, metrics, pluginLoader, time, retryWithToleranceOperator, statusBackingStore, (Executor) Runnable::run); } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTaskTest.java new file mode 100644 index 0000000000000..f26a9b66391fa --- /dev/null +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ExactlyOnceWorkerSourceTaskTest.java @@ -0,0 +1,1330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.runtime; + +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.errors.RecordTooLargeException; +import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.errors.ConnectException; +import org.apache.kafka.connect.integration.MonitorableSourceConnector; +import org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup; +import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperatorTest; +import org.apache.kafka.connect.runtime.isolation.Plugins; +import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.source.SourceTaskContext; +import org.apache.kafka.connect.source.TransactionContext; +import org.apache.kafka.connect.storage.CloseableOffsetStorageReader; +import org.apache.kafka.connect.storage.ClusterConfigState; +import org.apache.kafka.connect.storage.ConnectorOffsetBackingStore; +import org.apache.kafka.connect.storage.Converter; +import org.apache.kafka.connect.storage.HeaderConverter; +import org.apache.kafka.connect.storage.OffsetStorageWriter; +import org.apache.kafka.connect.storage.StatusBackingStore; +import org.apache.kafka.connect.storage.StringConverter; +import org.apache.kafka.connect.util.Callback; +import org.apache.kafka.connect.util.ConnectorTaskId; +import org.apache.kafka.connect.util.ParameterizedTest; +import org.apache.kafka.connect.util.ThreadedTest; +import org.apache.kafka.connect.util.TopicAdmin; +import org.apache.kafka.connect.util.TopicCreationGroup; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.easymock.IAnswer; +import org.easymock.IExpectationSetters; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.api.easymock.annotation.Mock; +import org.powermock.api.easymock.annotation.MockStrict; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.modules.junit4.PowerMockRunnerDelegate; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.apache.kafka.connect.integration.MonitorableSourceConnector.TOPIC_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.SourceConnectorConfig.TOPIC_CREATION_GROUPS_CONFIG; +import static org.apache.kafka.connect.runtime.SourceConnectorConfig.TRANSACTION_BOUNDARY_CONFIG; +import static org.apache.kafka.connect.runtime.SourceConnectorConfig.TRANSACTION_BOUNDARY_INTERVAL_CONFIG; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.DEFAULT_TOPIC_CREATION_PREFIX; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.EXCLUDE_REGEX_CONFIG; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.INCLUDE_REGEX_CONFIG; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.PARTITIONS_CONFIG; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.REPLICATION_FACTOR_CONFIG; +import static org.apache.kafka.connect.runtime.WorkerConfig.TOPIC_CREATION_ENABLE_CONFIG; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@PowerMockIgnore({"javax.management.*", + "org.apache.log4j.*"}) +@RunWith(PowerMockRunner.class) +@PowerMockRunnerDelegate(ParameterizedTest.class) +public class ExactlyOnceWorkerSourceTaskTest extends ThreadedTest { + private static final String TOPIC = "topic"; + private static final Map PARTITION = Collections.singletonMap("key", "partition".getBytes()); + private static final Map OFFSET = Collections.singletonMap("key", 12); + + // Connect-format data + private static final Schema KEY_SCHEMA = Schema.INT32_SCHEMA; + private static final Integer KEY = -1; + private static final Schema RECORD_SCHEMA = Schema.INT64_SCHEMA; + private static final Long RECORD = 12L; + // Serialized data. The actual format of this data doesn't matter -- we just want to see that the right version + // is used in the right place. + private static final byte[] SERIALIZED_KEY = "converted-key".getBytes(); + private static final byte[] SERIALIZED_RECORD = "converted-record".getBytes(); + + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private final ConnectorTaskId taskId = new ConnectorTaskId("job", 0); + private WorkerConfig config; + private SourceConnectorConfig sourceConfig; + private Plugins plugins; + private MockConnectMetrics metrics; + private Time time; + private CountDownLatch pollLatch; + @Mock private SourceTask sourceTask; + @Mock private Converter keyConverter; + @Mock private Converter valueConverter; + @Mock private HeaderConverter headerConverter; + @Mock private TransformationChain transformationChain; + @Mock private KafkaProducer producer; + @Mock private TopicAdmin admin; + @Mock private CloseableOffsetStorageReader offsetReader; + @Mock private OffsetStorageWriter offsetWriter; + @Mock private ClusterConfigState clusterConfigState; + private ExactlyOnceWorkerSourceTask workerTask; + @Mock private Future sendFuture; + @MockStrict private TaskStatus.Listener statusListener; + @Mock private StatusBackingStore statusBackingStore; + @Mock private ConnectorOffsetBackingStore offsetStore; + @Mock private Runnable preProducerCheck; + @Mock private Runnable postProducerCheck; + + private Capture producerCallbacks; + + private static final Map TASK_PROPS = new HashMap<>(); + static { + TASK_PROPS.put(TaskConfig.TASK_CLASS_CONFIG, TestSourceTask.class.getName()); + } + private static final TaskConfig TASK_CONFIG = new TaskConfig(TASK_PROPS); + + private static final SourceRecord SOURCE_RECORD = + new SourceRecord(PARTITION, OFFSET, "topic", null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + private static final List RECORDS = Collections.singletonList(SOURCE_RECORD); + + private final boolean enableTopicCreation; + + @ParameterizedTest.Parameters + public static Collection parameters() { + return Arrays.asList(false, true); + } + + public ExactlyOnceWorkerSourceTaskTest(boolean enableTopicCreation) { + this.enableTopicCreation = enableTopicCreation; + } + + @Override + public void setup() { + super.setup(); + Map workerProps = workerProps(); + plugins = new Plugins(workerProps); + config = new StandaloneConfig(workerProps); + sourceConfig = new SourceConnectorConfig(plugins, sourceConnectorProps(), true); + producerCallbacks = EasyMock.newCapture(); + metrics = new MockConnectMetrics(); + time = Time.SYSTEM; + EasyMock.expect(offsetStore.primaryOffsetsTopic()).andStubReturn("offsets-topic"); + pollLatch = new CountDownLatch(1); + } + + private Map workerProps() { + Map props = new HashMap<>(); + props.put("key.converter", "org.apache.kafka.connect.json.JsonConverter"); + props.put("value.converter", "org.apache.kafka.connect.json.JsonConverter"); + props.put("internal.key.converter", "org.apache.kafka.connect.json.JsonConverter"); + props.put("internal.value.converter", "org.apache.kafka.connect.json.JsonConverter"); + props.put("internal.key.converter.schemas.enable", "false"); + props.put("internal.value.converter.schemas.enable", "false"); + props.put("offset.storage.file.filename", "/tmp/connect.offsets"); + props.put(TOPIC_CREATION_ENABLE_CONFIG, String.valueOf(enableTopicCreation)); + return props; + } + + private Map sourceConnectorProps() { + return sourceConnectorProps(SourceTask.TransactionBoundary.DEFAULT); + } + + private Map sourceConnectorProps(SourceTask.TransactionBoundary transactionBoundary) { + // setup up props for the source connector + Map props = new HashMap<>(); + props.put("name", "foo-connector"); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getSimpleName()); + props.put(TASKS_MAX_CONFIG, String.valueOf(1)); + props.put(TOPIC_CONFIG, TOPIC); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(TOPIC_CREATION_GROUPS_CONFIG, String.join(",", "foo", "bar")); + props.put(DEFAULT_TOPIC_CREATION_PREFIX + REPLICATION_FACTOR_CONFIG, String.valueOf(1)); + props.put(DEFAULT_TOPIC_CREATION_PREFIX + PARTITIONS_CONFIG, String.valueOf(1)); + props.put(TRANSACTION_BOUNDARY_CONFIG, transactionBoundary.toString()); + props.put(SourceConnectorConfig.TOPIC_CREATION_PREFIX + "foo" + "." + INCLUDE_REGEX_CONFIG, TOPIC); + props.put(SourceConnectorConfig.TOPIC_CREATION_PREFIX + "bar" + "." + INCLUDE_REGEX_CONFIG, ".*"); + props.put(SourceConnectorConfig.TOPIC_CREATION_PREFIX + "bar" + "." + EXCLUDE_REGEX_CONFIG, TOPIC); + return props; + } + + @After + public void tearDown() { + if (metrics != null) metrics.stop(); + } + + private void createWorkerTask() { + createWorkerTask(TargetState.STARTED); + } + + private void createWorkerTask(TargetState initialState) { + createWorkerTask(initialState, keyConverter, valueConverter, headerConverter); + } + + private void createWorkerTask(TargetState initialState, Converter keyConverter, Converter valueConverter, HeaderConverter headerConverter) { + workerTask = new ExactlyOnceWorkerSourceTask(taskId, sourceTask, statusListener, initialState, keyConverter, valueConverter, headerConverter, + transformationChain, producer, admin, TopicCreationGroup.configuredGroups(sourceConfig), offsetReader, offsetWriter, offsetStore, + config, clusterConfigState, metrics, plugins.delegatingLoader(), time, RetryWithToleranceOperatorTest.NOOP_OPERATOR, statusBackingStore, + sourceConfig, Runnable::run, preProducerCheck, postProducerCheck); + } + + @Test + public void testStartPaused() throws Exception { + final CountDownLatch pauseLatch = new CountDownLatch(1); + + createWorkerTask(TargetState.PAUSED); + + expectCall(() -> statusListener.onPause(taskId)).andAnswer(() -> { + pauseLatch.countDown(); + return null; + }); + + // The task checks to see if there are offsets to commit before pausing + EasyMock.expect(offsetWriter.willFlush()).andReturn(false); + + expectClose(); + + expectCall(() -> statusListener.onShutdown(taskId)); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + Future taskFuture = executor.submit(workerTask); + + assertTrue(pauseLatch.await(5, TimeUnit.SECONDS)); + workerTask.stop(); + assertTrue(workerTask.awaitStop(1000)); + + taskFuture.get(); + + PowerMock.verifyAll(); + } + + @Test + public void testPause() throws Exception { + createWorkerTask(); + + expectPreflight(); + expectStartup(); + + AtomicInteger polls = new AtomicInteger(0); + AtomicInteger flushes = new AtomicInteger(0); + pollLatch = new CountDownLatch(10); + expectPolls(polls); + expectAnyFlushes(flushes); + + expectTopicCreation(TOPIC); + + expectCall(() -> statusListener.onPause(taskId)); + + expectCall(sourceTask::stop); + expectCall(() -> statusListener.onShutdown(taskId)); + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + Future taskFuture = executor.submit(workerTask); + assertTrue(awaitLatch(pollLatch)); + + workerTask.transitionTo(TargetState.PAUSED); + + int priorCount = polls.get(); + Thread.sleep(100); + + // since the transition is observed asynchronously, the count could be off by one loop iteration + assertTrue(polls.get() - priorCount <= 1); + + workerTask.stop(); + assertTrue(workerTask.awaitStop(1000)); + + taskFuture.get(); + + assertEquals("Task should have flushed offsets for every record poll, once on pause, and once for end-of-life offset commit", + flushes.get(), polls.get() + 2); + + PowerMock.verifyAll(); + } + + @Test + public void testFailureInPreProducerCheck() { + createWorkerTask(); + + Exception exception = new ConnectException("Failed to perform zombie fencing"); + expectCall(preProducerCheck::run).andThrow(exception); + + expectCall(() -> statusListener.onFailure(taskId, exception)); + + // Don't expect task to be stopped since it was never started to begin with + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + // No need to execute on a separate thread; preflight checks should all take place before the poll-send loop starts + workerTask.run(); + + PowerMock.verifyAll(); + } + + @Test + public void testFailureInOffsetStoreStart() { + createWorkerTask(); + + expectCall(preProducerCheck::run); + Exception exception = new ConnectException("No soup for you!"); + expectCall(offsetStore::start).andThrow(exception); + + expectCall(() -> statusListener.onFailure(taskId, exception)); + + // Don't expect task to be stopped since it was never started to begin with + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + // No need to execute on a separate thread; preflight checks should all take place before the poll-send loop starts + workerTask.run(); + + PowerMock.verifyAll(); + } + + @Test + public void testFailureInProducerInitialization() { + createWorkerTask(); + + expectCall(preProducerCheck::run); + expectCall(offsetStore::start); + expectCall(producer::initTransactions); + Exception exception = new ConnectException("You can't do that!"); + expectCall(postProducerCheck::run).andThrow(exception); + + expectCall(() -> statusListener.onFailure(taskId, exception)); + + // Don't expect task to be stopped since it was never started to begin with + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + // No need to execute on a separate thread; preflight checks should all take place before the poll-send loop starts + workerTask.run(); + + PowerMock.verifyAll(); + } + + @Test + public void testFailureInPostProducerCheck() { + createWorkerTask(); + + expectCall(preProducerCheck::run); + expectCall(offsetStore::start); + Exception exception = new ConnectException("New task configs for the connector have already been generated"); + expectCall(producer::initTransactions).andThrow(exception); + + expectCall(() -> statusListener.onFailure(taskId, exception)); + + // Don't expect task to be stopped since it was never started to begin with + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + // No need to execute on a separate thread; preflight checks should all take place before the poll-send loop starts + workerTask.run(); + + PowerMock.verifyAll(); + } + + @Test + public void testPollsInBackground() throws Exception { + createWorkerTask(); + + expectPreflight(); + expectStartup(); + + AtomicInteger polls = new AtomicInteger(0); + AtomicInteger flushes = new AtomicInteger(0); + pollLatch = new CountDownLatch(10); + expectPolls(polls); + expectAnyFlushes(flushes); + + expectTopicCreation(TOPIC); + + expectCall(sourceTask::stop); + expectCall(() -> statusListener.onShutdown(taskId)); + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + Future taskFuture = executor.submit(workerTask); + + assertTrue(awaitLatch(pollLatch)); + workerTask.stop(); + assertTrue(workerTask.awaitStop(1000)); + + taskFuture.get(); + assertPollMetrics(10); + assertTransactionMetrics(1); + + assertEquals("Task should have flushed offsets for every record poll and for end-of-life offset commit", + flushes.get(), polls.get() + 1); + + PowerMock.verifyAll(); + } + + @Test + public void testFailureInPoll() throws Exception { + createWorkerTask(); + + expectPreflight(); + expectStartup(); + + final CountDownLatch pollLatch = new CountDownLatch(1); + final RuntimeException exception = new RuntimeException(); + EasyMock.expect(sourceTask.poll()).andAnswer(() -> { + pollLatch.countDown(); + throw exception; + }); + + expectCall(() -> statusListener.onFailure(taskId, exception)); + expectCall(sourceTask::stop); + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + Future taskFuture = executor.submit(workerTask); + + assertTrue(awaitLatch(pollLatch)); + //Failure in poll should trigger automatic stop of the worker + assertTrue(workerTask.awaitStop(1000)); + + taskFuture.get(); + assertPollMetrics(0); + + PowerMock.verifyAll(); + } + + @Test + public void testFailureInPollAfterCancel() throws Exception { + createWorkerTask(); + + expectPreflight(); + expectStartup(); + + final CountDownLatch pollLatch = new CountDownLatch(1); + final CountDownLatch workerCancelLatch = new CountDownLatch(1); + final RuntimeException exception = new RuntimeException(); + EasyMock.expect(sourceTask.poll()).andAnswer(() -> { + pollLatch.countDown(); + assertTrue(awaitLatch(workerCancelLatch)); + throw exception; + }); + + expectCall(offsetReader::close); + expectCall(() -> producer.close(Duration.ZERO)); + expectCall(sourceTask::stop); + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + Future taskFuture = executor.submit(workerTask); + + assertTrue(awaitLatch(pollLatch)); + workerTask.cancel(); + workerCancelLatch.countDown(); + assertTrue(workerTask.awaitStop(1000)); + + taskFuture.get(); + assertPollMetrics(0); + + PowerMock.verifyAll(); + } + + @Test + public void testFailureInPollAfterStop() throws Exception { + createWorkerTask(); + + expectPreflight(); + expectStartup(); + + final CountDownLatch pollLatch = new CountDownLatch(1); + final CountDownLatch workerStopLatch = new CountDownLatch(1); + final RuntimeException exception = new RuntimeException(); + EasyMock.expect(sourceTask.poll()).andAnswer(() -> { + pollLatch.countDown(); + assertTrue(awaitLatch(workerStopLatch)); + throw exception; + }); + + expectCall(() -> statusListener.onShutdown(taskId)); + expectCall(sourceTask::stop); + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + Future taskFuture = executor.submit(workerTask); + + assertTrue(awaitLatch(pollLatch)); + workerTask.stop(); + workerStopLatch.countDown(); + assertTrue(workerTask.awaitStop(1000)); + + taskFuture.get(); + assertPollMetrics(0); + + PowerMock.verifyAll(); + } + + @Test + public void testPollReturnsNoRecords() throws Exception { + // Test that the task handles an empty list of records + createWorkerTask(); + + expectPreflight(); + expectStartup(); + + final CountDownLatch pollLatch = expectEmptyPolls(1, new AtomicInteger()); + EasyMock.expect(offsetWriter.willFlush()).andReturn(false).anyTimes(); + + expectCall(sourceTask::stop); + expectCall(() -> statusListener.onShutdown(taskId)); + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + Future taskFuture = executor.submit(workerTask); + + assertTrue(awaitLatch(pollLatch)); + workerTask.stop(); + assertTrue(workerTask.awaitStop(1000)); + + taskFuture.get(); + assertPollMetrics(0); + + PowerMock.verifyAll(); + } + + @Test + public void testPollBasedCommit() throws Exception { + Map connectorProps = sourceConnectorProps(SourceTask.TransactionBoundary.POLL); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + + createWorkerTask(); + + expectPreflight(); + expectStartup(); + + AtomicInteger polls = new AtomicInteger(); + AtomicInteger flushes = new AtomicInteger(); + expectPolls(polls); + expectAnyFlushes(flushes); + + expectTopicCreation(TOPIC); + + expectCall(sourceTask::stop); + expectCall(() -> statusListener.onShutdown(taskId)); + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + Future taskFuture = executor.submit(workerTask); + + assertTrue(awaitLatch(pollLatch)); + workerTask.stop(); + assertTrue(workerTask.awaitStop(1000)); + + taskFuture.get(); + + assertEquals("Task should have flushed offsets for every record poll, and for end-of-life offset commit", + flushes.get(), polls.get() + 1); + + assertPollMetrics(1); + assertTransactionMetrics(1); + + PowerMock.verifyAll(); + } + + @Test + public void testIntervalBasedCommit() throws Exception { + long commitInterval = 618; + Map connectorProps = sourceConnectorProps(SourceTask.TransactionBoundary.INTERVAL); + connectorProps.put(TRANSACTION_BOUNDARY_INTERVAL_CONFIG, Long.toString(commitInterval)); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + + time = new MockTime(); + + createWorkerTask(); + + expectPreflight(); + expectStartup(); + + expectPolls(); + final CountDownLatch firstPollLatch = new CountDownLatch(2); + final CountDownLatch secondPollLatch = new CountDownLatch(2); + final CountDownLatch thirdPollLatch = new CountDownLatch(2); + + AtomicInteger flushes = new AtomicInteger(); + expectFlush(FlushOutcome.SUCCEED, flushes); + expectFlush(FlushOutcome.SUCCEED, flushes); + expectFlush(FlushOutcome.SUCCEED, flushes); + + expectTopicCreation(TOPIC); + + expectCall(sourceTask::stop); + expectCall(() -> statusListener.onShutdown(taskId)); + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + Future taskFuture = executor.submit(workerTask); + + pollLatch = firstPollLatch; + assertTrue(awaitLatch(pollLatch)); + assertEquals("No flushes should have taken place before offset commit interval has elapsed", 0, flushes.get()); + time.sleep(commitInterval); + + pollLatch = secondPollLatch; + assertTrue(awaitLatch(pollLatch)); + assertEquals("One flush should have taken place after offset commit interval has elapsed", 1, flushes.get()); + time.sleep(commitInterval * 2); + + pollLatch = thirdPollLatch; + assertTrue(awaitLatch(pollLatch)); + assertEquals("Two flushes should have taken place after offset commit interval has elapsed again", 2, flushes.get()); + + workerTask.stop(); + assertTrue(workerTask.awaitStop(1000)); + + taskFuture.get(); + + assertEquals("Task should have flushed offsets twice based on offset commit interval, and performed final end-of-life offset commit", + 3, flushes.get()); + + assertPollMetrics(2); + + PowerMock.verifyAll(); + } + + @Test + public void testConnectorBasedCommit() throws Exception { + Map connectorProps = sourceConnectorProps(SourceTask.TransactionBoundary.CONNECTOR); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + createWorkerTask(); + + expectPreflight(); + expectStartup(); + + expectPolls(); + List pollLatches = IntStream.range(0, 7).mapToObj(i -> new CountDownLatch(3)).collect(Collectors.toList()); + + AtomicInteger flushes = new AtomicInteger(); + // First flush: triggered by TransactionContext::commitTransaction (batch) + expectFlush(FlushOutcome.SUCCEED, flushes); + + // Second flush: triggered by TransactionContext::commitTransaction (record) + expectFlush(FlushOutcome.SUCCEED, flushes); + + // Third flush: triggered by TransactionContext::abortTransaction (batch) + expectCall(producer::abortTransaction); + EasyMock.expect(offsetWriter.willFlush()).andReturn(true); + expectFlush(FlushOutcome.SUCCEED, flushes); + + // Third flush: triggered by TransactionContext::abortTransaction (record) + EasyMock.expect(offsetWriter.willFlush()).andReturn(true); + expectCall(producer::abortTransaction); + expectFlush(FlushOutcome.SUCCEED, flushes); + + expectTopicCreation(TOPIC); + + expectCall(sourceTask::stop); + expectCall(() -> statusListener.onShutdown(taskId)); + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + Future taskFuture = executor.submit(workerTask); + + TransactionContext transactionContext = workerTask.sourceTaskContext.transactionContext(); + + int poll = -1; + pollLatch = pollLatches.get(++poll); + assertTrue(awaitLatch(pollLatch)); + assertEquals("No flushes should have taken place without connector requesting transaction commit", 0, flushes.get()); + + transactionContext.commitTransaction(); + pollLatch = pollLatches.get(++poll); + assertTrue(awaitLatch(pollLatch)); + assertEquals("One flush should have taken place after connector requested batch commit", 1, flushes.get()); + + transactionContext.commitTransaction(SOURCE_RECORD); + pollLatch = pollLatches.get(++poll); + assertTrue(awaitLatch(pollLatch)); + assertEquals("Two flushes should have taken place after connector requested individual record commit", 2, flushes.get()); + + pollLatch = pollLatches.get(++poll); + assertTrue(awaitLatch(pollLatch)); + assertEquals("Only two flushes should still have taken place without connector re-requesting commit, even on identical records", 2, flushes.get()); + + transactionContext.abortTransaction(); + pollLatch = pollLatches.get(++poll); + assertTrue(awaitLatch(pollLatch)); + assertEquals("Three flushes should have taken place after connector requested batch abort", 3, flushes.get()); + + transactionContext.abortTransaction(SOURCE_RECORD); + pollLatch = pollLatches.get(++poll); + assertTrue(awaitLatch(pollLatch)); + assertEquals("Four flushes should have taken place after connector requested individual record abort", 4, flushes.get()); + + pollLatch = pollLatches.get(++poll); + assertTrue(awaitLatch(pollLatch)); + assertEquals("Only four flushes should still have taken place without connector re-requesting abort, even on identical records", 4, flushes.get()); + + workerTask.stop(); + assertTrue(workerTask.awaitStop(1000)); + + taskFuture.get(); + + assertEquals("Task should have flushed offsets four times based on connector-defined boundaries, and skipped final end-of-life offset commit", + 4, flushes.get()); + + assertPollMetrics(1); + assertTransactionMetrics(2); + + PowerMock.verifyAll(); + } + + @Test + public void testCommitFlushCallbackFailure() throws Exception { + testCommitFailure(FlushOutcome.FAIL_FLUSH_CALLBACK); + } + + @Test + public void testCommitTransactionFailure() throws Exception { + testCommitFailure(FlushOutcome.FAIL_TRANSACTION_COMMIT); + } + + private void testCommitFailure(FlushOutcome causeOfFailure) throws Exception { + createWorkerTask(); + + expectPreflight(); + expectStartup(); + + expectPolls(); + expectFlush(causeOfFailure); + + expectTopicCreation(TOPIC); + + expectCall(sourceTask::stop); + // Unlike the standard WorkerSourceTask class, this one fails permanently when offset commits don't succeed + final CountDownLatch taskFailure = new CountDownLatch(1); + expectCall(() -> statusListener.onFailure(EasyMock.eq(taskId), EasyMock.anyObject())) + .andAnswer(() -> { + taskFailure.countDown(); + return null; + }); + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + Future taskFuture = executor.submit(workerTask); + + assertTrue(awaitLatch(taskFailure)); + workerTask.stop(); + assertTrue(workerTask.awaitStop(1000)); + + taskFuture.get(); + assertPollMetrics(1); + + PowerMock.verifyAll(); + } + + @Test + public void testSendRecordsRetries() throws Exception { + createWorkerTask(); + + // Differentiate only by Kafka partition so we can reuse conversion expectations + SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, "topic", 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, "topic", 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record3 = new SourceRecord(PARTITION, OFFSET, "topic", 3, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + expectTopicCreation(TOPIC); + + // First round + expectSendRecordOnce(false); + expectCall(producer::beginTransaction); + // Any Producer retriable exception should work here + expectSendRecordSyncFailure(new org.apache.kafka.common.errors.TimeoutException("retriable sync failure")); + + // Second round + expectSendRecordOnce(true); + expectSendRecordOnce(false); + + PowerMock.replayAll(); + + // Try to send 3, make first pass, second fail. Should save last two + workerTask.toSend = Arrays.asList(record1, record2, record3); + workerTask.sendRecords(); + assertEquals(Arrays.asList(record2, record3), workerTask.toSend); + + // Next they all succeed + workerTask.sendRecords(); + assertNull(workerTask.toSend); + + PowerMock.verifyAll(); + } + + @Test + public void testSendRecordsProducerSendFailsImmediately() { + if (!enableTopicCreation) + // should only test with topic creation enabled + return; + + createWorkerTask(); + + SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); + + expectPreliminaryCalls(); + expectCall(producer::beginTransaction); + expectTopicCreation(TOPIC); + + EasyMock.expect(producer.send(EasyMock.anyObject(), EasyMock.anyObject())) + .andThrow(new KafkaException("Producer closed while send in progress", new InvalidTopicException(TOPIC))); + + PowerMock.replayAll(); + + workerTask.toSend = Arrays.asList(record1, record2); + assertThrows(ConnectException.class, workerTask::sendRecords); + } + + @Test + public void testSlowTaskStart() throws Exception { + final CountDownLatch startupLatch = new CountDownLatch(1); + final CountDownLatch finishStartupLatch = new CountDownLatch(1); + + createWorkerTask(); + + expectPreflight(); + + expectCall(() -> sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class))); + expectCall(() -> sourceTask.start(TASK_PROPS)); + EasyMock.expectLastCall().andAnswer(() -> { + startupLatch.countDown(); + assertTrue(awaitLatch(finishStartupLatch)); + return null; + }); + + expectCall(() -> statusListener.onStartup(taskId)); + + expectCall(sourceTask::stop); + EasyMock.expect(offsetWriter.willFlush()).andReturn(false); + + expectCall(() -> statusListener.onShutdown(taskId)); + + expectClose(); + + PowerMock.replayAll(); + + workerTask.initialize(TASK_CONFIG); + Future workerTaskFuture = executor.submit(workerTask); + + // Stopping immediately while the other thread has work to do should result in no polling, no offset commits, + // exiting the work thread immediately, and the stop() method will be invoked in the background thread since it + // cannot be invoked immediately in the thread trying to stop the task. + assertTrue(awaitLatch(startupLatch)); + workerTask.stop(); + finishStartupLatch.countDown(); + assertTrue(workerTask.awaitStop(1000)); + + workerTaskFuture.get(); + + PowerMock.verifyAll(); + } + + @Test + public void testCancel() { + createWorkerTask(); + + expectCall(offsetReader::close); + expectCall(() -> producer.close(Duration.ZERO)); + + PowerMock.replayAll(); + + // workerTask said something dumb on twitter + workerTask.cancel(); + + PowerMock.verifyAll(); + } + + private TopicAdmin.TopicCreationResponse createdTopic(String topic) { + Set created = Collections.singleton(topic); + Set existing = Collections.emptySet(); + return new TopicAdmin.TopicCreationResponse(created, existing); + } + + private void expectPreliminaryCalls() { + expectPreliminaryCalls(TOPIC); + } + + private void expectPreliminaryCalls(String topic) { + expectConvertHeadersAndKeyValue(topic, true, emptyHeaders()); + expectApplyTransformationChain(false); + offsetWriter.offset(PARTITION, OFFSET); + PowerMock.expectLastCall(); + } + + private CountDownLatch expectEmptyPolls(int minimum, final AtomicInteger count) throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(minimum); + // Note that we stub these to allow any number of calls because the thread will continue to + // run. The count passed in + latch returned just makes sure we get *at least* that number of + // calls + EasyMock.expect(sourceTask.poll()) + .andStubAnswer(() -> { + count.incrementAndGet(); + latch.countDown(); + Thread.sleep(10); + return Collections.emptyList(); + }); + return latch; + } + + private void expectPolls(final AtomicInteger pollCount) throws Exception { + expectCall(producer::beginTransaction).atLeastOnce(); + // Note that we stub these to allow any number of calls because the thread will continue to + // run. The count passed in + latch returned just makes sure we get *at least* that number of + // calls + EasyMock.expect(sourceTask.poll()) + .andStubAnswer(() -> { + pollCount.incrementAndGet(); + pollLatch.countDown(); + Thread.sleep(10); + return RECORDS; + }); + // Fallout of the poll() call + expectSendRecordAnyTimes(); + } + + private void expectPolls() throws Exception { + expectPolls(new AtomicInteger()); + } + + @SuppressWarnings("unchecked") + private void expectSendRecordSyncFailure(Throwable error) { + expectConvertHeadersAndKeyValue(false); + expectApplyTransformationChain(false); + + offsetWriter.offset(PARTITION, OFFSET); + PowerMock.expectLastCall(); + + EasyMock.expect( + producer.send(EasyMock.anyObject(ProducerRecord.class), + EasyMock.anyObject(org.apache.kafka.clients.producer.Callback.class))) + .andThrow(error); + } + + private Capture> expectSendRecordAnyTimes() { + return expectSendRecordSendSuccess(true, false); + } + + private Capture> expectSendRecordOnce(boolean isRetry) { + return expectSendRecordSendSuccess(false, isRetry); + } + + private Capture> expectSendRecordSendSuccess(boolean anyTimes, boolean isRetry) { + return expectSendRecord(TOPIC, anyTimes, isRetry, true, true, emptyHeaders()); + } + + private Capture> expectSendRecord( + String topic, + boolean anyTimes, + boolean isRetry, + boolean sendSuccess, + boolean isMockedConverters, + Headers headers + ) { + if (isMockedConverters) { + expectConvertHeadersAndKeyValue(topic, anyTimes, headers); + } + + expectApplyTransformationChain(anyTimes); + + Capture> sent = EasyMock.newCapture(); + + // 1. Offset data is passed to the offset storage. + if (!isRetry) { + offsetWriter.offset(PARTITION, OFFSET); + if (anyTimes) + PowerMock.expectLastCall().anyTimes(); + else + PowerMock.expectLastCall(); + } + + // 2. Converted data passed to the producer, which will need callbacks invoked for flush to work + IExpectationSetters> expect = EasyMock.expect( + producer.send(EasyMock.capture(sent), + EasyMock.capture(producerCallbacks))); + IAnswer> expectResponse = () -> { + synchronized (producerCallbacks) { + for (org.apache.kafka.clients.producer.Callback cb : producerCallbacks.getValues()) { + if (sendSuccess) { + cb.onCompletion(new RecordMetadata(new TopicPartition("foo", 0), 0, 0, + 0L, 0, 0), null); + } else { + cb.onCompletion(null, new TopicAuthorizationException("foo")); + } + } + producerCallbacks.reset(); + } + return sendFuture; + }; + if (anyTimes) + expect.andStubAnswer(expectResponse); + else + expect.andAnswer(expectResponse); + + if (sendSuccess) { + // 3. As a result of a successful producer send callback, we note the use of the topic + expectTaskGetTopic(anyTimes); + } + + return sent; + } + + private void expectConvertHeadersAndKeyValue(boolean anyTimes) { + expectConvertHeadersAndKeyValue(TOPIC, anyTimes, emptyHeaders()); + } + + private void expectConvertHeadersAndKeyValue(String topic, boolean anyTimes, Headers headers) { + for (Header header : headers) { + IExpectationSetters convertHeaderExpect = EasyMock.expect(headerConverter.fromConnectHeader(topic, header.key(), Schema.STRING_SCHEMA, new String(header.value()))); + if (anyTimes) + convertHeaderExpect.andStubReturn(header.value()); + else + convertHeaderExpect.andReturn(header.value()); + } + IExpectationSetters convertKeyExpect = EasyMock.expect(keyConverter.fromConnectData(topic, headers, KEY_SCHEMA, KEY)); + if (anyTimes) + convertKeyExpect.andStubReturn(SERIALIZED_KEY); + else + convertKeyExpect.andReturn(SERIALIZED_KEY); + IExpectationSetters convertValueExpect = EasyMock.expect(valueConverter.fromConnectData(topic, headers, RECORD_SCHEMA, RECORD)); + if (anyTimes) + convertValueExpect.andStubReturn(SERIALIZED_RECORD); + else + convertValueExpect.andReturn(SERIALIZED_RECORD); + } + + private void expectApplyTransformationChain(boolean anyTimes) { + final Capture recordCapture = EasyMock.newCapture(); + IExpectationSetters convertKeyExpect = EasyMock.expect(transformationChain.apply(EasyMock.capture(recordCapture))); + if (anyTimes) + convertKeyExpect.andStubAnswer(recordCapture::getValue); + else + convertKeyExpect.andAnswer(recordCapture::getValue); + } + + private void expectTaskGetTopic(boolean anyTimes) { + final Capture connectorCapture = EasyMock.newCapture(); + final Capture topicCapture = EasyMock.newCapture(); + IExpectationSetters expect = EasyMock.expect(statusBackingStore.getTopic( + EasyMock.capture(connectorCapture), + EasyMock.capture(topicCapture))); + if (anyTimes) { + expect.andStubAnswer(() -> new TopicStatus( + topicCapture.getValue(), + new ConnectorTaskId(connectorCapture.getValue(), 0), + time.milliseconds())); + } else { + expect.andAnswer(() -> new TopicStatus( + topicCapture.getValue(), + new ConnectorTaskId(connectorCapture.getValue(), 0), + time.milliseconds())); + } + if (connectorCapture.hasCaptured() && topicCapture.hasCaptured()) { + assertEquals("job", connectorCapture.getValue()); + assertEquals(TOPIC, topicCapture.getValue()); + } + } + + private boolean awaitLatch(CountDownLatch latch) { + try { + return latch.await(5000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + // ignore + } + return false; + } + + private enum FlushOutcome { + SUCCEED, + SUCCEED_ANY_TIMES, + FAIL_FLUSH_CALLBACK, + FAIL_TRANSACTION_COMMIT + } + + private CountDownLatch expectFlush(FlushOutcome outcome, AtomicInteger flushCount) { + CountDownLatch result = new CountDownLatch(1); + org.easymock.IExpectationSetters flushBegin = EasyMock + .expect(offsetWriter.beginFlush()) + .andAnswer(() -> { + flushCount.incrementAndGet(); + result.countDown(); + return true; + }); + if (FlushOutcome.SUCCEED_ANY_TIMES.equals(outcome)) { + flushBegin.anyTimes(); + } + + Capture> flushCallback = EasyMock.newCapture(); + org.easymock.IExpectationSetters> offsetFlush = + EasyMock.expect(offsetWriter.doFlush(EasyMock.capture(flushCallback))); + switch (outcome) { + case SUCCEED: + // The worker task doesn't actually use the returned future + offsetFlush.andReturn(null); + expectCall(producer::commitTransaction); + expectCall(() -> sourceTask.commitRecord(EasyMock.anyObject(), EasyMock.anyObject())); + expectCall(sourceTask::commit); + break; + case SUCCEED_ANY_TIMES: + // The worker task doesn't actually use the returned future + offsetFlush.andReturn(null).anyTimes(); + expectCall(producer::commitTransaction).anyTimes(); + expectCall(() -> sourceTask.commitRecord(EasyMock.anyObject(), EasyMock.anyObject())).anyTimes(); + expectCall(sourceTask::commit).anyTimes(); + break; + case FAIL_FLUSH_CALLBACK: + expectCall(producer::commitTransaction); + offsetFlush.andAnswer(() -> { + flushCallback.getValue().onCompletion(new RecordTooLargeException(), null); + return null; + }); + expectCall(offsetWriter::cancelFlush); + break; + case FAIL_TRANSACTION_COMMIT: + offsetFlush.andReturn(null); + expectCall(producer::commitTransaction) + .andThrow(new RecordTooLargeException()); + expectCall(offsetWriter::cancelFlush); + break; + default: + fail("Unexpected flush outcome: " + outcome); + } + return result; + } + + private CountDownLatch expectFlush(FlushOutcome outcome) { + return expectFlush(outcome, new AtomicInteger()); + } + + private CountDownLatch expectAnyFlushes(AtomicInteger flushCount) { + EasyMock.expect(offsetWriter.willFlush()).andReturn(true).anyTimes(); + return expectFlush(FlushOutcome.SUCCEED_ANY_TIMES, flushCount); + } + + private void assertTransactionMetrics(int minimumMaxSizeExpected) { + MetricGroup transactionGroup = workerTask.transactionMetricsGroup().metricGroup(); + double actualMin = metrics.currentMetricValueAsDouble(transactionGroup, "transaction-size-min"); + double actualMax = metrics.currentMetricValueAsDouble(transactionGroup, "transaction-size-max"); + double actualAvg = metrics.currentMetricValueAsDouble(transactionGroup, "transaction-size-avg"); + assertTrue(actualMin >= 0); + assertTrue(actualMax >= minimumMaxSizeExpected); + + if (actualMax - actualMin <= 0.000001d) { + assertEquals(actualMax, actualAvg, 0.000002d); + } else { + assertTrue("Average transaction size should be greater than minimum transaction size", actualAvg > actualMin); + assertTrue("Average transaction size should be less than maximum transaction size", actualAvg < actualMax); + } + } + + private void assertPollMetrics(int minimumPollCountExpected) { + MetricGroup sourceTaskGroup = workerTask.sourceTaskMetricsGroup().metricGroup(); + MetricGroup taskGroup = workerTask.taskMetricsGroup().metricGroup(); + double pollRate = metrics.currentMetricValueAsDouble(sourceTaskGroup, "source-record-poll-rate"); + double pollTotal = metrics.currentMetricValueAsDouble(sourceTaskGroup, "source-record-poll-total"); + if (minimumPollCountExpected > 0) { + assertEquals(RECORDS.size(), metrics.currentMetricValueAsDouble(taskGroup, "batch-size-max"), 0.000001d); + assertEquals(RECORDS.size(), metrics.currentMetricValueAsDouble(taskGroup, "batch-size-avg"), 0.000001d); + assertTrue(pollRate > 0.0d); + } else { + assertTrue(pollRate == 0.0d); + } + assertTrue(pollTotal >= minimumPollCountExpected); + + double writeRate = metrics.currentMetricValueAsDouble(sourceTaskGroup, "source-record-write-rate"); + double writeTotal = metrics.currentMetricValueAsDouble(sourceTaskGroup, "source-record-write-total"); + if (minimumPollCountExpected > 0) { + assertTrue(writeRate > 0.0d); + } else { + assertTrue(writeRate == 0.0d); + } + assertTrue(writeTotal >= minimumPollCountExpected); + + double pollBatchTimeMax = metrics.currentMetricValueAsDouble(sourceTaskGroup, "poll-batch-max-time-ms"); + double pollBatchTimeAvg = metrics.currentMetricValueAsDouble(sourceTaskGroup, "poll-batch-avg-time-ms"); + if (minimumPollCountExpected > 0) { + assertTrue(pollBatchTimeMax >= 0.0d); + } + assertTrue(Double.isNaN(pollBatchTimeAvg) || pollBatchTimeAvg > 0.0d); + double activeCount = metrics.currentMetricValueAsDouble(sourceTaskGroup, "source-record-active-count"); + double activeCountMax = metrics.currentMetricValueAsDouble(sourceTaskGroup, "source-record-active-count-max"); + assertEquals(0, activeCount, 0.000001d); + if (minimumPollCountExpected > 0) { + assertEquals(RECORDS.size(), activeCountMax, 0.000001d); + } + } + + private RecordHeaders emptyHeaders() { + return new RecordHeaders(); + } + + private abstract static class TestSourceTask extends SourceTask { + } + + @FunctionalInterface + private interface MockedMethodCall { + void invoke() throws Exception; + } + + private static org.easymock.IExpectationSetters expectCall(MockedMethodCall call) { + try { + call.invoke(); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Mocked method invocation threw a checked exception", e); + } + return EasyMock.expectLastCall(); + } + + private void expectPreflight() { + expectCall(preProducerCheck::run); + expectCall(offsetStore::start); + expectCall(producer::initTransactions); + expectCall(postProducerCheck::run); + } + + private void expectStartup() { + expectCall(() -> sourceTask.initialize(EasyMock.anyObject(SourceTaskContext.class))); + expectCall(() -> sourceTask.start(TASK_PROPS)); + expectCall(() -> statusListener.onStartup(taskId)); + } + + private void expectClose() { + expectCall(offsetStore::stop); + expectCall(() -> producer.close(EasyMock.anyObject(Duration.class))); + expectCall(() -> admin.close(EasyMock.anyObject(Duration.class))); + expectCall(transformationChain::close); + expectCall(offsetReader::close); + } + + private void expectTopicCreation(String topic) { + if (config.topicCreationEnable()) { + EasyMock.expect(admin.describeTopics(topic)).andReturn(Collections.emptyMap()); + Capture newTopicCapture = EasyMock.newCapture(); + EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andReturn(createdTopic(topic)); + } + } +} diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/SubmittedRecordsTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/SubmittedRecordsTest.java index 4028249a78ad8..136b0619ddefb 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/SubmittedRecordsTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/SubmittedRecordsTest.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.connect.runtime; -import org.apache.kafka.connect.runtime.SubmittedRecords.SubmittedRecord; import org.junit.Before; import org.junit.Test; @@ -89,7 +88,7 @@ public void testNoCommittedRecords() { public void testSingleAck() { Map offset = newOffset(); - SubmittedRecord submittedRecord = submittedRecords.submit(PARTITION1, offset); + SubmittedRecords.SubmittedRecord submittedRecord = submittedRecords.submit(PARTITION1, offset); CommittableOffsets committableOffsets = submittedRecords.committableOffsets(); // Record has been submitted but not yet acked; cannot commit offsets for it yet assertFalse(committableOffsets.isEmpty()); @@ -120,10 +119,10 @@ public void testMultipleAcksAcrossMultiplePartitions() { Map partition2Offset1 = newOffset(); Map partition2Offset2 = newOffset(); - SubmittedRecord partition1Record1 = submittedRecords.submit(PARTITION1, partition1Offset1); - SubmittedRecord partition1Record2 = submittedRecords.submit(PARTITION1, partition1Offset2); - SubmittedRecord partition2Record1 = submittedRecords.submit(PARTITION2, partition2Offset1); - SubmittedRecord partition2Record2 = submittedRecords.submit(PARTITION2, partition2Offset2); + SubmittedRecords.SubmittedRecord partition1Record1 = submittedRecords.submit(PARTITION1, partition1Offset1); + SubmittedRecords.SubmittedRecord partition1Record2 = submittedRecords.submit(PARTITION1, partition1Offset2); + SubmittedRecords.SubmittedRecord partition2Record1 = submittedRecords.submit(PARTITION2, partition2Offset1); + SubmittedRecords.SubmittedRecord partition2Record2 = submittedRecords.submit(PARTITION2, partition2Offset2); CommittableOffsets committableOffsets = submittedRecords.committableOffsets(); // No records ack'd yet; can't commit any offsets @@ -172,14 +171,14 @@ public void testMultipleAcksAcrossMultiplePartitions() { @Test public void testRemoveLastSubmittedRecord() { - SubmittedRecord submittedRecord = submittedRecords.submit(PARTITION1, newOffset()); + SubmittedRecords.SubmittedRecord submittedRecord = submittedRecords.submit(PARTITION1, newOffset()); CommittableOffsets committableOffsets = submittedRecords.committableOffsets(); assertEquals(Collections.emptyMap(), committableOffsets.offsets()); assertMetadata(committableOffsets, 0, 1, 1, 1, PARTITION1); - assertTrue("First attempt to remove record from submitted queue should succeed", submittedRecords.removeLastOccurrence(submittedRecord)); - assertFalse("Attempt to remove already-removed record from submitted queue should fail", submittedRecords.removeLastOccurrence(submittedRecord)); + assertTrue("First attempt to remove record from submitted queue should succeed", submittedRecord.drop()); + assertFalse("Attempt to remove already-removed record from submitted queue should fail", submittedRecord.drop()); committableOffsets = submittedRecords.committableOffsets(); // Even if SubmittedRecords::remove is broken, we haven't ack'd anything yet, so there should be no committable offsets @@ -196,14 +195,14 @@ public void testRemoveNotLastSubmittedRecord() { Map partition1Offset = newOffset(); Map partition2Offset = newOffset(); - SubmittedRecord recordToRemove = submittedRecords.submit(PARTITION1, partition1Offset); - SubmittedRecord lastSubmittedRecord = submittedRecords.submit(PARTITION2, partition2Offset); + SubmittedRecords.SubmittedRecord recordToRemove = submittedRecords.submit(PARTITION1, partition1Offset); + SubmittedRecords.SubmittedRecord lastSubmittedRecord = submittedRecords.submit(PARTITION2, partition2Offset); CommittableOffsets committableOffsets = submittedRecords.committableOffsets(); assertMetadata(committableOffsets, 0, 2, 2, 1, PARTITION1, PARTITION2); assertNoEmptyDeques(); - assertTrue("First attempt to remove record from submitted queue should succeed", submittedRecords.removeLastOccurrence(recordToRemove)); + assertTrue("First attempt to remove record from submitted queue should succeed", recordToRemove.drop()); committableOffsets = submittedRecords.committableOffsets(); // Even if SubmittedRecords::remove is broken, we haven't ack'd anything yet, so there should be no committable offsets @@ -235,7 +234,7 @@ public void testRemoveNotLastSubmittedRecord() { @Test public void testNullPartitionAndOffset() { - SubmittedRecord submittedRecord = submittedRecords.submit(null, null); + SubmittedRecords.SubmittedRecord submittedRecord = submittedRecords.submit(null, null); CommittableOffsets committableOffsets = submittedRecords.committableOffsets(); assertMetadata(committableOffsets, 0, 1, 1, 1, (Map) null); @@ -254,7 +253,7 @@ public void testAwaitMessagesNoneSubmitted() { @Test public void testAwaitMessagesAfterAllAcknowledged() { - SubmittedRecord recordToAck = submittedRecords.submit(PARTITION1, newOffset()); + SubmittedRecords.SubmittedRecord recordToAck = submittedRecords.submit(PARTITION1, newOffset()); assertFalse(submittedRecords.awaitAllMessages(0, TimeUnit.MILLISECONDS)); recordToAck.ack(); assertTrue(submittedRecords.awaitAllMessages(0, TimeUnit.MILLISECONDS)); @@ -262,27 +261,27 @@ public void testAwaitMessagesAfterAllAcknowledged() { @Test public void testAwaitMessagesAfterAllRemoved() { - SubmittedRecord recordToRemove1 = submittedRecords.submit(PARTITION1, newOffset()); - SubmittedRecord recordToRemove2 = submittedRecords.submit(PARTITION1, newOffset()); + SubmittedRecords.SubmittedRecord recordToRemove1 = submittedRecords.submit(PARTITION1, newOffset()); + SubmittedRecords.SubmittedRecord recordToRemove2 = submittedRecords.submit(PARTITION1, newOffset()); assertFalse( "Await should fail since neither of the in-flight records has been removed so far", submittedRecords.awaitAllMessages(0, TimeUnit.MILLISECONDS) ); - submittedRecords.removeLastOccurrence(recordToRemove1); + recordToRemove1.drop(); assertFalse( "Await should fail since only one of the two submitted records has been removed so far", submittedRecords.awaitAllMessages(0, TimeUnit.MILLISECONDS) ); - submittedRecords.removeLastOccurrence(recordToRemove1); + recordToRemove1.drop(); assertFalse( "Await should fail since only one of the two submitted records has been removed so far, " + "even though that record has been removed twice", submittedRecords.awaitAllMessages(0, TimeUnit.MILLISECONDS) ); - submittedRecords.removeLastOccurrence(recordToRemove2); + recordToRemove2.drop(); assertTrue( "Await should succeed since both submitted records have now been removed", submittedRecords.awaitAllMessages(0, TimeUnit.MILLISECONDS) @@ -297,8 +296,8 @@ public void testAwaitMessagesTimesOut() { @Test public void testAwaitMessagesReturnsAfterAsynchronousAck() throws Exception { - SubmittedRecord inFlightRecord1 = submittedRecords.submit(PARTITION1, newOffset()); - SubmittedRecord inFlightRecord2 = submittedRecords.submit(PARTITION2, newOffset()); + SubmittedRecords.SubmittedRecord inFlightRecord1 = submittedRecords.submit(PARTITION1, newOffset()); + SubmittedRecords.SubmittedRecord inFlightRecord2 = submittedRecords.submit(PARTITION2, newOffset()); AtomicBoolean awaitResult = new AtomicBoolean(); CountDownLatch awaitComplete = new CountDownLatch(1); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerConnectorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerConnectorTest.java index f99b4c1067aff..4fec93613e785 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerConnectorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerConnectorTest.java @@ -24,7 +24,7 @@ import org.apache.kafka.connect.sink.SinkConnectorContext; import org.apache.kafka.connect.source.SourceConnector; import org.apache.kafka.connect.source.SourceConnectorContext; -import org.apache.kafka.connect.storage.OffsetStorageReader; +import org.apache.kafka.connect.storage.CloseableOffsetStorageReader; import org.easymock.Capture; import org.apache.kafka.connect.util.Callback; import org.easymock.EasyMock; @@ -65,7 +65,7 @@ public class WorkerConnectorTest extends EasyMockSupport { @Mock Connector connector; @Mock CloseableConnectorContext ctx; @Mock ConnectorStatus.Listener listener; - @Mock OffsetStorageReader offsetStorageReader; + @Mock CloseableOffsetStorageReader offsetStorageReader; @Mock ClassLoader classLoader; @Before @@ -99,6 +99,9 @@ public void testInitializeFailure() throws InterruptedException { ctx.close(); expectLastCall(); + offsetStorageReader.close(); + expectLastCall(); + replayAll(); WorkerConnector workerConnector = new WorkerConnector(CONNECTOR, connector, connectorConfig, ctx, metrics, listener, offsetStorageReader, classLoader); @@ -134,6 +137,9 @@ public void testFailureIsFinalState() { ctx.close(); expectLastCall(); + offsetStorageReader.close(); + expectLastCall(); + Callback onStateChange = createStrictMock(Callback.class); onStateChange.onCompletion(EasyMock.anyObject(Exception.class), EasyMock.isNull()); expectLastCall(); @@ -177,6 +183,9 @@ public void testStartupAndShutdown() { ctx.close(); expectLastCall(); + offsetStorageReader.close(); + expectLastCall(); + Callback onStateChange = createStrictMock(Callback.class); onStateChange.onCompletion(EasyMock.isNull(), EasyMock.eq(TargetState.STARTED)); expectLastCall(); @@ -223,6 +232,9 @@ public void testStartupAndPause() { ctx.close(); expectLastCall(); + offsetStorageReader.close(); + expectLastCall(); + Callback onStateChange = createStrictMock(Callback.class); onStateChange.onCompletion(EasyMock.isNull(), EasyMock.eq(TargetState.STARTED)); expectLastCall(); @@ -273,6 +285,9 @@ public void testOnResume() { ctx.close(); expectLastCall(); + offsetStorageReader.close(); + expectLastCall(); + Callback onStateChange = createStrictMock(Callback.class); onStateChange.onCompletion(EasyMock.isNull(), EasyMock.eq(TargetState.PAUSED)); expectLastCall(); @@ -316,6 +331,9 @@ public void testStartupPaused() { ctx.close(); expectLastCall(); + offsetStorageReader.close(); + expectLastCall(); + Callback onStateChange = createStrictMock(Callback.class); onStateChange.onCompletion(EasyMock.isNull(), EasyMock.eq(TargetState.PAUSED)); expectLastCall(); @@ -358,6 +376,9 @@ public void testStartupFailure() { ctx.close(); expectLastCall(); + offsetStorageReader.close(); + expectLastCall(); + Callback onStateChange = createStrictMock(Callback.class); onStateChange.onCompletion(EasyMock.anyObject(Exception.class), EasyMock.isNull()); expectLastCall(); @@ -407,6 +428,9 @@ public void testShutdownFailure() { ctx.close(); expectLastCall(); + offsetStorageReader.close(); + expectLastCall(); + replayAll(); WorkerConnector workerConnector = new WorkerConnector(CONNECTOR, connector, connectorConfig, ctx, metrics, listener, offsetStorageReader, classLoader); @@ -447,6 +471,9 @@ public void testTransitionStartedToStarted() { ctx.close(); expectLastCall(); + offsetStorageReader.close(); + expectLastCall(); + Callback onStateChange = createStrictMock(Callback.class); onStateChange.onCompletion(EasyMock.isNull(), EasyMock.eq(TargetState.STARTED)); expectLastCall().times(2); @@ -495,6 +522,9 @@ public void testTransitionPausedToPaused() { ctx.close(); expectLastCall(); + offsetStorageReader.close(); + expectLastCall(); + Callback onStateChange = createStrictMock(Callback.class); onStateChange.onCompletion(EasyMock.isNull(), EasyMock.eq(TargetState.STARTED)); expectLastCall(); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java index 1600dcfe110d1..ad6456ad51096 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskTest.java @@ -39,7 +39,7 @@ import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.errors.RetriableException; import org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.runtime.WorkerSinkTask.SinkTaskMetricsGroup; import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperatorTest; import org.apache.kafka.connect.runtime.isolation.PluginClassLoader; diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java index a7c6a8ace5198..5294423829563 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java @@ -29,7 +29,7 @@ import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperatorTest; import org.apache.kafka.connect.runtime.isolation.PluginClassLoader; import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java index 094062432b65b..5ee4819879f11 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSourceTaskTest.java @@ -16,17 +16,12 @@ */ package org.apache.kafka.connect.runtime; -import java.util.Collection; import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.InvalidRecordException; import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.MetricName; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.TopicPartitionInfo; import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.errors.TopicAuthorizationException; import org.apache.kafka.common.header.Header; @@ -34,14 +29,9 @@ import org.apache.kafka.common.header.internals.RecordHeaders; import org.apache.kafka.common.utils.Time; import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.errors.RetriableException; -import org.apache.kafka.connect.header.ConnectHeaders; import org.apache.kafka.connect.integration.MonitorableSourceConnector; import org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup; -import org.apache.kafka.connect.runtime.WorkerSourceTask.SourceTaskMetricsGroup; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator; import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperatorTest; import org.apache.kafka.connect.runtime.isolation.Plugins; @@ -50,6 +40,8 @@ import org.apache.kafka.connect.source.SourceTask; import org.apache.kafka.connect.source.SourceTaskContext; import org.apache.kafka.connect.storage.CloseableOffsetStorageReader; +import org.apache.kafka.connect.storage.ClusterConfigState; +import org.apache.kafka.connect.storage.ConnectorOffsetBackingStore; import org.apache.kafka.connect.storage.Converter; import org.apache.kafka.connect.storage.HeaderConverter; import org.apache.kafka.connect.storage.OffsetStorageWriter; @@ -76,10 +68,9 @@ import org.powermock.modules.junit4.PowerMockRunnerDelegate; import org.powermock.reflect.Whitebox; -import java.nio.ByteBuffer; import java.time.Duration; -import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -106,13 +97,12 @@ import static org.apache.kafka.connect.runtime.TopicCreationConfig.REPLICATION_FACTOR_CONFIG; import static org.apache.kafka.connect.runtime.WorkerConfig.TOPIC_CREATION_ENABLE_CONFIG; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @PowerMockIgnore({"javax.management.*", - "org.apache.log4j.*"}) + "org.apache.log4j.*"}) @RunWith(PowerMockRunner.class) @PowerMockRunnerDelegate(ParameterizedTest.class) public class WorkerSourceTaskTest extends ThreadedTest { @@ -147,6 +137,7 @@ public class WorkerSourceTaskTest extends ThreadedTest { @Mock private TopicAdmin admin; @Mock private CloseableOffsetStorageReader offsetReader; @Mock private OffsetStorageWriter offsetWriter; + @Mock private ConnectorOffsetBackingStore offsetStore; @Mock private ClusterConfigState clusterConfigState; private WorkerSourceTask workerTask; @Mock private Future sendFuture; @@ -235,16 +226,11 @@ private void createWorkerTask(TargetState initialState, RetryWithToleranceOperat createWorkerTask(initialState, keyConverter, valueConverter, headerConverter, retryWithToleranceOperator); } - private void createWorkerTask(TargetState initialState, Converter keyConverter, Converter valueConverter, - HeaderConverter headerConverter) { - createWorkerTask(initialState, keyConverter, valueConverter, headerConverter, RetryWithToleranceOperatorTest.NOOP_OPERATOR); - } - private void createWorkerTask(TargetState initialState, Converter keyConverter, Converter valueConverter, HeaderConverter headerConverter, RetryWithToleranceOperator retryWithToleranceOperator) { workerTask = new WorkerSourceTask(taskId, sourceTask, statusListener, initialState, keyConverter, valueConverter, headerConverter, - transformationChain, producer, admin, TopicCreationGroup.configuredGroups(sourceConfig), - offsetReader, offsetWriter, config, clusterConfigState, metrics, plugins.delegatingLoader(), Time.SYSTEM, + transformationChain, producer, admin, TopicCreationGroup.configuredGroups(sourceConfig), + offsetReader, offsetWriter, offsetStore, config, clusterConfigState, metrics, plugins.delegatingLoader(), Time.SYSTEM, retryWithToleranceOperator, statusBackingStore, Runnable::run); } @@ -644,93 +630,6 @@ public void testCommitFailure() throws Exception { PowerMock.verifyAll(); } - @Test - public void testSendRecordsConvertsData() throws Exception { - createWorkerTask(); - - List records = new ArrayList<>(); - // Can just use the same record for key and value - records.add(new SourceRecord(PARTITION, OFFSET, "topic", null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD)); - - Capture> sent = expectSendRecordAnyTimes(); - - expectTopicCreation(TOPIC); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", records); - Whitebox.invokeMethod(workerTask, "sendRecords"); - assertEquals(SERIALIZED_KEY, sent.getValue().key()); - assertEquals(SERIALIZED_RECORD, sent.getValue().value()); - - PowerMock.verifyAll(); - } - - @Test - public void testSendRecordsPropagatesTimestamp() throws Exception { - final Long timestamp = System.currentTimeMillis(); - - createWorkerTask(); - - List records = Collections.singletonList( - new SourceRecord(PARTITION, OFFSET, "topic", null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD, timestamp) - ); - - Capture> sent = expectSendRecordAnyTimes(); - - expectTopicCreation(TOPIC); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", records); - Whitebox.invokeMethod(workerTask, "sendRecords"); - assertEquals(timestamp, sent.getValue().timestamp()); - - PowerMock.verifyAll(); - } - - @Test - public void testSendRecordsCorruptTimestamp() throws Exception { - final Long timestamp = -3L; - createWorkerTask(); - - List records = Collections.singletonList( - new SourceRecord(PARTITION, OFFSET, "topic", null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD, timestamp) - ); - - Capture> sent = expectSendRecordAnyTimes(); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", records); - assertThrows(InvalidRecordException.class, () -> Whitebox.invokeMethod(workerTask, "sendRecords")); - assertFalse(sent.hasCaptured()); - - PowerMock.verifyAll(); - } - - @Test - public void testSendRecordsNoTimestamp() throws Exception { - final Long timestamp = -1L; - createWorkerTask(); - - List records = Collections.singletonList( - new SourceRecord(PARTITION, OFFSET, "topic", null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD, timestamp) - ); - - Capture> sent = expectSendRecordAnyTimes(); - - expectTopicCreation(TOPIC); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", records); - Whitebox.invokeMethod(workerTask, "sendRecords"); - assertNull(sent.getValue().timestamp()); - - PowerMock.verifyAll(); - } - @Test public void testSendRecordsRetries() throws Exception { createWorkerTask(); @@ -775,6 +674,8 @@ public void testSendRecordsProducerCallbackFail() throws Exception { expectTopicCreation(TOPIC); expectSendRecordProducerCallbackFail(); + expectApplyTransformationChain(false); + expectConvertHeadersAndKeyValue(false); PowerMock.replayAll(); @@ -797,7 +698,7 @@ public void testSendRecordsProducerSendFailsImmediately() { expectTopicCreation(TOPIC); EasyMock.expect(producer.send(EasyMock.anyObject(), EasyMock.anyObject())) - .andThrow(new KafkaException("Producer closed while send in progress", new InvalidTopicException(TOPIC))); + .andThrow(new KafkaException("Producer closed while send in progress", new InvalidTopicException(TOPIC))); PowerMock.replayAll(); @@ -842,7 +743,6 @@ public void testSourceTaskIgnoresProducerException() throws Exception { SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - expectSendRecordOnce(); expectSendRecordProducerCallbackFail(); sourceTask.commitRecord(EasyMock.anyObject(SourceRecord.class), EasyMock.isNull()); @@ -919,440 +819,12 @@ public void testCancel() { PowerMock.verifyAll(); } - @Test - public void testMetricsGroup() { - SourceTaskMetricsGroup group = new SourceTaskMetricsGroup(taskId, metrics); - SourceTaskMetricsGroup group1 = new SourceTaskMetricsGroup(taskId1, metrics); - for (int i = 0; i != 10; ++i) { - group.recordPoll(100, 1000 + i * 100); - group.recordWrite(10); - } - for (int i = 0; i != 20; ++i) { - group1.recordPoll(100, 1000 + i * 100); - group1.recordWrite(10); - } - assertEquals(1900.0, metrics.currentMetricValueAsDouble(group.metricGroup(), "poll-batch-max-time-ms"), 0.001d); - assertEquals(1450.0, metrics.currentMetricValueAsDouble(group.metricGroup(), "poll-batch-avg-time-ms"), 0.001d); - assertEquals(33.333, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-poll-rate"), 0.001d); - assertEquals(1000, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-poll-total"), 0.001d); - assertEquals(3.3333, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-write-rate"), 0.001d); - assertEquals(100, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-write-total"), 0.001d); - assertEquals(900.0, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-active-count"), 0.001d); - - // Close the group - group.close(); - - for (MetricName metricName : group.metricGroup().metrics().metrics().keySet()) { - // Metrics for this group should no longer exist - assertFalse(group.metricGroup().groupId().includes(metricName)); - } - // Sensors for this group should no longer exist - assertNull(group.metricGroup().metrics().getSensor("sink-record-read")); - assertNull(group.metricGroup().metrics().getSensor("sink-record-send")); - assertNull(group.metricGroup().metrics().getSensor("sink-record-active-count")); - assertNull(group.metricGroup().metrics().getSensor("partition-count")); - assertNull(group.metricGroup().metrics().getSensor("offset-seq-number")); - assertNull(group.metricGroup().metrics().getSensor("offset-commit-completion")); - assertNull(group.metricGroup().metrics().getSensor("offset-commit-completion-skip")); - assertNull(group.metricGroup().metrics().getSensor("put-batch-time")); - - assertEquals(2900.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "poll-batch-max-time-ms"), 0.001d); - assertEquals(1950.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "poll-batch-avg-time-ms"), 0.001d); - assertEquals(66.667, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-poll-rate"), 0.001d); - assertEquals(2000, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-poll-total"), 0.001d); - assertEquals(6.667, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-write-rate"), 0.001d); - assertEquals(200, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-write-total"), 0.001d); - assertEquals(1800.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-active-count"), 0.001d); - } - - @Test - public void testHeaders() throws Exception { - Headers headers = new RecordHeaders(); - headers.add("header_key", "header_value".getBytes()); - - org.apache.kafka.connect.header.Headers connectHeaders = new ConnectHeaders(); - connectHeaders.add("header_key", new SchemaAndValue(Schema.STRING_SCHEMA, "header_value")); - - createWorkerTask(); - - List records = new ArrayList<>(); - records.add(new SourceRecord(PARTITION, OFFSET, TOPIC, null, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD, null, connectHeaders)); - - expectTopicCreation(TOPIC); - - Capture> sent = expectSendRecord(TOPIC, true, true, true, true, headers); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", records); - Whitebox.invokeMethod(workerTask, "sendRecords"); - assertEquals(SERIALIZED_KEY, sent.getValue().key()); - assertEquals(SERIALIZED_RECORD, sent.getValue().value()); - assertEquals(headers, sent.getValue().headers()); - - PowerMock.verifyAll(); - } - - @Test - public void testHeadersWithCustomConverter() throws Exception { - StringConverter stringConverter = new StringConverter(); - TestConverterWithHeaders testConverter = new TestConverterWithHeaders(); - - createWorkerTask(TargetState.STARTED, stringConverter, testConverter, stringConverter); - - List records = new ArrayList<>(); - - String stringA = "Árvíztűrő tükörfúrógép"; - org.apache.kafka.connect.header.Headers headersA = new ConnectHeaders(); - String encodingA = "latin2"; - headersA.addString("encoding", encodingA); - - records.add(new SourceRecord(PARTITION, OFFSET, "topic", null, Schema.STRING_SCHEMA, "a", Schema.STRING_SCHEMA, stringA, null, headersA)); - - String stringB = "Тестовое сообщение"; - org.apache.kafka.connect.header.Headers headersB = new ConnectHeaders(); - String encodingB = "koi8_r"; - headersB.addString("encoding", encodingB); - - records.add(new SourceRecord(PARTITION, OFFSET, "topic", null, Schema.STRING_SCHEMA, "b", Schema.STRING_SCHEMA, stringB, null, headersB)); - - expectTopicCreation(TOPIC); - - Capture> sentRecordA = expectSendRecord(TOPIC, false, true, true, false, null); - Capture> sentRecordB = expectSendRecord(TOPIC, false, true, true, false, null); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", records); - Whitebox.invokeMethod(workerTask, "sendRecords"); - - assertEquals(ByteBuffer.wrap("a".getBytes()), ByteBuffer.wrap(sentRecordA.getValue().key())); - assertEquals( - ByteBuffer.wrap(stringA.getBytes(encodingA)), - ByteBuffer.wrap(sentRecordA.getValue().value()) - ); - assertEquals(encodingA, new String(sentRecordA.getValue().headers().lastHeader("encoding").value())); - - assertEquals(ByteBuffer.wrap("b".getBytes()), ByteBuffer.wrap(sentRecordB.getValue().key())); - assertEquals( - ByteBuffer.wrap(stringB.getBytes(encodingB)), - ByteBuffer.wrap(sentRecordB.getValue().value()) - ); - assertEquals(encodingB, new String(sentRecordB.getValue().headers().lastHeader("encoding").value())); - - PowerMock.verifyAll(); - } - - @Test - public void testTopicCreateWhenTopicExists() throws Exception { - if (!enableTopicCreation) - // should only test with topic creation enabled - return; - - createWorkerTask(); - - SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - - expectPreliminaryCalls(); - TopicPartitionInfo topicPartitionInfo = new TopicPartitionInfo(0, null, Collections.emptyList(), Collections.emptyList()); - TopicDescription topicDesc = new TopicDescription(TOPIC, false, Collections.singletonList(topicPartitionInfo)); - EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.singletonMap(TOPIC, topicDesc)); - - expectSendRecordTaskCommitRecordSucceed(false); - expectSendRecordTaskCommitRecordSucceed(false); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2)); - Whitebox.invokeMethod(workerTask, "sendRecords"); - } - - @Test - public void testSendRecordsTopicDescribeRetries() throws Exception { - if (!enableTopicCreation) - // should only test with topic creation enabled - return; - - createWorkerTask(); - - SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - - expectPreliminaryCalls(); - // First round - call to describe the topic times out - EasyMock.expect(admin.describeTopics(TOPIC)) - .andThrow(new RetriableException(new TimeoutException("timeout"))); - - // Second round - calls to describe and create succeed - expectTopicCreation(TOPIC); - // Exactly two records are sent - expectSendRecordTaskCommitRecordSucceed(false); - expectSendRecordTaskCommitRecordSucceed(false); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2)); - Whitebox.invokeMethod(workerTask, "sendRecords"); - assertEquals(Arrays.asList(record1, record2), Whitebox.getInternalState(workerTask, "toSend")); - - // Next they all succeed - Whitebox.invokeMethod(workerTask, "sendRecords"); - assertNull(Whitebox.getInternalState(workerTask, "toSend")); - } - - @Test - public void testSendRecordsTopicCreateRetries() throws Exception { - if (!enableTopicCreation) - // should only test with topic creation enabled - return; - - createWorkerTask(); - - SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - - // First call to describe the topic times out - expectPreliminaryCalls(); - EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap()); - Capture newTopicCapture = EasyMock.newCapture(); - EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))) - .andThrow(new RetriableException(new TimeoutException("timeout"))); - - // Second round - expectTopicCreation(TOPIC); - expectSendRecordTaskCommitRecordSucceed(false); - expectSendRecordTaskCommitRecordSucceed(false); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2)); - Whitebox.invokeMethod(workerTask, "sendRecords"); - assertEquals(Arrays.asList(record1, record2), Whitebox.getInternalState(workerTask, "toSend")); - - // Next they all succeed - Whitebox.invokeMethod(workerTask, "sendRecords"); - assertNull(Whitebox.getInternalState(workerTask, "toSend")); - } - - @Test - public void testSendRecordsTopicDescribeRetriesMidway() throws Exception { - if (!enableTopicCreation) - // should only test with topic creation enabled - return; - - createWorkerTask(); - - // Differentiate only by Kafka partition so we can reuse conversion expectations - SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - SourceRecord record3 = new SourceRecord(PARTITION, OFFSET, OTHER_TOPIC, 3, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - - // First round - expectPreliminaryCalls(OTHER_TOPIC); - expectTopicCreation(TOPIC); - expectSendRecordTaskCommitRecordSucceed(false); - expectSendRecordTaskCommitRecordSucceed(false); - - // First call to describe the topic times out - EasyMock.expect(admin.describeTopics(OTHER_TOPIC)) - .andThrow(new RetriableException(new TimeoutException("timeout"))); - - // Second round - expectTopicCreation(OTHER_TOPIC); - expectSendRecord(OTHER_TOPIC, false, true, true, true, emptyHeaders()); - - PowerMock.replayAll(); - - // Try to send 3, make first pass, second fail. Should save last two - Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2, record3)); - Whitebox.invokeMethod(workerTask, "sendRecords"); - assertEquals(Arrays.asList(record3), Whitebox.getInternalState(workerTask, "toSend")); - - // Next they all succeed - Whitebox.invokeMethod(workerTask, "sendRecords"); - assertNull(Whitebox.getInternalState(workerTask, "toSend")); - - PowerMock.verifyAll(); - } - - @Test - public void testSendRecordsTopicCreateRetriesMidway() throws Exception { - if (!enableTopicCreation) - // should only test with topic creation enabled - return; - - createWorkerTask(); - - // Differentiate only by Kafka partition so we can reuse conversion expectations - SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - SourceRecord record3 = new SourceRecord(PARTITION, OFFSET, OTHER_TOPIC, 3, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - - // First round - expectPreliminaryCalls(OTHER_TOPIC); - expectTopicCreation(TOPIC); - expectSendRecordTaskCommitRecordSucceed(false); - expectSendRecordTaskCommitRecordSucceed(false); - - EasyMock.expect(admin.describeTopics(OTHER_TOPIC)).andReturn(Collections.emptyMap()); - // First call to create the topic times out - Capture newTopicCapture = EasyMock.newCapture(); - EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))) - .andThrow(new RetriableException(new TimeoutException("timeout"))); - - // Second round - expectTopicCreation(OTHER_TOPIC); - expectSendRecord(OTHER_TOPIC, false, true, true, true, emptyHeaders()); - - PowerMock.replayAll(); - - // Try to send 3, make first pass, second fail. Should save last two - Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2, record3)); - Whitebox.invokeMethod(workerTask, "sendRecords"); - assertEquals(Arrays.asList(record3), Whitebox.getInternalState(workerTask, "toSend")); - - // Next they all succeed - Whitebox.invokeMethod(workerTask, "sendRecords"); - assertNull(Whitebox.getInternalState(workerTask, "toSend")); - - PowerMock.verifyAll(); - } - - @Test - public void testTopicDescribeFails() { - if (!enableTopicCreation) - // should only test with topic creation enabled - return; - - createWorkerTask(); - - SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - - expectPreliminaryCalls(); - EasyMock.expect(admin.describeTopics(TOPIC)) - .andThrow(new ConnectException(new TopicAuthorizationException("unauthorized"))); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2)); - assertThrows(ConnectException.class, () -> Whitebox.invokeMethod(workerTask, "sendRecords")); - } - - @Test - public void testTopicCreateFails() throws Exception { - if (!enableTopicCreation) - // should only test with topic creation enabled - return; - - createWorkerTask(); - - SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - - expectPreliminaryCalls(); - EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap()); - - Capture newTopicCapture = EasyMock.newCapture(); - EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))) - .andThrow(new ConnectException(new TopicAuthorizationException("unauthorized"))); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2)); - assertThrows(ConnectException.class, () -> Whitebox.invokeMethod(workerTask, "sendRecords")); - assertTrue(newTopicCapture.hasCaptured()); - } - - @Test - public void testTopicCreateFailsWithExceptionWhenCreateReturnsTopicNotCreatedOrFound() throws Exception { - if (!enableTopicCreation) - // should only test with topic creation enabled - return; - - createWorkerTask(); - - SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - - expectPreliminaryCalls(); - EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap()); - - Capture newTopicCapture = EasyMock.newCapture(); - EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andReturn(TopicAdmin.EMPTY_CREATION); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2)); - assertThrows(ConnectException.class, () -> Whitebox.invokeMethod(workerTask, "sendRecords")); - assertTrue(newTopicCapture.hasCaptured()); - } - - @Test - public void testTopicCreateSucceedsWhenCreateReturnsExistingTopicFound() throws Exception { - if (!enableTopicCreation) - // should only test with topic creation enabled - return; - - createWorkerTask(); - - SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - - expectPreliminaryCalls(); - EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap()); - - Capture newTopicCapture = EasyMock.newCapture(); - EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andReturn(foundTopic(TOPIC)); - - expectSendRecordTaskCommitRecordSucceed(false); - expectSendRecordTaskCommitRecordSucceed(false); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2)); - Whitebox.invokeMethod(workerTask, "sendRecords"); - } - - @Test - public void testTopicCreateSucceedsWhenCreateReturnsNewTopicFound() throws Exception { - if (!enableTopicCreation) - // should only test with topic creation enabled - return; - - createWorkerTask(); - - SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD); - - expectPreliminaryCalls(); - EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap()); - - Capture newTopicCapture = EasyMock.newCapture(); - EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andReturn(createdTopic(TOPIC)); - - expectSendRecordTaskCommitRecordSucceed(false); - expectSendRecordTaskCommitRecordSucceed(false); - - PowerMock.replayAll(); - - Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2)); - Whitebox.invokeMethod(workerTask, "sendRecords"); - } - private TopicAdmin.TopicCreationResponse createdTopic(String topic) { Set created = Collections.singleton(topic); Set existing = Collections.emptySet(); return new TopicAdmin.TopicCreationResponse(created, existing); } - private TopicAdmin.TopicCreationResponse foundTopic(String topic) { - Set created = Collections.emptySet(); - Set existing = Collections.singleton(topic); - return new TopicAdmin.TopicCreationResponse(created, existing); - } - private void expectPreliminaryCalls() { expectPreliminaryCalls(TOPIC); } @@ -1404,9 +876,9 @@ private void expectSendRecordSyncFailure(Throwable error) throws InterruptedExce expectApplyTransformationChain(false); EasyMock.expect( - producer.send(EasyMock.anyObject(ProducerRecord.class), - EasyMock.anyObject(org.apache.kafka.clients.producer.Callback.class))) - .andThrow(error); + producer.send(EasyMock.anyObject(ProducerRecord.class), + EasyMock.anyObject(org.apache.kafka.clients.producer.Callback.class))) + .andThrow(error); } private Capture> expectSendRecordAnyTimes() throws InterruptedException { @@ -1430,12 +902,12 @@ private Capture> expectSendRecordTaskCommitRecord } private Capture> expectSendRecord( - String topic, - boolean anyTimes, - boolean sendSuccess, - boolean commitSuccess, - boolean isMockedConverters, - Headers headers + String topic, + boolean anyTimes, + boolean sendSuccess, + boolean commitSuccess, + boolean isMockedConverters, + Headers headers ) throws InterruptedException { if (isMockedConverters) { expectConvertHeadersAndKeyValue(topic, anyTimes, headers); @@ -1447,14 +919,14 @@ private Capture> expectSendRecord( // 1. Converted data passed to the producer, which will need callbacks invoked for flush to work IExpectationSetters> expect = EasyMock.expect( - producer.send(EasyMock.capture(sent), - EasyMock.capture(producerCallbacks))); + producer.send(EasyMock.capture(sent), + EasyMock.capture(producerCallbacks))); IAnswer> expectResponse = () -> { synchronized (producerCallbacks) { for (org.apache.kafka.clients.producer.Callback cb : producerCallbacks.getValues()) { if (sendSuccess) { cb.onCompletion(new RecordMetadata(new TopicPartition("foo", 0), 0, 0, - 0L, 0, 0), null); + 0L, 0, 0), null); } else { cb.onCompletion(null, new TopicAuthorizationException("foo")); } @@ -1625,6 +1097,12 @@ private void expectClose() { transformationChain.close(); EasyMock.expectLastCall(); + + offsetReader.close(); + EasyMock.expectLastCall(); + + offsetStore.stop(); + EasyMock.expectLastCall(); } private void expectTopicCreation(String topic) { diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java index 046a9d90d3c32..88820a71a7d37 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTest.java @@ -18,10 +18,15 @@ import java.util.Collection; import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.FenceProducersResult; +import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.Configurable; +import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.config.AbstractConfig; import org.apache.kafka.common.config.ConfigDef; @@ -39,11 +44,15 @@ import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaAndValue; import org.apache.kafka.connect.errors.ConnectException; +import org.apache.kafka.connect.health.ConnectorType; import org.apache.kafka.connect.json.JsonConverter; import org.apache.kafka.connect.json.JsonConverterConfig; import org.apache.kafka.connect.runtime.ConnectMetrics.MetricGroup; import org.apache.kafka.connect.runtime.MockConnectMetrics.MockMetricsReporter; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; +import org.apache.kafka.connect.runtime.distributed.DistributedConfig; +import org.apache.kafka.connect.runtime.errors.WorkerErrantRecordReporter; +import org.apache.kafka.connect.sink.SinkRecord; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator; import org.apache.kafka.connect.runtime.isolation.DelegatingClassLoader; import org.apache.kafka.connect.runtime.isolation.PluginClassLoader; @@ -56,6 +65,7 @@ import org.apache.kafka.connect.source.SourceConnector; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.storage.ConnectorOffsetBackingStore; import org.apache.kafka.connect.storage.Converter; import org.apache.kafka.connect.storage.HeaderConverter; import org.apache.kafka.connect.storage.OffsetBackingStore; @@ -99,13 +109,32 @@ import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.function.Supplier; + import org.powermock.modules.junit4.PowerMockRunnerDelegate; +import static org.apache.kafka.clients.admin.AdminClientConfig.RETRY_BACKOFF_MS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.ISOLATION_LEVEL_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.TRANSACTIONAL_ID_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX; +import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX; import static org.apache.kafka.connect.runtime.TopicCreationConfig.DEFAULT_TOPIC_CREATION_PREFIX; import static org.apache.kafka.connect.runtime.TopicCreationConfig.PARTITIONS_CONFIG; import static org.apache.kafka.connect.runtime.TopicCreationConfig.REPLICATION_FACTOR_CONFIG; +import static org.apache.kafka.connect.runtime.WorkerConfig.BOOTSTRAP_SERVERS_CONFIG; import static org.apache.kafka.connect.runtime.WorkerConfig.TOPIC_CREATION_ENABLE_CONFIG; +import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.CONFIG_TOPIC_CONFIG; +import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG; +import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.GROUP_ID_CONFIG; +import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG; +import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.STATUS_STORAGE_TOPIC_CONFIG; import static org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperatorTest.NOOP_OPERATOR; +import static org.apache.kafka.connect.sink.SinkTask.TOPICS_CONFIG; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expectLastCall; @@ -116,12 +145,13 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @RunWith(PowerMockRunner.class) @PowerMockRunnerDelegate(ParameterizedTest.class) @PrepareForTest({Worker.class, Plugins.class, ConnectUtils.class}) -@PowerMockIgnore("javax.management.*") +@PowerMockIgnore({"javax.management.*", "javax.crypto.*"}) public class WorkerTest extends ThreadedTest { private static final String CONNECTOR_ID = "test-connector"; @@ -168,7 +198,7 @@ public class WorkerTest extends ThreadedTest { private String mockFileProviderTestId; private Map connectorProps; - private boolean enableTopicCreation; + private final boolean enableTopicCreation; @ParameterizedTest.Parameters public static Collection parameters() { @@ -203,7 +233,7 @@ public void setup() { defaultProducerConfigs.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1"); defaultProducerConfigs.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, Integer.toString(Integer.MAX_VALUE)); - defaultConsumerConfigs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + defaultConsumerConfigs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); defaultConsumerConfigs.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); defaultConsumerConfigs.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); defaultConsumerConfigs @@ -221,7 +251,7 @@ public void testStartAndStopConnector() throws Throwable { expectConverters(); expectStartStorage(); - final String connectorClass = WorkerTestConnector.class.getName(); + final String connectorClass = WorkerTestSourceConnector.class.getName(); // Create EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader).times(2); @@ -231,7 +261,7 @@ public void testStartAndStopConnector() throws Throwable { .andReturn(sourceConnector); EasyMock.expect(sourceConnector.version()).andReturn("1.0"); - connectorProps.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, connectorClass); + connectorProps.put(CONNECTOR_CLASS_CONFIG, connectorClass); EasyMock.expect(sourceConnector.version()).andReturn("1.0"); @@ -317,7 +347,7 @@ public void testStartConnectorFailure() throws Exception { expectFileConfigProvider(); final String nonConnectorClass = "java.util.HashMap"; - connectorProps.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, nonConnectorClass); // Bad connector class name + connectorProps.put(CONNECTOR_CLASS_CONFIG, nonConnectorClass); // Bad connector class name Exception exception = new ConnectException("Failed to find Connector"); EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader); @@ -378,7 +408,7 @@ public void testAddConnectorByAlias() throws Throwable { EasyMock.expect(plugins.newConnector(connectorAlias)).andReturn(sinkConnector); EasyMock.expect(sinkConnector.version()).andReturn("1.0"); - connectorProps.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, connectorAlias); + connectorProps.put(CONNECTOR_CLASS_CONFIG, connectorAlias); connectorProps.put(SinkConnectorConfig.TOPICS_CONFIG, "gfieyls, wfru"); EasyMock.expect(sinkConnector.version()).andReturn("1.0"); @@ -452,7 +482,7 @@ public void testAddConnectorByShortAlias() throws Throwable { EasyMock.expect(plugins.newConnector(shortConnectorAlias)).andReturn(sinkConnector); EasyMock.expect(sinkConnector.version()).andReturn("1.0"); - connectorProps.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, shortConnectorAlias); + connectorProps.put(CONNECTOR_CLASS_CONFIG, shortConnectorAlias); connectorProps.put(SinkConnectorConfig.TOPICS_CONFIG, "gfieyls, wfru"); EasyMock.expect(sinkConnector.version()).andReturn("1.0"); @@ -533,7 +563,7 @@ public void testReconfigureConnectorTasks() throws Throwable { expectStartStorage(); expectFileConfigProvider(); - final String connectorClass = WorkerTestConnector.class.getName(); + final String connectorClass = WorkerTestSourceConnector.class.getName(); EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader).times(3); EasyMock.expect(plugins.delegatingLoader()).andReturn(delegatingLoader).times(1); @@ -543,7 +573,7 @@ public void testReconfigureConnectorTasks() throws Throwable { EasyMock.expect(sinkConnector.version()).andReturn("1.0"); connectorProps.put(SinkConnectorConfig.TOPICS_CONFIG, "foo,bar"); - connectorProps.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, connectorClass); + connectorProps.put(CONNECTOR_CLASS_CONFIG, connectorClass); EasyMock.expect(sinkConnector.version()).andReturn("1.0"); EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)) @@ -611,7 +641,7 @@ public void testReconfigureConnectorTasks() throws Throwable { Map expectedTaskProps = new HashMap<>(); expectedTaskProps.put("foo", "bar"); expectedTaskProps.put(TaskConfig.TASK_CLASS_CONFIG, TestSourceTask.class.getName()); - expectedTaskProps.put(SinkTask.TOPICS_CONFIG, "foo,bar"); + expectedTaskProps.put(TOPICS_CONFIG, "foo,bar"); assertEquals(2, taskConfigs.size()); assertEquals(expectedTaskProps, taskConfigs.get(0)); assertEquals(expectedTaskProps, taskConfigs.get(1)); @@ -637,7 +667,7 @@ public void testAddRemoveTask() throws Exception { EasyMock.expect(workerTask.id()).andStubReturn(TASK_ID); EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader).times(2); - expectNewWorkerTask(); + expectNewWorkerSourceTask(); Map origProps = new HashMap<>(); origProps.put(TaskConfig.TASK_CLASS_CONFIG, TestSourceTask.class.getName()); @@ -662,7 +692,7 @@ public void testAddRemoveTask() throws Exception { EasyMock.expect(executorService.submit(workerTask)).andReturn(null); EasyMock.expect(plugins.delegatingLoader()).andReturn(delegatingLoader); - EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestConnector.class.getName())) + EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestSourceConnector.class.getName())) .andReturn(pluginLoader); EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)).andReturn(delegatingLoader) .times(2); @@ -671,8 +701,8 @@ public void testAddRemoveTask() throws Exception { EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader) .times(2); - plugins.connectorClass(WorkerTestConnector.class.getName()); - EasyMock.expectLastCall().andReturn(WorkerTestConnector.class); + plugins.connectorClass(WorkerTestSourceConnector.class.getName()); + EasyMock.expectLastCall().andReturn(WorkerTestSourceConnector.class); // Remove workerTask.stop(); EasyMock.expectLastCall(); @@ -693,7 +723,7 @@ public void testAddRemoveTask() throws Exception { worker.start(); assertStatistics(worker, 0, 0); assertEquals(Collections.emptySet(), worker.taskIds()); - worker.startTask(TASK_ID, ClusterConfigState.EMPTY, anyConnectorConfigMap(), origProps, taskStatusListener, TargetState.STARTED); + worker.startSourceTask(TASK_ID, ClusterConfigState.EMPTY, anyConnectorConfigMap(), origProps, taskStatusListener, TargetState.STARTED); assertStatistics(worker, 0, 1); assertEquals(new HashSet<>(Arrays.asList(TASK_ID)), worker.taskIds()); worker.stopAndAwaitTask(TASK_ID); @@ -706,6 +736,160 @@ public void testAddRemoveTask() throws Exception { PowerMock.verifyAll(); } + @Test + public void testAddSinkTask() throws Exception { + // Most of the other cases use source tasks; we make sure to get code coverage for sink tasks here as well + expectConverters(); + expectStartStorage(); + expectFileConfigProvider(); + + EasyMock.expect(workerTask.id()).andStubReturn(TASK_ID); + + EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader); + WorkerSinkTask workerTask = EasyMock.mock(WorkerSinkTask.class); + SinkTask task = EasyMock.mock(SinkTask.class); + expectNewWorkerSinkTask(workerTask, task); + Map origProps = new HashMap<>(); + origProps.put(CONNECTOR_CLASS_CONFIG, WorkerSinkTask.class.getName()); + origProps.put(TaskConfig.TASK_CLASS_CONFIG, TestSinkTask.class.getName()); + + TaskConfig taskConfig = new TaskConfig(origProps); + // We should expect this call, but the pluginLoader being swapped in is only mocked. + // EasyMock.expect(pluginLoader.loadClass(TestSinkTask.class.getName())) + // .andReturn((Class) TestSinkTask.class); + EasyMock.expect(plugins.newTask(TestSinkTask.class)).andReturn(task); + EasyMock.expect(task.version()).andReturn("1.0"); + + workerTask.initialize(taskConfig); + EasyMock.expectLastCall(); + + // Expect that the worker will create converters and will find them using the current classloader ... + assertNotNull(taskKeyConverter); + assertNotNull(taskValueConverter); + assertNotNull(taskHeaderConverter); + expectTaskKeyConverters(ClassLoaderUsage.CURRENT_CLASSLOADER, taskKeyConverter); + expectTaskValueConverters(ClassLoaderUsage.CURRENT_CLASSLOADER, taskValueConverter); + expectTaskHeaderConverter(ClassLoaderUsage.CURRENT_CLASSLOADER, taskHeaderConverter); + + EasyMock.expect(executorService.submit(workerTask)).andReturn(null); + + EasyMock.expect(plugins.delegatingLoader()).andReturn(delegatingLoader); + EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestSinkConnector.class.getName())) + .andReturn(pluginLoader); + EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)).andReturn(delegatingLoader); + + EasyMock.expect(workerTask.loader()).andReturn(pluginLoader); + + EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader); + plugins.connectorClass(WorkerTestSinkConnector.class.getName()); + EasyMock.expectLastCall().andReturn(WorkerTestSinkConnector.class); + + expectClusterId(); + + PowerMock.replayAll(); + + worker = new Worker(WORKER_ID, new MockTime(), plugins, config, offsetBackingStore, executorService, + noneConnectorClientConfigOverridePolicy); + worker.herder = herder; + worker.start(); + assertStatistics(worker, 0, 0); + assertStartupStatistics(worker, 0, 0, 0, 0); + assertEquals(Collections.emptySet(), worker.taskIds()); + + Map connectorConfigs = anyConnectorConfigMap(); + connectorConfigs.put(TOPICS_CONFIG, "t1"); + connectorConfigs.put(CONNECTOR_CLASS_CONFIG, WorkerTestSinkConnector.class.getName()); + + worker.startSinkTask(TASK_ID, ClusterConfigState.EMPTY, connectorConfigs, origProps, taskStatusListener, TargetState.STARTED); + assertStatistics(worker, 0, 1); + assertEquals(new HashSet<>(Arrays.asList(TASK_ID)), worker.taskIds()); + + PowerMock.verifyAll(); + } + + @Test + public void testAddExactlyOnceSourceTask() throws Exception { + Map workerProps = new HashMap<>(); + workerProps.put("key.converter", "org.apache.kafka.connect.json.JsonConverter"); + workerProps.put("value.converter", "org.apache.kafka.connect.json.JsonConverter"); + workerProps.put(CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); + workerProps.put("config.providers", "file"); + workerProps.put("config.providers.file.class", MockFileConfigProvider.class.getName()); + mockFileProviderTestId = UUID.randomUUID().toString(); + workerProps.put("config.providers.file.param.testId", mockFileProviderTestId); + workerProps.put(TOPIC_CREATION_ENABLE_CONFIG, String.valueOf(enableTopicCreation)); + workerProps.put(GROUP_ID_CONFIG, "connect-cluster"); + workerProps.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:2606"); + workerProps.put(OFFSET_STORAGE_TOPIC_CONFIG, "connect-offsets"); + workerProps.put(CONFIG_TOPIC_CONFIG, "connect-configs"); + workerProps.put(STATUS_STORAGE_TOPIC_CONFIG, "connect-statuses"); + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "enabled"); + config = new DistributedConfig(workerProps); + + // Most of the other cases use vanilla source tasks; we make sure to get code coverage for exactly-once source tasks here as well + expectConverters(); + expectStartStorage(); + expectFileConfigProvider(); + + EasyMock.expect(workerTask.id()).andStubReturn(TASK_ID); + + EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader); + ExactlyOnceWorkerSourceTask workerTask = EasyMock.mock(ExactlyOnceWorkerSourceTask.class); + Runnable preProducer = EasyMock.mock(Runnable.class); + Runnable postProducer = EasyMock.mock(Runnable.class); + expectNewExactlyOnceWorkerSourceTask(workerTask, preProducer, postProducer); + Map origProps = new HashMap<>(); + origProps.put(TaskConfig.TASK_CLASS_CONFIG, TestSourceTask.class.getName()); + + TaskConfig taskConfig = new TaskConfig(origProps); + // We should expect this call, but the pluginLoader being swapped in is only mocked. + // EasyMock.expect(pluginLoader.loadClass(TestSinkTask.class.getName())) + // .andReturn((Class) TestSinkTask.class); + EasyMock.expect(plugins.newTask(TestSourceTask.class)).andReturn(task); + EasyMock.expect(task.version()).andReturn("1.0"); + + workerTask.initialize(taskConfig); + EasyMock.expectLastCall(); + + // Expect that the worker will create converters and will find them using the current classloader ... + assertNotNull(taskKeyConverter); + assertNotNull(taskValueConverter); + assertNotNull(taskHeaderConverter); + expectTaskKeyConverters(ClassLoaderUsage.CURRENT_CLASSLOADER, taskKeyConverter); + expectTaskValueConverters(ClassLoaderUsage.CURRENT_CLASSLOADER, taskValueConverter); + expectTaskHeaderConverter(ClassLoaderUsage.CURRENT_CLASSLOADER, taskHeaderConverter); + + EasyMock.expect(executorService.submit(workerTask)).andReturn(null); + + EasyMock.expect(plugins.delegatingLoader()).andReturn(delegatingLoader); + EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestSourceConnector.class.getName())) + .andReturn(pluginLoader); + EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)).andReturn(delegatingLoader); + + EasyMock.expect(workerTask.loader()).andReturn(pluginLoader); + + EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader); + plugins.connectorClass(WorkerTestSourceConnector.class.getName()); + EasyMock.expectLastCall().andReturn(WorkerTestSourceConnector.class); + + expectClusterId(); + + PowerMock.replayAll(); + + worker = new Worker(WORKER_ID, new MockTime(), plugins, config, offsetBackingStore, executorService, + noneConnectorClientConfigOverridePolicy); + worker.herder = herder; + worker.start(); + assertStatistics(worker, 0, 0); + assertStartupStatistics(worker, 0, 0, 0, 0); + assertEquals(Collections.emptySet(), worker.taskIds()); + worker.startExactlyOnceSourceTask(TASK_ID, ClusterConfigState.EMPTY, anyConnectorConfigMap(), origProps, taskStatusListener, TargetState.STARTED, preProducer, postProducer); + assertStatistics(worker, 0, 1); + assertEquals(new HashSet<>(Arrays.asList(TASK_ID)), worker.taskIds()); + + PowerMock.verifyAll(); + } + @Test public void testTaskStatusMetricsStatuses() throws Exception { expectConverters(); @@ -715,7 +899,7 @@ public void testTaskStatusMetricsStatuses() throws Exception { EasyMock.expect(workerTask.id()).andStubReturn(TASK_ID); EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader).times(2); - expectNewWorkerTask(); + expectNewWorkerSourceTask(); Map origProps = new HashMap<>(); origProps.put(TaskConfig.TASK_CLASS_CONFIG, TestSourceTask.class.getName()); @@ -740,15 +924,15 @@ public void testTaskStatusMetricsStatuses() throws Exception { EasyMock.expect(executorService.submit(workerTask)).andReturn(null); EasyMock.expect(plugins.delegatingLoader()).andReturn(delegatingLoader); - EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestConnector.class.getName())) + EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestSourceConnector.class.getName())) .andReturn(pluginLoader); EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)).andReturn(delegatingLoader) .times(2); EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader) .times(2); - plugins.connectorClass(WorkerTestConnector.class.getName()); - EasyMock.expectLastCall().andReturn(WorkerTestConnector.class); + plugins.connectorClass(WorkerTestSourceConnector.class.getName()); + EasyMock.expectLastCall().andReturn(WorkerTestSourceConnector.class); EasyMock.expect(workerTask.awaitStop(EasyMock.anyLong())).andStubReturn(true); EasyMock.expectLastCall(); @@ -800,7 +984,7 @@ public void testTaskStatusMetricsStatuses() throws Exception { assertStatistics(worker, 0, 0); assertStartupStatistics(worker, 0, 0, 0, 0); assertEquals(Collections.emptySet(), worker.taskIds()); - worker.startTask( + worker.startSourceTask( TASK_ID, ClusterConfigState.EMPTY, anyConnectorConfigMap(), @@ -874,7 +1058,7 @@ public void testStartTaskFailure() { EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader); EasyMock.expect(plugins.delegatingLoader()).andReturn(delegatingLoader); - EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestConnector.class.getName())) + EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestSourceConnector.class.getName())) .andReturn(pluginLoader); // We would normally expect this since the plugin loader would have been swapped in. However, since we mock out @@ -902,7 +1086,7 @@ public void testStartTaskFailure() { assertStatistics(worker, 0, 0); assertStartupStatistics(worker, 0, 0, 0, 0); - assertFalse(worker.startTask(TASK_ID, ClusterConfigState.EMPTY, anyConnectorConfigMap(), origProps, taskStatusListener, TargetState.STARTED)); + assertFalse(worker.startSourceTask(TASK_ID, ClusterConfigState.EMPTY, anyConnectorConfigMap(), origProps, taskStatusListener, TargetState.STARTED)); assertStartupStatistics(worker, 0, 0, 1, 1); assertStatistics(worker, 0, 0); @@ -921,7 +1105,7 @@ public void testCleanupTasksOnStop() throws Exception { EasyMock.expect(workerTask.id()).andStubReturn(TASK_ID); EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader).times(2); - expectNewWorkerTask(); + expectNewWorkerSourceTask(); Map origProps = new HashMap<>(); origProps.put(TaskConfig.TASK_CLASS_CONFIG, TestSourceTask.class.getName()); @@ -949,7 +1133,7 @@ public void testCleanupTasksOnStop() throws Exception { EasyMock.expect(executorService.submit(workerTask)).andReturn(null); EasyMock.expect(plugins.delegatingLoader()).andReturn(delegatingLoader); - EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestConnector.class.getName())) + EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestSourceConnector.class.getName())) .andReturn(pluginLoader); EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)).andReturn(delegatingLoader) @@ -959,8 +1143,8 @@ public void testCleanupTasksOnStop() throws Exception { EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader) .times(2); - plugins.connectorClass(WorkerTestConnector.class.getName()); - EasyMock.expectLastCall().andReturn(WorkerTestConnector.class); + plugins.connectorClass(WorkerTestSourceConnector.class.getName()); + EasyMock.expectLastCall().andReturn(WorkerTestSourceConnector.class); // Remove on Worker.stop() workerTask.stop(); EasyMock.expectLastCall(); @@ -982,7 +1166,7 @@ public void testCleanupTasksOnStop() throws Exception { worker.herder = herder; worker.start(); assertStatistics(worker, 0, 0); - worker.startTask(TASK_ID, ClusterConfigState.EMPTY, anyConnectorConfigMap(), origProps, taskStatusListener, TargetState.STARTED); + worker.startSourceTask(TASK_ID, ClusterConfigState.EMPTY, anyConnectorConfigMap(), origProps, taskStatusListener, TargetState.STARTED); assertStatistics(worker, 0, 1); worker.stop(); assertStatistics(worker, 0, 0); @@ -999,7 +1183,7 @@ public void testConverterOverrides() throws Exception { EasyMock.expect(workerTask.id()).andStubReturn(TASK_ID); EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader).times(2); - expectNewWorkerTask(); + expectNewWorkerSourceTask(); Map origProps = new HashMap<>(); origProps.put(TaskConfig.TASK_CLASS_CONFIG, TestSourceTask.class.getName()); @@ -1027,7 +1211,7 @@ public void testConverterOverrides() throws Exception { EasyMock.expect(executorService.submit(workerTask)).andReturn(null); EasyMock.expect(plugins.delegatingLoader()).andReturn(delegatingLoader); - EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestConnector.class.getName())) + EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestSourceConnector.class.getName())) .andReturn(pluginLoader); EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)).andReturn(delegatingLoader) @@ -1037,8 +1221,8 @@ public void testConverterOverrides() throws Exception { EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader) .times(2); - plugins.connectorClass(WorkerTestConnector.class.getName()); - EasyMock.expectLastCall().andReturn(WorkerTestConnector.class); + plugins.connectorClass(WorkerTestSourceConnector.class.getName()); + EasyMock.expectLastCall().andReturn(WorkerTestSourceConnector.class); // Remove workerTask.stop(); @@ -1065,7 +1249,7 @@ public void testConverterOverrides() throws Exception { connProps.put("key.converter.extra.config", "foo"); connProps.put(ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG, TestConfigurableConverter.class.getName()); connProps.put("value.converter.extra.config", "bar"); - worker.startTask(TASK_ID, ClusterConfigState.EMPTY, connProps, origProps, taskStatusListener, TargetState.STARTED); + worker.startSourceTask(TASK_ID, ClusterConfigState.EMPTY, connProps, origProps, taskStatusListener, TargetState.STARTED); assertStatistics(worker, 0, 1); assertEquals(new HashSet<>(Arrays.asList(TASK_ID)), worker.taskIds()); worker.stopAndAwaitTask(TASK_ID); @@ -1082,14 +1266,14 @@ public void testConverterOverrides() throws Exception { @Test public void testProducerConfigsWithoutOverrides() { - EasyMock.expect(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX)).andReturn( + EasyMock.expect(connectorConfig.originalsWithPrefix(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX)).andReturn( new HashMap<>()); PowerMock.replayAll(); Map expectedConfigs = new HashMap<>(defaultProducerConfigs); expectedConfigs.put("client.id", "connector-producer-job-0"); expectedConfigs.put("metrics.context.connect.kafka.cluster.id", CLUSTER_ID); assertEquals(expectedConfigs, - Worker.producerConfigs(TASK_ID, "connector-producer-" + TASK_ID, config, connectorConfig, null, noneConnectorClientConfigOverridePolicy, CLUSTER_ID)); + Worker.baseProducerConfigs(CONNECTOR_ID, "connector-producer-" + TASK_ID, config, connectorConfig, null, noneConnectorClientConfigOverridePolicy, CLUSTER_ID)); } @Test @@ -1106,11 +1290,11 @@ public void testProducerConfigsWithOverrides() { expectedConfigs.put("client.id", "producer-test-id"); expectedConfigs.put("metrics.context.connect.kafka.cluster.id", CLUSTER_ID); - EasyMock.expect(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX)).andReturn( + EasyMock.expect(connectorConfig.originalsWithPrefix(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX)).andReturn( new HashMap<>()); PowerMock.replayAll(); assertEquals(expectedConfigs, - Worker.producerConfigs(TASK_ID, "connector-producer-" + TASK_ID, configWithOverrides, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID)); + Worker.baseProducerConfigs(CONNECTOR_ID, "connector-producer-" + TASK_ID, configWithOverrides, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID)); } @Test @@ -1131,11 +1315,11 @@ public void testProducerConfigsWithClientOverrides() { Map connConfig = new HashMap<>(); connConfig.put("linger.ms", "5000"); connConfig.put("batch.size", "1000"); - EasyMock.expect(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX)) + EasyMock.expect(connectorConfig.originalsWithPrefix(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX)) .andReturn(connConfig); PowerMock.replayAll(); assertEquals(expectedConfigs, - Worker.producerConfigs(TASK_ID, "connector-producer-" + TASK_ID, configWithOverrides, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID)); + Worker.baseProducerConfigs(CONNECTOR_ID, "connector-producer-" + TASK_ID, configWithOverrides, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID)); } @Test @@ -1147,8 +1331,9 @@ public void testConsumerConfigsWithoutOverrides() { EasyMock.expect(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX)).andReturn(new HashMap<>()); PowerMock.replayAll(); - assertEquals(expectedConfigs, Worker.consumerConfigs(new ConnectorTaskId("test", 1), config, connectorConfig, - null, noneConnectorClientConfigOverridePolicy, CLUSTER_ID)); + ConnectorTaskId id = new ConnectorTaskId("test", 1); + assertEquals(expectedConfigs, Worker.baseConsumerConfigs("test", "connector-consumer-" + id, config, connectorConfig, + null, noneConnectorClientConfigOverridePolicy, CLUSTER_ID, ConnectorType.SINK)); } @Test @@ -1168,8 +1353,9 @@ public void testConsumerConfigsWithOverrides() { EasyMock.expect(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX)).andReturn(new HashMap<>()); PowerMock.replayAll(); - assertEquals(expectedConfigs, Worker.consumerConfigs(new ConnectorTaskId("test", 1), configWithOverrides, connectorConfig, - null, noneConnectorClientConfigOverridePolicy, CLUSTER_ID)); + ConnectorTaskId id = new ConnectorTaskId("test", 1); + assertEquals(expectedConfigs, Worker.baseConsumerConfigs(id.connector(), "connector-consumer-" + id, configWithOverrides, connectorConfig, + null, noneConnectorClientConfigOverridePolicy, CLUSTER_ID, ConnectorType.SINK)); } @@ -1194,8 +1380,9 @@ public void testConsumerConfigsWithClientOverrides() { EasyMock.expect(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX)) .andReturn(connConfig); PowerMock.replayAll(); - assertEquals(expectedConfigs, Worker.consumerConfigs(new ConnectorTaskId("test", 1), configWithOverrides, connectorConfig, - null, allConnectorClientConfigOverridePolicy, CLUSTER_ID)); + ConnectorTaskId id = new ConnectorTaskId("test", 1); + assertEquals(expectedConfigs, Worker.baseConsumerConfigs(id.connector(), "connector-consumer-" + id, configWithOverrides, connectorConfig, + null, allConnectorClientConfigOverridePolicy, CLUSTER_ID, ConnectorType.SINK)); } @Test @@ -1211,8 +1398,9 @@ public void testConsumerConfigsClientOverridesWithNonePolicy() { EasyMock.expect(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX)) .andReturn(connConfig); PowerMock.replayAll(); - assertThrows(ConnectException.class, () -> Worker.consumerConfigs(new ConnectorTaskId("test", 1), - configWithOverrides, connectorConfig, null, noneConnectorClientConfigOverridePolicy, CLUSTER_ID)); + ConnectorTaskId id = new ConnectorTaskId("test", 1); + assertThrows(ConnectException.class, () -> Worker.baseConsumerConfigs(id.connector(), "connector-consumer-" + id, + configWithOverrides, connectorConfig, null, noneConnectorClientConfigOverridePolicy, CLUSTER_ID, ConnectorType.SINK)); } @Test @@ -1238,8 +1426,8 @@ public void testAdminConfigsClientOverridesWithAllPolicy() { EasyMock.expect(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX)) .andReturn(connConfig); PowerMock.replayAll(); - assertEquals(expectedConfigs, Worker.adminConfigs(new ConnectorTaskId("test", 1), "", configWithOverrides, connectorConfig, - null, allConnectorClientConfigOverridePolicy, CLUSTER_ID)); + assertEquals(expectedConfigs, Worker.adminConfigs("test", "", configWithOverrides, connectorConfig, + null, allConnectorClientConfigOverridePolicy, CLUSTER_ID, ConnectorType.SINK)); } @Test @@ -1255,9 +1443,484 @@ public void testAdminConfigsClientOverridesWithNonePolicy() { EasyMock.expect(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX)) .andReturn(connConfig); PowerMock.replayAll(); - assertThrows(ConnectException.class, () -> Worker.adminConfigs(new ConnectorTaskId("test", 1), - "", configWithOverrides, connectorConfig, null, noneConnectorClientConfigOverridePolicy, CLUSTER_ID)); + assertThrows(ConnectException.class, () -> Worker.adminConfigs("test", + "", configWithOverrides, connectorConfig, null, noneConnectorClientConfigOverridePolicy, CLUSTER_ID, ConnectorType.SINK)); + } + + @Test + public void testRegularSourceOffsetsConsumerConfigs() { + final Map connectorConsumerOverrides = new HashMap<>(); + EasyMock.expect(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX)) + .andReturn(connectorConsumerOverrides) + .anyTimes(); + PowerMock.replayAll(); + + Map workerProps = new HashMap<>(this.workerProps); + workerProps.put("exactly.once.source.support", "enabled"); + workerProps.put("bootstrap.servers", "localhost:4761"); + workerProps.put("group.id", "connect-cluster"); + workerProps.put("config.storage.topic", "connect-configs"); + workerProps.put("offset.storage.topic", "connect-offsets"); + workerProps.put("status.storage.topic", "connect-statuses"); + config = new DistributedConfig(workerProps); + + Map consumerConfigs = Worker.regularSourceOffsetsConsumerConfigs( + "test", "", config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID); + assertEquals("localhost:4761", consumerConfigs.get(BOOTSTRAP_SERVERS_CONFIG)); + assertEquals("read_committed", consumerConfigs.get(ISOLATION_LEVEL_CONFIG)); + + workerProps.put("consumer." + BOOTSTRAP_SERVERS_CONFIG, "localhost:9021"); + workerProps.put("consumer." + ISOLATION_LEVEL_CONFIG, "read_uncommitted"); + config = new DistributedConfig(workerProps); + consumerConfigs = Worker.regularSourceOffsetsConsumerConfigs( + "test", "", config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID); + assertEquals("localhost:9021", consumerConfigs.get(BOOTSTRAP_SERVERS_CONFIG)); + // User is allowed to override the isolation level for regular (non-exactly-once) source connectors and their tasks + assertEquals("read_uncommitted", consumerConfigs.get(ISOLATION_LEVEL_CONFIG)); + + workerProps.remove("consumer." + ISOLATION_LEVEL_CONFIG); + connectorConsumerOverrides.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:489"); + connectorConsumerOverrides.put(ISOLATION_LEVEL_CONFIG, "read_uncommitted"); + config = new DistributedConfig(workerProps); + consumerConfigs = Worker.regularSourceOffsetsConsumerConfigs( + "test", "", config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID); + assertEquals("localhost:489", consumerConfigs.get(BOOTSTRAP_SERVERS_CONFIG)); + // User is allowed to override the isolation level for regular (non-exactly-once) source connectors and their tasks + assertEquals("read_uncommitted", consumerConfigs.get(ISOLATION_LEVEL_CONFIG)); + } + + @Test + public void testExactlyOnceSourceOffsetsConsumerConfigs() { + final Map connectorConsumerOverrides = new HashMap<>(); + EasyMock.expect(connectorConfig.originalsWithPrefix(ConnectorConfig.CONNECTOR_CLIENT_CONSUMER_OVERRIDES_PREFIX)) + .andReturn(connectorConsumerOverrides) + .anyTimes(); + PowerMock.replayAll(); + + Map workerProps = new HashMap<>(this.workerProps); + workerProps.put("exactly.once.source.support", "enabled"); + workerProps.put("bootstrap.servers", "localhost:4761"); + workerProps.put("group.id", "connect-cluster"); + workerProps.put("config.storage.topic", "connect-configs"); + workerProps.put("offset.storage.topic", "connect-offsets"); + workerProps.put("status.storage.topic", "connect-statuses"); + config = new DistributedConfig(workerProps); + + Map consumerConfigs = Worker.exactlyOnceSourceOffsetsConsumerConfigs( + "test", "", config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID); + assertEquals("localhost:4761", consumerConfigs.get(BOOTSTRAP_SERVERS_CONFIG)); + assertEquals("read_committed", consumerConfigs.get(ISOLATION_LEVEL_CONFIG)); + + workerProps.put("consumer." + BOOTSTRAP_SERVERS_CONFIG, "localhost:9021"); + workerProps.put("consumer." + ISOLATION_LEVEL_CONFIG, "read_uncommitted"); + config = new DistributedConfig(workerProps); + consumerConfigs = Worker.exactlyOnceSourceOffsetsConsumerConfigs( + "test", "", config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID); + assertEquals("localhost:9021", consumerConfigs.get(BOOTSTRAP_SERVERS_CONFIG)); + // User is not allowed to override isolation level when exactly-once support is enabled + assertEquals("read_committed", consumerConfigs.get(ISOLATION_LEVEL_CONFIG)); + + workerProps.remove("consumer." + ISOLATION_LEVEL_CONFIG); + connectorConsumerOverrides.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:489"); + connectorConsumerOverrides.put(ISOLATION_LEVEL_CONFIG, "read_uncommitted"); + config = new DistributedConfig(workerProps); + consumerConfigs = Worker.exactlyOnceSourceOffsetsConsumerConfigs( + "test", "", config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID); + assertEquals("localhost:489", consumerConfigs.get(BOOTSTRAP_SERVERS_CONFIG)); + // User is not allowed to override isolation level when exactly-once support is enabled + assertEquals("read_committed", consumerConfigs.get(ISOLATION_LEVEL_CONFIG)); + } + + @Test + public void testExactlyOnceSourceTaskProducerConfigs() { + final Map connectorProducerOverrides = new HashMap<>(); + EasyMock.expect(connectorConfig.originalsWithPrefix(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX)) + .andReturn(connectorProducerOverrides) + .anyTimes(); + PowerMock.replayAll(); + + final String groupId = "connect-cluster"; + final String transactionalId = Worker.transactionalId(groupId, TASK_ID.connector(), TASK_ID.task()); + + Map workerProps = new HashMap<>(this.workerProps); + workerProps.put("exactly.once.source.support", "enabled"); + workerProps.put("bootstrap.servers", "localhost:4761"); + workerProps.put("group.id", groupId); + workerProps.put("config.storage.topic", "connect-configs"); + workerProps.put("offset.storage.topic", "connect-offsets"); + workerProps.put("status.storage.topic", "connect-statuses"); + config = new DistributedConfig(workerProps); + + Map producerConfigs = Worker.exactlyOnceSourceTaskProducerConfigs( + TASK_ID, config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID); + assertEquals("localhost:4761", producerConfigs.get(BOOTSTRAP_SERVERS_CONFIG)); + assertEquals("true", producerConfigs.get(ENABLE_IDEMPOTENCE_CONFIG)); + assertEquals(transactionalId, producerConfigs.get(TRANSACTIONAL_ID_CONFIG)); + + workerProps.put("producer." + BOOTSTRAP_SERVERS_CONFIG, "localhost:9021"); + workerProps.put("producer." + ENABLE_IDEMPOTENCE_CONFIG, "false"); + workerProps.put("producer." + TRANSACTIONAL_ID_CONFIG, "some-other-transactional-id"); + config = new DistributedConfig(workerProps); + producerConfigs = Worker.exactlyOnceSourceTaskProducerConfigs( + TASK_ID, config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID); + assertEquals("localhost:9021", producerConfigs.get(BOOTSTRAP_SERVERS_CONFIG)); + // User is not allowed to override idempotence or transactional ID for exactly-once source tasks + assertEquals("true", producerConfigs.get(ENABLE_IDEMPOTENCE_CONFIG)); + assertEquals(transactionalId, producerConfigs.get(TRANSACTIONAL_ID_CONFIG)); + + workerProps.remove("producer." + ENABLE_IDEMPOTENCE_CONFIG); + workerProps.remove("producer." + TRANSACTIONAL_ID_CONFIG); + connectorProducerOverrides.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:489"); + connectorProducerOverrides.put(ENABLE_IDEMPOTENCE_CONFIG, "false"); + connectorProducerOverrides.put(TRANSACTIONAL_ID_CONFIG, "yet-another-transactional-id"); + config = new DistributedConfig(workerProps); + producerConfigs = Worker.exactlyOnceSourceTaskProducerConfigs( + TASK_ID, config, connectorConfig, null, allConnectorClientConfigOverridePolicy, CLUSTER_ID); + assertEquals("localhost:489", producerConfigs.get(BOOTSTRAP_SERVERS_CONFIG)); + // User is not allowed to override idempotence or transactional ID for exactly-once source tasks + assertEquals("true", producerConfigs.get(ENABLE_IDEMPOTENCE_CONFIG)); + assertEquals(transactionalId, producerConfigs.get(TRANSACTIONAL_ID_CONFIG)); + } + + @Test + public void testOffsetStoreForRegularSourceConnector() { + expectClusterId(); + expectConverters(); + expectFileConfigProvider(); + expectStartStorage(); + expectStopStorage(); + PowerMock.replayAll(); + + final String workerOffsetsTopic = "worker-offsets"; + final String workerBootstrapServers = "localhost:4761"; + Map workerProps = new HashMap<>(this.workerProps); + workerProps.put("exactly.once.source.support", "disabled"); + workerProps.put("bootstrap.servers", workerBootstrapServers); + workerProps.put("group.id", "connect-cluster"); + workerProps.put("config.storage.topic", "connect-configs"); + workerProps.put("offset.storage.topic", workerOffsetsTopic); + workerProps.put("status.storage.topic", "connect-statuses"); + config = new DistributedConfig(workerProps); + + worker = new Worker(WORKER_ID, new MockTime(), plugins, config, offsetBackingStore, allConnectorClientConfigOverridePolicy); + worker.start(); + + Map connectorProps = new HashMap<>(); + connectorProps.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID); + connectorProps.put(CONNECTOR_CLASS_CONFIG, WorkerTestSourceConnector.class.getName()); + connectorProps.put(ConnectorConfig.TASKS_MAX_CONFIG, "1"); + SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With no connector-specific offsets topic in the config, we should only use the worker-global offsets store + ConnectorOffsetBackingStore connectorStore = worker.offsetStoreForRegularSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertFalse(connectorStore.hasConnectorSpecificStore()); + + connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, "connector-offsets-topic"); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With a connector-specific offsets topic in the config (whose name differs from the worker's offsets topic), we should use both a + // connector-specific store and the worker-global store + connectorStore = worker.offsetStoreForRegularSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, workerOffsetsTopic); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and no overridden bootstrap.servers + // for the connector, we should only use a connector-specific offsets store + connectorStore = worker.offsetStoreForRegularSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector); + assertFalse(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + connectorProps.put(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, workerBootstrapServers); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers + // for the connector that exactly matches the worker's, we should only use a connector-specific offsets store + connectorStore = worker.offsetStoreForRegularSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector); + assertFalse(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + connectorProps.put(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, "localhost:1111"); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers + // for the connector that doesn't match the worker's, we should use both a connector-specific store and the worker-global store + connectorStore = worker.offsetStoreForRegularSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + connectorProps.remove(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With no connector-specific offsets topic in the config, even with an overridden bootstrap.servers + // for the connector that doesn't match the worker's, we should still only use the worker-global offsets store + connectorStore = worker.offsetStoreForRegularSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertFalse(connectorStore.hasConnectorSpecificStore()); + + worker.stop(); + + PowerMock.verifyAll(); + } + + @Test + public void testOffsetStoreForExactlyOnceSourceConnector() { + expectClusterId(); + expectConverters(); + expectFileConfigProvider(); + expectStartStorage(); + expectStopStorage(); + PowerMock.replayAll(); + + final String workerOffsetsTopic = "worker-offsets"; + final String workerBootstrapServers = "localhost:4761"; + Map workerProps = new HashMap<>(this.workerProps); + workerProps.put("exactly.once.source.support", "enabled"); + workerProps.put("bootstrap.servers", workerBootstrapServers); + workerProps.put("group.id", "connect-cluster"); + workerProps.put("config.storage.topic", "connect-configs"); + workerProps.put("offset.storage.topic", workerOffsetsTopic); + workerProps.put("status.storage.topic", "connect-statuses"); + config = new DistributedConfig(workerProps); + + worker = new Worker(WORKER_ID, new MockTime(), plugins, config, offsetBackingStore, allConnectorClientConfigOverridePolicy); + worker.start(); + + Map connectorProps = new HashMap<>(); + connectorProps.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID); + connectorProps.put(CONNECTOR_CLASS_CONFIG, WorkerTestSourceConnector.class.getName()); + connectorProps.put(ConnectorConfig.TASKS_MAX_CONFIG, "1"); + SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With no connector-specific offsets topic in the config, we should only use a connector-specific offsets store + ConnectorOffsetBackingStore connectorStore = worker.offsetStoreForExactlyOnceSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector); + assertFalse(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, "connector-offsets-topic"); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With a connector-specific offsets topic in the config (whose name differs from the worker's offsets topic), we should use both a + // connector-specific store and the worker-global store + connectorStore = worker.offsetStoreForExactlyOnceSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, workerOffsetsTopic); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and no overridden bootstrap.servers + // for the connector, we should only use a connector-specific store + connectorStore = worker.offsetStoreForExactlyOnceSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector); + assertFalse(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + connectorProps.put(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, workerBootstrapServers); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers + // for the connector that exactly matches the worker's, we should only use a connector-specific store + connectorStore = worker.offsetStoreForExactlyOnceSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector); + assertFalse(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + connectorProps.put(CONNECTOR_CLIENT_PRODUCER_OVERRIDES_PREFIX + BOOTSTRAP_SERVERS_CONFIG, "localhost:1111"); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers + // for the connector that doesn't match the worker's, we should use both a connector-specific store and the worker-global store + connectorStore = worker.offsetStoreForExactlyOnceSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + connectorProps.remove(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With no connector-specific offsets topic in the config and an overridden bootstrap.servers + // for the connector that doesn't match the worker's, we should use both a connector-specific store and the worker-global store + connectorStore = worker.offsetStoreForExactlyOnceSourceConnector(sourceConfig, CONNECTOR_ID, sourceConnector); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + worker.stop(); + + PowerMock.verifyAll(); + } + + @Test + public void testOffsetStoreForRegularSourceTask() { + expectClusterId(); + expectConverters(); + expectFileConfigProvider(); + expectStartStorage(); + expectStopStorage(); + + Map producerProps = new HashMap<>(); + Producer producer = EasyMock.mock(Producer.class); + TopicAdmin topicAdmin = EasyMock.mock(TopicAdmin.class); + final AtomicBoolean topicAdminCreated = new AtomicBoolean(false); + final Supplier topicAdminSupplier = () -> { + topicAdminCreated.set(true); + return topicAdmin; + }; + + PowerMock.replayAll(producer, topicAdmin); + + final String workerOffsetsTopic = "worker-offsets"; + final String workerBootstrapServers = "localhost:4761"; + Map workerProps = new HashMap<>(this.workerProps); + workerProps.put("exactly.once.source.support", "disabled"); + workerProps.put("bootstrap.servers", workerBootstrapServers); + workerProps.put("group.id", "connect-cluster"); + workerProps.put("config.storage.topic", "connect-configs"); + workerProps.put("offset.storage.topic", workerOffsetsTopic); + workerProps.put("status.storage.topic", "connect-statuses"); + config = new DistributedConfig(workerProps); + + worker = new Worker(WORKER_ID, new MockTime(), plugins, config, offsetBackingStore, allConnectorClientConfigOverridePolicy); + worker.start(); + + Map connectorProps = new HashMap<>(); + connectorProps.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID); + connectorProps.put(CONNECTOR_CLASS_CONFIG, WorkerTestSourceConnector.class.getName()); + connectorProps.put(ConnectorConfig.TASKS_MAX_CONFIG, "1"); + SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + producerProps.put(BOOTSTRAP_SERVERS_CONFIG, workerBootstrapServers); + topicAdminCreated.set(false); + // With no connector-specific offsets topic in the config, we should only use the worker-global store + ConnectorOffsetBackingStore connectorStore = worker.offsetStoreForRegularSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdminSupplier); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertFalse(connectorStore.hasConnectorSpecificStore()); + assertFalse(topicAdminCreated.get()); + + connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, "connector-offsets-topic"); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + topicAdminCreated.set(false); + // With a connector-specific offsets topic in the config (whose name differs from the worker's offsets topic), we should use both a + // connector-specific store and the worker-global store + connectorStore = worker.offsetStoreForRegularSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdminSupplier); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + assertTrue(topicAdminCreated.get()); + + connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, workerOffsetsTopic); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + topicAdminCreated.set(false); + // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and no overridden bootstrap.servers + // for the connector, we should only use a connector-specific store + connectorStore = worker.offsetStoreForRegularSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdminSupplier); + assertFalse(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + assertTrue(topicAdminCreated.get()); + + producerProps.put(BOOTSTRAP_SERVERS_CONFIG, workerBootstrapServers); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + topicAdminCreated.set(false); + // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers + // for the connector that exactly matches the worker's, we should only use a connector-specific store + connectorStore = worker.offsetStoreForRegularSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdminSupplier); + assertFalse(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + assertTrue(topicAdminCreated.get()); + + producerProps.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:1111"); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + topicAdminCreated.set(false); + // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers + // for the connector that doesn't match the worker's, we should use both a connector-specific store and the worker-global store + connectorStore = worker.offsetStoreForRegularSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdminSupplier); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + assertTrue(topicAdminCreated.get()); + + connectorProps.remove(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + topicAdminCreated.set(false); + // With no connector-specific offsets topic in the config and an overridden bootstrap.servers + // for the connector that doesn't match the worker's, we should still only use the worker-global store + connectorStore = worker.offsetStoreForRegularSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdminSupplier); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertFalse(connectorStore.hasConnectorSpecificStore()); + assertFalse(topicAdminCreated.get()); + + worker.stop(); + + PowerMock.verifyAll(); + } + + @Test + public void testOffsetStoreForExactlyOnceSourceTask() { + expectClusterId(); + expectConverters(); + expectFileConfigProvider(); + expectStartStorage(); + expectStopStorage(); + + Map producerProps = new HashMap<>(); + Producer producer = EasyMock.mock(Producer.class); + TopicAdmin topicAdmin = EasyMock.mock(TopicAdmin.class); + + PowerMock.replayAll(producer, topicAdmin); + + final String workerOffsetsTopic = "worker-offsets"; + final String workerBootstrapServers = "localhost:4761"; + Map workerProps = new HashMap<>(this.workerProps); + workerProps.put("exactly.once.source.support", "enabled"); + workerProps.put("bootstrap.servers", workerBootstrapServers); + workerProps.put("group.id", "connect-cluster"); + workerProps.put("config.storage.topic", "connect-configs"); + workerProps.put("offset.storage.topic", workerOffsetsTopic); + workerProps.put("status.storage.topic", "connect-statuses"); + config = new DistributedConfig(workerProps); + + worker = new Worker(WORKER_ID, new MockTime(), plugins, config, offsetBackingStore, allConnectorClientConfigOverridePolicy); + worker.start(); + + Map connectorProps = new HashMap<>(); + connectorProps.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID); + connectorProps.put(CONNECTOR_CLASS_CONFIG, WorkerTestSourceConnector.class.getName()); + connectorProps.put(ConnectorConfig.TASKS_MAX_CONFIG, "1"); + SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + producerProps.put(BOOTSTRAP_SERVERS_CONFIG, workerBootstrapServers); + // With no connector-specific offsets topic in the config, we should only use a connector-specific offsets store + ConnectorOffsetBackingStore connectorStore = worker.offsetStoreForExactlyOnceSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdmin); + assertFalse(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, "connector-offsets-topic"); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With a connector-specific offsets topic in the config (whose name differs from the worker's offsets topic), we should use both a + // connector-specific store and the worker-global store + connectorStore = worker.offsetStoreForExactlyOnceSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdmin); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + connectorProps.put(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG, workerOffsetsTopic); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and no overridden bootstrap.servers + // for the connector, we should only use a connector-specific store + connectorStore = worker.offsetStoreForExactlyOnceSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdmin); + assertFalse(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + producerProps.put(BOOTSTRAP_SERVERS_CONFIG, workerBootstrapServers); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers + // for the connector that exactly matches the worker's, we should only use a connector-specific store + connectorStore = worker.offsetStoreForExactlyOnceSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdmin); + assertFalse(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + producerProps.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:1111"); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With a connector-specific offsets topic in the config whose name matches the worker's offsets topic, and an overridden bootstrap.servers + // for the connector that doesn't match the worker's, we should use both a connector-specific store and the worker-global store + connectorStore = worker.offsetStoreForExactlyOnceSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdmin); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + connectorProps.remove(SourceConnectorConfig.OFFSETS_TOPIC_CONFIG); + sourceConfig = new SourceConnectorConfig(plugins, connectorProps, enableTopicCreation); + // With no connector-specific offsets topic in the config and an overridden bootstrap.servers + // for the connector that doesn't match the worker's, we should use both a connector-specific store and the worker-global store + connectorStore = worker.offsetStoreForExactlyOnceSourceTask(TASK_ID, sourceConfig, sourceConnector.getClass(), producer, producerProps, topicAdmin); + assertTrue(connectorStore.hasWorkerGlobalStore()); + assertTrue(connectorStore.hasConnectorSpecificStore()); + + worker.stop(); + PowerMock.verifyAll(); } @Test @@ -1268,7 +1931,7 @@ public void testWorkerMetrics() throws Exception { // Create EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader).times(2); - EasyMock.expect(plugins.newConnector(WorkerTestConnector.class.getName())) + EasyMock.expect(plugins.newConnector(WorkerTestSourceConnector.class.getName())) .andReturn(sourceConnector); EasyMock.expect(sourceConnector.version()).andReturn("1.0"); @@ -1276,7 +1939,7 @@ public void testWorkerMetrics() throws Exception { props.put(SinkConnectorConfig.TOPICS_CONFIG, "foo,bar"); props.put(ConnectorConfig.TASKS_MAX_CONFIG, "1"); props.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID); - props.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, WorkerTestConnector.class.getName()); + props.put(CONNECTOR_CLASS_CONFIG, WorkerTestSourceConnector.class.getName()); EasyMock.expect(sourceConnector.version()).andReturn("1.0"); @@ -1330,6 +1993,60 @@ public void testWorkerMetrics() throws Exception { assertNotNull(server.getObjectInstance(new ObjectName("kafka.connect:type=grp1"))); } + @Test + public void testZombieFencing() { + KafkaFuture expectedZombieFenceFuture = EasyMock.mock(KafkaFuture.class); + KafkaFuture fenceProducersFuture = EasyMock.mock(KafkaFuture.class); + EasyMock.expect(fenceProducersFuture.whenComplete(anyObject())).andReturn(expectedZombieFenceFuture); + FenceProducersResult fenceProducersResult = EasyMock.mock(FenceProducersResult.class); + EasyMock.expect(fenceProducersResult.all()).andReturn(fenceProducersFuture); + Admin admin = EasyMock.mock(Admin.class); + EasyMock.expect(admin.fenceProducers(anyObject(), anyObject())).andReturn(fenceProducersResult); + + EasyMock.expect(plugins.currentThreadLoader()).andReturn(delegatingLoader); + EasyMock.expect(plugins.delegatingLoader()).andReturn(delegatingLoader); + EasyMock.expect(delegatingLoader.connectorLoader(WorkerTestSourceConnector.class.getName())) + .andReturn(pluginLoader); + EasyMock.expect(Plugins.compareAndSwapLoaders(pluginLoader)).andReturn(delegatingLoader); + plugins.connectorClass(WorkerTestSourceConnector.class.getName()); + EasyMock.expectLastCall().andReturn(WorkerTestSourceConnector.class); + EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader); + + expectConverters(); + expectStartStorage(); + expectFileConfigProvider(); + + expectClusterId(); + + PowerMock.replayAll(admin, fenceProducersResult, fenceProducersFuture, expectedZombieFenceFuture); + + worker = new Worker(WORKER_ID, new MockTime(), plugins, config, offsetBackingStore, executorService, + allConnectorClientConfigOverridePolicy); + worker.herder = herder; + worker.start(); + + Map connectorConfig = anyConnectorConfigMap(); + connectorConfig.put(CONNECTOR_CLIENT_ADMIN_OVERRIDES_PREFIX + RETRY_BACKOFF_MS_CONFIG, "4761"); + + AtomicReference> adminConfig = new AtomicReference<>(); + Function, Admin> mockAdminConstructor = actualAdminConfig -> { + adminConfig.set(actualAdminConfig); + return admin; + }; + + KafkaFuture actualZombieFenceFuture = + worker.fenceZombies(CONNECTOR_ID, 12, connectorConfig, mockAdminConstructor); + + assertEquals(expectedZombieFenceFuture, actualZombieFenceFuture); + assertNotNull(adminConfig.get()); + assertEquals("Admin should be configured with user-specified overrides", + "4761", + adminConfig.get().get(RETRY_BACKOFF_MS_CONFIG) + ); + + PowerMock.verifyAll(); + } + private void assertStatusMetrics(long expected, String metricName) { MetricGroup statusMetrics = worker.connectorStatusMetricsGroup().metricGroup(TASK_ID.connector()); if (expected == 0L) { @@ -1462,7 +2179,7 @@ private void expectTaskHeaderConverter(ClassLoaderUsage classLoaderUsage, Header private Map anyConnectorConfigMap() { Map props = new HashMap<>(); props.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_ID); - props.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, WorkerTestConnector.class.getName()); + props.put(CONNECTOR_CLASS_CONFIG, WorkerTestSourceConnector.class.getName()); props.put(ConnectorConfig.TASKS_MAX_CONFIG, "1"); props.put(DEFAULT_TOPIC_CREATION_PREFIX + REPLICATION_FACTOR_CONFIG, String.valueOf(1)); props.put(DEFAULT_TOPIC_CREATION_PREFIX + PARTITIONS_CONFIG, String.valueOf(1)); @@ -1474,7 +2191,7 @@ private void expectClusterId() { EasyMock.expect(ConnectUtils.lookupKafkaClusterId(EasyMock.anyObject())).andReturn("test-cluster").anyTimes(); } - private void expectNewWorkerTask() throws Exception { + private void expectNewWorkerSourceTask() throws Exception { PowerMock.expectNew( WorkerSourceTask.class, EasyMock.eq(TASK_ID), EasyMock.eq(task), @@ -1489,6 +2206,7 @@ private void expectNewWorkerTask() throws Exception { EasyMock.>anyObject(), anyObject(OffsetStorageReader.class), anyObject(OffsetStorageWriter.class), + anyObject(ConnectorOffsetBackingStore.class), EasyMock.eq(config), anyObject(ClusterConfigState.class), anyObject(ConnectMetrics.class), @@ -1499,8 +2217,61 @@ private void expectNewWorkerTask() throws Exception { anyObject(Executor.class)) .andReturn(workerTask); } + + private void expectNewWorkerSinkTask(WorkerSinkTask workerTask, SinkTask task) throws Exception { + PowerMock.expectNew( + WorkerSinkTask.class, EasyMock.eq(TASK_ID), + EasyMock.eq(task), + anyObject(TaskStatus.Listener.class), + EasyMock.eq(TargetState.STARTED), + EasyMock.eq(config), + anyObject(ClusterConfigState.class), + anyObject(ConnectMetrics.class), + anyObject(JsonConverter.class), + anyObject(JsonConverter.class), + anyObject(JsonConverter.class), + EasyMock.eq(new TransformationChain<>(Collections.emptyList(), NOOP_OPERATOR)), + anyObject(Consumer.class), + EasyMock.eq(pluginLoader), + anyObject(Time.class), + anyObject(RetryWithToleranceOperator.class), + anyObject(WorkerErrantRecordReporter.class), + anyObject(StatusBackingStore.class)) + .andReturn(workerTask); + } + + private void expectNewExactlyOnceWorkerSourceTask(ExactlyOnceWorkerSourceTask workerTask, Runnable preProducer, Runnable postProducer) throws Exception { + PowerMock.expectNew( + ExactlyOnceWorkerSourceTask.class, EasyMock.eq(TASK_ID), + EasyMock.eq(task), + anyObject(TaskStatus.Listener.class), + EasyMock.eq(TargetState.STARTED), + anyObject(JsonConverter.class), + anyObject(JsonConverter.class), + anyObject(JsonConverter.class), + EasyMock.eq(new TransformationChain<>(Collections.emptyList(), NOOP_OPERATOR)), + anyObject(KafkaProducer.class), + anyObject(TopicAdmin.class), + EasyMock.>anyObject(), + anyObject(OffsetStorageReader.class), + anyObject(OffsetStorageWriter.class), + anyObject(ConnectorOffsetBackingStore.class), + EasyMock.eq(config), + anyObject(ClusterConfigState.class), + anyObject(ConnectMetrics.class), + EasyMock.eq(pluginLoader), + anyObject(Time.class), + anyObject(RetryWithToleranceOperator.class), + anyObject(StatusBackingStore.class), + anyObject(SourceConnectorConfig.class), + anyObject(Executor.class), + EasyMock.eq(preProducer), + EasyMock.eq(postProducer)) + .andReturn(workerTask); + } + /* Name here needs to be unique as we are testing the aliasing mechanism */ - public static class WorkerTestConnector extends SourceConnector { + public static class WorkerTestSourceConnector extends SourceConnector { private static final ConfigDef CONFIG_DEF = new ConfigDef() .define("configName", ConfigDef.Type.STRING, ConfigDef.Importance.HIGH, "Test configName."); @@ -1559,6 +2330,60 @@ public void stop() { } } + public static class WorkerTestSinkConnector extends SinkConnector { + @Override + public String version() { + return "1.0"; + } + + @Override + public void start(Map props) { + + } + + @Override + public Class taskClass() { + return TestSinkTask.class; + } + + @Override + public List> taskConfigs(int maxTasks) { + return null; + } + + @Override + public void stop() { + + } + + @Override + public ConfigDef config() { + return new ConfigDef(); + } + } + + private static class TestSinkTask extends SinkTask { + public TestSinkTask() { + } + + @Override + public String version() { + return "1.0"; + } + + @Override + public void start(Map props) { + } + + @Override + public void put(Collection records) { + } + + @Override + public void stop() { + } + } + public static class TestConverter implements Converter { public Map configs; diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTestUtils.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTestUtils.java index ed77018f2883b..084d865cc5eca 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTestUtils.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTestUtils.java @@ -16,7 +16,7 @@ */ package org.apache.kafka.connect.runtime; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.runtime.distributed.ExtendedAssignment; import org.apache.kafka.connect.runtime.distributed.ExtendedWorkerState; import org.apache.kafka.connect.util.ConnectorTaskId; @@ -70,6 +70,9 @@ public static ClusterConfigState clusterConfigState(long offset, connectorConfigs(1, connectorNum), connectorTargetStates(1, connectorNum, TargetState.STARTED), taskConfigs(0, connectorNum, connectorNum * taskNum), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptySet(), Collections.emptySet()); } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTransactionContextTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTransactionContextTest.java new file mode 100644 index 0000000000000..3bc2b2155d1f1 --- /dev/null +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerTransactionContextTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.runtime; + +import org.apache.kafka.connect.source.SourceRecord; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class WorkerTransactionContextTest { + + private static final SourceRecord RECORD = new SourceRecord(null, null, "t", null, 0, null, null); + + private WorkerTransactionContext context = new WorkerTransactionContext(); + + @Test + public void shouldNotifyOfBatchCommit() { + context.commitTransaction(); + assertFalse(context.shouldAbortBatch()); + assertFalse(context.shouldAbortOn(RECORD)); + assertFalse(context.shouldCommitOn(RECORD)); + assertTrue(context.shouldCommitBatch()); + } + + @Test + public void shouldNotifyOfRecordCommit() { + context.commitTransaction(RECORD); + assertFalse(context.shouldAbortBatch()); + assertFalse(context.shouldAbortOn(RECORD)); + assertFalse(context.shouldCommitBatch()); + assertTrue(context.shouldCommitOn(RECORD)); + } + + @Test + public void shouldNotifyOfBatchAbort() { + context.abortTransaction(); + assertFalse(context.shouldAbortOn(RECORD)); + assertFalse(context.shouldCommitOn(RECORD)); + assertFalse(context.shouldCommitBatch()); + assertTrue(context.shouldAbortBatch()); + } + + @Test + public void shouldNotifyOfRecordAbort() { + context.abortTransaction(RECORD); + assertFalse(context.shouldAbortBatch()); + assertFalse(context.shouldCommitOn(RECORD)); + assertFalse(context.shouldCommitBatch()); + assertTrue(context.shouldAbortOn(RECORD)); + } + + @Test + public void shouldNotCommitBatchRepeatedly() { + context.commitTransaction(); + assertTrue(context.shouldCommitBatch()); + assertFalse(context.shouldCommitBatch()); + } + + @Test + public void shouldNotCommitRecordRepeatedly() { + context.commitTransaction(RECORD); + assertTrue(context.shouldCommitOn(RECORD)); + assertFalse(context.shouldCommitOn(RECORD)); + } + + @Test + public void shouldNotAbortBatchRepeatedly() { + context.abortTransaction(); + assertTrue(context.shouldAbortBatch()); + assertFalse(context.shouldAbortBatch()); + } + + @Test + public void shouldNotAbortRecordRepeatedly() { + context.abortTransaction(RECORD); + assertTrue(context.shouldAbortOn(RECORD)); + assertFalse(context.shouldAbortOn(RECORD)); + } + + @Test + public void shouldDisallowConflictingRequests() { + context.commitTransaction(); + context.abortTransaction(); + assertThrows(IllegalStateException.class, context::shouldCommitBatch); + assertThrows(IllegalStateException.class, context::shouldAbortBatch); + + context = new WorkerTransactionContext(); + context.commitTransaction(RECORD); + context.abortTransaction(RECORD); + assertThrows(IllegalStateException.class, () -> context.shouldCommitOn(RECORD)); + assertThrows(IllegalStateException.class, () -> context.shouldAbortOn(RECORD)); + } + +} diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocolCompatibilityTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocolCompatibilityTest.java index a3144c04d3462..15bb13c94406f 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocolCompatibilityTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/ConnectProtocolCompatibilityTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.connect.runtime.distributed; import org.apache.kafka.connect.runtime.TargetState; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.storage.KafkaConfigBackingStore; import org.apache.kafka.connect.util.ConnectorTaskId; import org.junit.After; @@ -67,6 +68,9 @@ public void setup() { Collections.singletonMap(connectorId1, new HashMap<>()), Collections.singletonMap(connectorId1, TargetState.STARTED), Collections.singletonMap(taskId1x0, new HashMap<>()), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptySet(), Collections.emptySet()); } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedConfigTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedConfigTest.java index e95232748b2e1..57c1c066c8625 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedConfigTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedConfigTest.java @@ -27,10 +27,14 @@ import java.util.List; import java.util.Map; +import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG; +import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.GROUP_ID_CONFIG; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; public class DistributedConfigTest { @@ -294,4 +298,42 @@ public void shouldRemoveCompactionFromStatusTopicSettings() { assertEquals(expectedTopicSettings, actual); assertNotEquals(topicSettings, actual); } + + @Test + public void shouldIdentifyNeedForTransactionalLeader() { + Map workerProps = configs(); + + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "disabled"); + assertFalse(new DistributedConfig(workerProps).transactionalLeaderEnabled()); + + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "preparing"); + assertTrue(new DistributedConfig(workerProps).transactionalLeaderEnabled()); + + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "enabled"); + assertTrue(new DistributedConfig(workerProps).transactionalLeaderEnabled()); + } + + @Test + public void shouldConstructExpectedTransactionalId() { + Map workerProps = configs(); + + workerProps.put(GROUP_ID_CONFIG, "why did i stay up all night writing unit tests"); + assertEquals( + "connect-cluster-why did i stay up all night writing unit tests", + new DistributedConfig(workerProps).transactionalProducerId() + ); + + workerProps.put(GROUP_ID_CONFIG, "connect-cluster"); + assertEquals( + "connect-cluster-connect-cluster", + new DistributedConfig(workerProps).transactionalProducerId() + ); + + workerProps.put(GROUP_ID_CONFIG, "\u2603"); + assertEquals( + "connect-cluster-\u2603", + new DistributedConfig(workerProps).transactionalProducerId() + ); + } + } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java index 245bb7544396d..bdd5a8e9499e9 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java @@ -17,9 +17,11 @@ package org.apache.kafka.connect.runtime.distributed; import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.config.ConfigValue; +import org.apache.kafka.common.errors.AuthorizationException; import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.connect.connector.Connector; import org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy; import org.apache.kafka.connect.connector.policy.NoneConnectorClientConfigOverridePolicy; import org.apache.kafka.connect.errors.AlreadyExistsException; @@ -33,6 +35,7 @@ import org.apache.kafka.connect.runtime.RestartRequest; import org.apache.kafka.connect.runtime.SessionKey; import org.apache.kafka.connect.runtime.SinkConnectorConfig; +import org.apache.kafka.connect.runtime.SourceConnectorConfig; import org.apache.kafka.connect.runtime.TargetState; import org.apache.kafka.connect.runtime.TaskConfig; import org.apache.kafka.connect.runtime.TopicStatus; @@ -44,6 +47,7 @@ import org.apache.kafka.connect.runtime.isolation.PluginClassLoader; import org.apache.kafka.connect.runtime.isolation.Plugins; import org.apache.kafka.connect.runtime.rest.InternalRequestSignature; +import org.apache.kafka.connect.runtime.rest.RestClient; import org.apache.kafka.connect.runtime.rest.entities.ConfigInfos; import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo; import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo; @@ -52,14 +56,19 @@ import org.apache.kafka.connect.runtime.rest.errors.BadRequestException; import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException; import org.apache.kafka.connect.sink.SinkConnector; +import org.apache.kafka.connect.source.ConnectorTransactionBoundaries; +import org.apache.kafka.connect.source.ExactlyOnceSupport; import org.apache.kafka.connect.source.SourceConnector; import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.storage.ConfigBackingStore; import org.apache.kafka.connect.storage.StatusBackingStore; import org.apache.kafka.connect.util.Callback; import org.apache.kafka.connect.util.ConnectorTaskId; import org.apache.kafka.connect.util.FutureCallback; +import org.apache.kafka.connect.util.ThreadedTest; import org.easymock.Capture; +import org.easymock.CaptureType; import org.easymock.EasyMock; import org.junit.After; import org.junit.Before; @@ -82,33 +91,42 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static java.util.Collections.singletonList; import static javax.ws.rs.core.Response.Status.FORBIDDEN; +import static org.apache.kafka.connect.runtime.SourceConnectorConfig.ExactlyOnceSupportLevel.REQUIRED; import static org.apache.kafka.connect.runtime.distributed.ConnectProtocol.CONNECT_PROTOCOL_V0; +import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG; import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.INTER_WORKER_KEY_GENERATION_ALGORITHM_DEFAULT; import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V1; import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V2; +import static org.apache.kafka.connect.source.SourceTask.TransactionBoundary.CONNECTOR; +import static org.easymock.EasyMock.anyLong; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.capture; import static org.easymock.EasyMock.newCapture; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @SuppressWarnings("deprecation") @RunWith(PowerMockRunner.class) -@PrepareForTest({DistributedHerder.class, Plugins.class}) +@PrepareForTest({DistributedHerder.class, Plugins.class, RestClient.class}) @PowerMockIgnore({"javax.management.*", "javax.crypto.*"}) -public class DistributedHerderTest { +public class DistributedHerderTest extends ThreadedTest { private static final Map HERDER_CONFIG = new HashMap<>(); static { HERDER_CONFIG.put(DistributedConfig.STATUS_STORAGE_TOPIC_CONFIG, "status-topic"); @@ -173,13 +191,13 @@ public class DistributedHerderTest { } private static final ClusterConfigState SNAPSHOT = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), - TASK_CONFIGS_MAP, Collections.emptySet()); + TASK_CONFIGS_MAP, Collections.emptyMap(), Collections.emptyMap(), Collections.emptySet(), Collections.emptySet()); private static final ClusterConfigState SNAPSHOT_PAUSED_CONN1 = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.PAUSED), - TASK_CONFIGS_MAP, Collections.emptySet()); + TASK_CONFIGS_MAP, Collections.emptyMap(), Collections.emptyMap(), Collections.emptySet(), Collections.emptySet()); private static final ClusterConfigState SNAPSHOT_UPDATED_CONN1_CONFIG = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG_UPDATED), Collections.singletonMap(CONN1, TargetState.STARTED), - TASK_CONFIGS_MAP, Collections.emptySet()); + TASK_CONFIGS_MAP, Collections.emptyMap(), Collections.emptyMap(), Collections.emptySet(), Collections.emptySet()); private static final String WORKER_ID = "localhost:8083"; private static final String KAFKA_CLUSTER_ID = "I4ZmrWqfT2e-upky_4fdPA"; @@ -202,6 +220,7 @@ public class DistributedHerderTest { private ConfigBackingStore.UpdateListener configUpdateListener; private WorkerRebalanceListener rebalanceListener; + private ExecutorService herderExecutor; private SinkConnectorConfig conn1SinkConfig; private SinkConnectorConfig conn1SinkConfigUpdated; @@ -243,6 +262,10 @@ public void setUp() throws Exception { @After public void tearDown() { if (metrics != null) metrics.stop(); + if (herderExecutor != null) { + herderExecutor.shutdownNow(); + herderExecutor = null; + } } @Test @@ -252,7 +275,7 @@ public void testJoinAssignment() throws Exception { EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); EasyMock.expect(worker.getPlugins()).andReturn(plugins); expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1)); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); Capture> onStart = newCapture(); worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED), capture(onStart)); @@ -265,7 +288,7 @@ public void testJoinAssignment() throws Exception { EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); PowerMock.expectLastCall(); EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS); - worker.startTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -287,7 +310,7 @@ public void testRebalance() throws Exception { EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); EasyMock.expect(worker.getPlugins()).andReturn(plugins); expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1)); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); Capture> onFirstStart = newCapture(); worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED), capture(onFirstStart)); @@ -299,7 +322,7 @@ public void testRebalance() throws Exception { PowerMock.expectLastCall(); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS); - worker.startTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -348,7 +371,7 @@ public void testIncrementalCooperativeRebalanceForNewMember() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V1); expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -372,7 +395,7 @@ public void testIncrementalCooperativeRebalanceForNewMember() throws Exception { EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS); - worker.startTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -443,9 +466,9 @@ public void testIncrementalCooperativeRebalanceWithDelay() throws Exception { ConnectProtocol.Assignment.NO_ERROR, 1, Collections.emptyList(), Arrays.asList(TASK2), rebalanceDelay); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); - worker.startTask(EasyMock.eq(TASK2), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK2), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -477,7 +500,7 @@ public void testIncrementalCooperativeRebalanceWithDelay() throws Exception { EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS); - worker.startTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -504,7 +527,7 @@ public void testRebalanceFailedConnector() throws Exception { EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); EasyMock.expect(worker.getPlugins()).andReturn(plugins); expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1)); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); Capture> onFirstStart = newCapture(); worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED), capture(onFirstStart)); @@ -516,7 +539,7 @@ public void testRebalanceFailedConnector() throws Exception { PowerMock.expectLastCall(); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS); - worker.startTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -575,7 +598,7 @@ public void revokeAndReassign(boolean incompleteRebalance) throws TimeoutExcepti EasyMock.expect(worker.getPlugins()).andReturn(plugins); // The lists need to be mutable because assignments might be removed expectRebalance(configOffset, new ArrayList<>(singletonList(CONN1)), new ArrayList<>(singletonList(TASK1))); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); Capture> onFirstStart = newCapture(); worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED), capture(onFirstStart)); @@ -587,7 +610,7 @@ public void revokeAndReassign(boolean incompleteRebalance) throws TimeoutExcepti PowerMock.expectLastCall(); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS); - worker.startTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -601,7 +624,7 @@ public void revokeAndReassign(boolean incompleteRebalance) throws TimeoutExcepti configOffset++; expectRebalance(configOffset, Arrays.asList(), Arrays.asList()); // give it the wrong snapshot, as if we're out of sync/can't reach the broker - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.requestRejoin(); PowerMock.expectLastCall(); // tick exits early because we failed, and doesn't do the poll at the end of the method @@ -619,8 +642,8 @@ public void revokeAndReassign(boolean incompleteRebalance) throws TimeoutExcepti ClusterConfigState secondSnapshot = new ClusterConfigState( configOffset, null, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), - TASK_CONFIGS_MAP, Collections.emptySet()); - expectPostRebalanceCatchup(secondSnapshot); + TASK_CONFIGS_MAP, Collections.emptyMap(), Collections.emptyMap(), Collections.emptySet(), Collections.emptySet()); + expectConfigRefreshAndSnapshot(secondSnapshot); } member.requestRejoin(); PowerMock.expectLastCall(); @@ -684,8 +707,8 @@ public void testHaltCleansUpWorker() { public void testCreateConnector() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.wakeup(); PowerMock.expectLastCall(); @@ -739,8 +762,8 @@ public void testCreateConnector() throws Exception { public void testCreateConnectorFailedValidation() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); HashMap config = new HashMap<>(CONN2_CONFIG); config.remove(ConnectorConfig.NAME_CONFIG); @@ -791,22 +814,231 @@ public void testCreateConnectorFailedValidation() throws Exception { PowerMock.verifyAll(); } - @SuppressWarnings("unchecked") @Test - public void testConnectorNameConflictsWithWorkerGroupId() throws Exception { + public void testConnectorNameConflictsWithWorkerGroupId() { Map config = new HashMap<>(CONN2_CONFIG); config.put(ConnectorConfig.NAME_CONFIG, "test-group"); - Connector connectorMock = PowerMock.createMock(SinkConnector.class); + SinkConnector connectorMock = PowerMock.createMock(SinkConnector.class); + + PowerMock.replayAll(connectorMock); // CONN2 creation should fail because the worker group id (connect-test-group) conflicts with // the consumer group id we would use for this sink - Map validatedConfigs = - herder.validateBasicConnectorConfig(connectorMock, ConnectorConfig.configDef(), config); + Map validatedConfigs = herder.validateSinkConnectorConfig( + connectorMock, SinkConnectorConfig.configDef(), config); ConfigValue nameConfig = validatedConfigs.get(ConnectorConfig.NAME_CONFIG); - assertNotNull(nameConfig.errorMessages()); assertFalse(nameConfig.errorMessages().isEmpty()); + + PowerMock.verifyAll(); + } + + @Test + public void testExactlyOnceSourceSupportValidation() { + herder = exactlyOnceHerder(); + Map config = new HashMap<>(); + config.put(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG, REQUIRED.toString()); + + SourceConnector connectorMock = PowerMock.createMock(SourceConnector.class); + EasyMock.expect(connectorMock.exactlyOnceSupport(EasyMock.eq(config))) + .andReturn(ExactlyOnceSupport.SUPPORTED); + + PowerMock.replayAll(connectorMock); + + Map validatedConfigs = herder.validateSourceConnectorConfig( + connectorMock, SourceConnectorConfig.configDef(), config); + + ConfigValue exactlyOnceSupportConfig = validatedConfigs.get(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG); + assertTrue(exactlyOnceSupportConfig.errorMessages().isEmpty()); + + PowerMock.verifyAll(); + } + + @Test + public void testExactlyOnceSourceSupportValidationOnUnsupportedConnector() { + herder = exactlyOnceHerder(); + Map config = new HashMap<>(); + config.put(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG, REQUIRED.toString()); + + SourceConnector connectorMock = PowerMock.createMock(SourceConnector.class); + EasyMock.expect(connectorMock.exactlyOnceSupport(EasyMock.eq(config))) + .andReturn(ExactlyOnceSupport.UNSUPPORTED); + + PowerMock.replayAll(connectorMock); + + Map validatedConfigs = herder.validateSourceConnectorConfig( + connectorMock, SourceConnectorConfig.configDef(), config); + + ConfigValue exactlyOnceSupportConfig = validatedConfigs.get(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG); + assertFalse(exactlyOnceSupportConfig.errorMessages().isEmpty()); + + PowerMock.verifyAll(); + } + + @Test + public void testExactlyOnceSourceSupportValidationOnUnknownConnector() { + herder = exactlyOnceHerder(); + Map config = new HashMap<>(); + config.put(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG, REQUIRED.toString()); + + SourceConnector connectorMock = PowerMock.createMock(SourceConnector.class); + EasyMock.expect(connectorMock.exactlyOnceSupport(EasyMock.eq(config))) + .andReturn(null); + + PowerMock.replayAll(connectorMock); + + Map validatedConfigs = herder.validateSourceConnectorConfig( + connectorMock, SourceConnectorConfig.configDef(), config); + + ConfigValue exactlyOnceSupportConfig = validatedConfigs.get(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG); + assertFalse(exactlyOnceSupportConfig.errorMessages().isEmpty()); + + PowerMock.verifyAll(); + } + + @Test + public void testExactlyOnceSourceSupportValidationHandlesConnectorErrorsGracefully() { + herder = exactlyOnceHerder(); + Map config = new HashMap<>(); + config.put(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG, REQUIRED.toString()); + + SourceConnector connectorMock = PowerMock.createMock(SourceConnector.class); + String errorMessage = "time to add a new unit test :)"; + EasyMock.expect(connectorMock.exactlyOnceSupport(EasyMock.eq(config))) + .andThrow(new NullPointerException(errorMessage)); + + PowerMock.replayAll(connectorMock); + + Map validatedConfigs = herder.validateSourceConnectorConfig( + connectorMock, SourceConnectorConfig.configDef(), config); + + ConfigValue exactlyOnceSupportConfig = validatedConfigs.get(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG); + assertTrue(exactlyOnceSupportConfig.errorMessages().stream().anyMatch(m -> m.contains(errorMessage))); + + PowerMock.verifyAll(); + } + + @Test + public void testExactlyOnceSourceSupportValidationWhenExactlyOnceNotEnabledOnWorker() { + Map config = new HashMap<>(); + config.put(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG, REQUIRED.toString()); + + SourceConnector connectorMock = PowerMock.createMock(SourceConnector.class); + EasyMock.expect(connectorMock.exactlyOnceSupport(EasyMock.eq(config))) + .andReturn(ExactlyOnceSupport.SUPPORTED); + + PowerMock.replayAll(connectorMock); + + Map validatedConfigs = herder.validateSourceConnectorConfig( + connectorMock, SourceConnectorConfig.configDef(), config); + + ConfigValue exactlyOnceSupportConfig = validatedConfigs.get(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG); + assertFalse(exactlyOnceSupportConfig.errorMessages().isEmpty()); + + PowerMock.verifyAll(); + } + + @Test + public void testExactlyOnceSourceSupportValidationHandlesInvalidValuesGracefully() { + herder = exactlyOnceHerder(); + Map config = new HashMap<>(); + config.put(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG, "invalid"); + + SourceConnector connectorMock = PowerMock.createMock(SourceConnector.class); + + PowerMock.replayAll(connectorMock); + + Map validatedConfigs = herder.validateSourceConnectorConfig( + connectorMock, SourceConnectorConfig.configDef(), config); + + ConfigValue exactlyOnceSupportConfig = validatedConfigs.get(SourceConnectorConfig.EXACTLY_ONCE_SUPPORT_CONFIG); + assertFalse(exactlyOnceSupportConfig.errorMessages().isEmpty()); + + PowerMock.verifyAll(); + } + + @Test + public void testConnectorTransactionBoundaryValidation() { + herder = exactlyOnceHerder(); + Map config = new HashMap<>(); + config.put(SourceConnectorConfig.TRANSACTION_BOUNDARY_CONFIG, CONNECTOR.toString()); + + SourceConnector connectorMock = PowerMock.createMock(SourceConnector.class); + EasyMock.expect(connectorMock.canDefineTransactionBoundaries(EasyMock.eq(config))) + .andReturn(ConnectorTransactionBoundaries.SUPPORTED); + + PowerMock.replayAll(connectorMock); + + Map validatedConfigs = herder.validateSourceConnectorConfig( + connectorMock, SourceConnectorConfig.configDef(), config); + + ConfigValue transactionBoundaryConfig = validatedConfigs.get(SourceConnectorConfig.TRANSACTION_BOUNDARY_CONFIG); + assertTrue(transactionBoundaryConfig.errorMessages().isEmpty()); + + PowerMock.verifyAll(); + } + + @Test + public void testConnectorTransactionBoundaryValidationOnUnsupportedConnector() { + herder = exactlyOnceHerder(); + Map config = new HashMap<>(); + config.put(SourceConnectorConfig.TRANSACTION_BOUNDARY_CONFIG, CONNECTOR.toString()); + + SourceConnector connectorMock = PowerMock.createMock(SourceConnector.class); + EasyMock.expect(connectorMock.canDefineTransactionBoundaries(EasyMock.eq(config))) + .andReturn(ConnectorTransactionBoundaries.UNSUPPORTED); + + PowerMock.replayAll(connectorMock); + + Map validatedConfigs = herder.validateSourceConnectorConfig( + connectorMock, SourceConnectorConfig.configDef(), config); + + ConfigValue transactionBoundaryConfig = validatedConfigs.get(SourceConnectorConfig.TRANSACTION_BOUNDARY_CONFIG); + assertFalse(transactionBoundaryConfig.errorMessages().isEmpty()); + + PowerMock.verifyAll(); + } + + @Test + public void testConnectorTransactionBoundaryValidationHandlesConnectorErrorsGracefully() { + herder = exactlyOnceHerder(); + Map config = new HashMap<>(); + config.put(SourceConnectorConfig.TRANSACTION_BOUNDARY_CONFIG, CONNECTOR.toString()); + + SourceConnector connectorMock = PowerMock.createMock(SourceConnector.class); + String errorMessage = "Wait I thought we tested for this?"; + EasyMock.expect(connectorMock.canDefineTransactionBoundaries(EasyMock.eq(config))) + .andThrow(new ConnectException(errorMessage)); + + PowerMock.replayAll(connectorMock); + + Map validatedConfigs = herder.validateSourceConnectorConfig( + connectorMock, SourceConnectorConfig.configDef(), config); + + ConfigValue transactionBoundaryConfig = validatedConfigs.get(SourceConnectorConfig.TRANSACTION_BOUNDARY_CONFIG); + assertTrue(transactionBoundaryConfig.errorMessages().stream().anyMatch(m -> m.contains(errorMessage))); + + PowerMock.verifyAll(); + } + + @Test + public void testConnectorTransactionBoundaryValidationHandlesInvalidValuesGracefully() { + herder = exactlyOnceHerder(); + Map config = new HashMap<>(); + config.put(SourceConnectorConfig.TRANSACTION_BOUNDARY_CONFIG, "CONNECTOR.toString()"); + + SourceConnector connectorMock = PowerMock.createMock(SourceConnector.class); + + PowerMock.replayAll(connectorMock); + + Map validatedConfigs = herder.validateSourceConnectorConfig( + connectorMock, SourceConnectorConfig.configDef(), config); + + ConfigValue transactionBoundaryConfig = validatedConfigs.get(SourceConnectorConfig.TRANSACTION_BOUNDARY_CONFIG); + assertFalse(transactionBoundaryConfig.errorMessages().isEmpty()); + + PowerMock.verifyAll(); } @Test @@ -823,8 +1055,8 @@ public void testCreateConnectorAlreadyExists() throws Exception { return null; }); - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.wakeup(); PowerMock.expectLastCall(); @@ -862,8 +1094,8 @@ public void testDestroyConnector() throws Exception { EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); // Start with one connector EasyMock.expect(worker.getPlugins()).andReturn(plugins); - expectRebalance(1, Arrays.asList(CONN1), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Arrays.asList(CONN1), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); Capture> onStart = newCapture(); worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED), capture(onStart)); @@ -896,9 +1128,9 @@ public void testDestroyConnector() throws Exception { statusBackingStore.deleteTopic(EasyMock.eq(CONN1), EasyMock.eq(BAR_TOPIC)); PowerMock.expectLastCall().times(2); expectRebalance(Arrays.asList(CONN1), Arrays.asList(TASK1), - ConnectProtocol.Assignment.NO_ERROR, 2, - Collections.emptyList(), Collections.emptyList(), 0); - expectPostRebalanceCatchup(ClusterConfigState.EMPTY); + ConnectProtocol.Assignment.NO_ERROR, 2, "leader", "leaderUrl", + Collections.emptyList(), Collections.emptyList(), 0, true); + expectConfigRefreshAndSnapshot(ClusterConfigState.EMPTY); member.requestRejoin(); PowerMock.expectLastCall(); PowerMock.replayAll(); @@ -926,8 +1158,8 @@ public void testRestartConnector() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); EasyMock.expect(worker.getPlugins()).andReturn(plugins); - expectRebalance(1, singletonList(CONN1), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, singletonList(CONN1), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); Capture> onFirstStart = newCapture(); @@ -979,8 +1211,8 @@ public void testRestartUnknownConnector() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -1014,7 +1246,7 @@ public void testRestartConnectorRedirectToLeader() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -1048,8 +1280,8 @@ public void testRestartConnectorRedirectToOwner() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -1097,8 +1329,8 @@ public void testRestartConnectorAndTasksUnknownConnector() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -1131,7 +1363,7 @@ public void testRestartConnectorAndTasksNotLeader() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -1166,8 +1398,8 @@ public void testRestartConnectorAndTasksUnknownStatus() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -1206,8 +1438,8 @@ public void testRestartConnectorAndTasksSuccess() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -1304,13 +1536,15 @@ public void testDoRestartConnectorAndTasksOnlyTasks() throws Exception { EasyMock.expect(herder.assignment.connectors()).andReturn(Collections.emptyList()).anyTimes(); EasyMock.expect(herder.assignment.tasks()).andReturn(Collections.singletonList(taskId)).anyTimes(); + herder.configState = SNAPSHOT; + worker.stopAndAwaitTasks(Collections.singletonList(taskId)); PowerMock.expectLastCall(); herder.onRestart(taskId); EasyMock.expectLastCall(); - worker.startTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.anyObject(TargetState.class)); PowerMock.expectLastCall().andReturn(true); @@ -1335,6 +1569,8 @@ public void testDoRestartConnectorAndTasksBoth() throws Exception { EasyMock.expect(herder.assignment.connectors()).andReturn(Collections.singletonList(CONN1)).anyTimes(); EasyMock.expect(herder.assignment.tasks()).andReturn(Collections.singletonList(taskId)).anyTimes(); + herder.configState = SNAPSHOT; + worker.stopAndAwaitConnector(CONN1); PowerMock.expectLastCall(); @@ -1352,7 +1588,7 @@ public void testDoRestartConnectorAndTasksBoth() throws Exception { herder.onRestart(taskId); EasyMock.expectLastCall(); - worker.startTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.anyObject(TargetState.class)); PowerMock.expectLastCall().andReturn(true); @@ -1368,11 +1604,11 @@ public void testRestartTask() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); - expectRebalance(1, Collections.emptyList(), singletonList(TASK0)); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Collections.emptyList(), singletonList(TASK0), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); - worker.startTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); @@ -1386,7 +1622,7 @@ public void testRestartTask() throws Exception { worker.stopAndAwaitTask(TASK0); PowerMock.expectLastCall(); - worker.startTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); @@ -1407,7 +1643,7 @@ public void testRestartUnknownTask() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -1454,7 +1690,7 @@ public void testRestartTaskRedirectToLeader() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -1488,8 +1724,8 @@ public void testRestartTaskRedirectToOwner() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -1581,7 +1817,7 @@ public void testConnectorConfigUpdate() throws Exception { // join expectRebalance(1, Arrays.asList(CONN1), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); Capture> onFirstStart = newCapture(); worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), @@ -1648,7 +1884,7 @@ public void testConnectorPaused() throws Exception { // join expectRebalance(1, Arrays.asList(CONN1), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); Capture> onStart = newCapture(); worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), @@ -1708,7 +1944,7 @@ public void testConnectorResumed() throws Exception { // start with the connector paused expectRebalance(1, Arrays.asList(CONN1), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT_PAUSED_CONN1); + expectConfigRefreshAndSnapshot(SNAPSHOT_PAUSED_CONN1); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); Capture> onStart = newCapture(); worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), @@ -1771,8 +2007,8 @@ public void testUnknownConnectorPaused() throws Exception { // join expectRebalance(1, Collections.emptyList(), singletonList(TASK0)); - expectPostRebalanceCatchup(SNAPSHOT); - worker.startTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + expectConfigRefreshAndSnapshot(SNAPSHOT); + worker.startSourceTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -1810,8 +2046,8 @@ public void testConnectorPausedRunningTaskOnly() throws Exception { // join expectRebalance(1, Collections.emptyList(), singletonList(TASK0)); - expectPostRebalanceCatchup(SNAPSHOT); - worker.startTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + expectConfigRefreshAndSnapshot(SNAPSHOT); + worker.startSourceTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -1858,8 +2094,8 @@ public void testConnectorResumedRunningTaskOnly() throws Exception { // join expectRebalance(1, Collections.emptyList(), singletonList(TASK0)); - expectPostRebalanceCatchup(SNAPSHOT_PAUSED_CONN1); - worker.startTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + expectConfigRefreshAndSnapshot(SNAPSHOT_PAUSED_CONN1); + worker.startSourceTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.PAUSED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -1928,7 +2164,7 @@ public void testTaskConfigAdded() { expectRebalance(Collections.emptyList(), Collections.emptyList(), ConnectProtocol.Assignment.NO_ERROR, 1, Collections.emptyList(), Arrays.asList(TASK0)); - worker.startTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK0), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -1950,8 +2186,8 @@ public void testJoinLeaderCatchUpFails() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); expectRebalance(Collections.emptyList(), Collections.emptyList(), - ConnectProtocol.Assignment.CONFIG_MISMATCH, 1, Collections.emptyList(), - Collections.emptyList()); + ConnectProtocol.Assignment.CONFIG_MISMATCH, 1, "leader", "leaderUrl", Collections.emptyList(), + Collections.emptyList(), 0, true); // Reading to end of log times out configBackingStore.refresh(EasyMock.anyLong(), EasyMock.anyObject(TimeUnit.class)); EasyMock.expectLastCall().andThrow(new TimeoutException()); @@ -1960,8 +2196,8 @@ public void testJoinLeaderCatchUpFails() throws Exception { member.requestRejoin(); // After backoff, restart the process and this time succeed - expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1)); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); Capture> onStart = newCapture(); @@ -1975,7 +2211,7 @@ public void testJoinLeaderCatchUpFails() throws Exception { PowerMock.expectLastCall(); EasyMock.expect(worker.getPlugins()).andReturn(plugins); EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS); - worker.startTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); @@ -1983,7 +2219,7 @@ public void testJoinLeaderCatchUpFails() throws Exception { PowerMock.expectLastCall(); // one more tick, to make sure we don't keep trying to read to the config topic unnecessarily - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -2018,8 +2254,8 @@ public void testJoinLeaderCatchUpRetriesForIncrementalCooperative() throws Excep // Join group and as leader fail to do assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V1); - expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1)); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -2027,7 +2263,7 @@ public void testJoinLeaderCatchUpRetriesForIncrementalCooperative() throws Excep // The leader got its assignment expectRebalance(Collections.emptyList(), Collections.emptyList(), ConnectProtocol.Assignment.NO_ERROR, - 1, Arrays.asList(CONN1), Arrays.asList(TASK1), 0); + 1, "leader", "leaderUrl", Arrays.asList(CONN1), Arrays.asList(TASK1), 0, true); EasyMock.expect(worker.getPlugins()).andReturn(plugins); Capture> onStart = newCapture(); @@ -2042,7 +2278,7 @@ public void testJoinLeaderCatchUpRetriesForIncrementalCooperative() throws Excep EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS); - worker.startTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -2051,8 +2287,8 @@ public void testJoinLeaderCatchUpRetriesForIncrementalCooperative() throws Excep // Another rebalance is triggered but this time it fails to read to the max offset and // triggers a re-sync expectRebalance(Collections.emptyList(), Collections.emptyList(), - ConnectProtocol.Assignment.CONFIG_MISMATCH, 1, Collections.emptyList(), - Collections.emptyList()); + ConnectProtocol.Assignment.CONFIG_MISMATCH, 1, "leader", "leaderUrl", + Collections.emptyList(), Collections.emptyList(), 0, true); // The leader will retry a few times to read to the end of the config log int retries = 2; @@ -2068,8 +2304,8 @@ public void testJoinLeaderCatchUpRetriesForIncrementalCooperative() throws Excep // After a few retries succeed to read the log to the end expectRebalance(Collections.emptyList(), Collections.emptyList(), ConnectProtocol.Assignment.NO_ERROR, - 1, Arrays.asList(CONN1), Arrays.asList(TASK1), 0); - expectPostRebalanceCatchup(SNAPSHOT); + 1, "leader", "leaderUrl", Arrays.asList(CONN1), Arrays.asList(TASK1), 0, true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -2109,16 +2345,16 @@ public void testJoinLeaderCatchUpFailsForIncrementalCooperative() throws Excepti // Join group and as leader fail to do assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V1); - expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1)); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // The leader got its assignment expectRebalance(Collections.emptyList(), Collections.emptyList(), - ConnectProtocol.Assignment.NO_ERROR, - 1, Arrays.asList(CONN1), Arrays.asList(TASK1), 0); + ConnectProtocol.Assignment.NO_ERROR, 1, + "leader", "leaderUrl", Arrays.asList(CONN1), Arrays.asList(TASK1), 0, true); EasyMock.expect(worker.getPlugins()).andReturn(plugins); // and the new assignment started @@ -2134,7 +2370,7 @@ public void testJoinLeaderCatchUpFailsForIncrementalCooperative() throws Excepti EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS); - worker.startTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), + worker.startSourceTask(EasyMock.eq(TASK1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); @@ -2143,8 +2379,8 @@ public void testJoinLeaderCatchUpFailsForIncrementalCooperative() throws Excepti // Another rebalance is triggered but this time it fails to read to the max offset and // triggers a re-sync expectRebalance(Collections.emptyList(), Collections.emptyList(), - ConnectProtocol.Assignment.CONFIG_MISMATCH, 1, Collections.emptyList(), - Collections.emptyList()); + ConnectProtocol.Assignment.CONFIG_MISMATCH, 1, "leader", "leaderUrl", + Collections.emptyList(), Collections.emptyList(), 0, true); // The leader will exhaust the retries while trying to read to the end of the config log int maxRetries = 5; @@ -2165,8 +2401,9 @@ public void testJoinLeaderCatchUpFailsForIncrementalCooperative() throws Excepti // The worker gets back the assignment that had given up expectRebalance(Collections.emptyList(), Collections.emptyList(), ConnectProtocol.Assignment.NO_ERROR, - 1, Arrays.asList(CONN1), Arrays.asList(TASK1), 0); - expectPostRebalanceCatchup(SNAPSHOT); + 1, "leader", "leaderUrl", Arrays.asList(CONN1), Arrays.asList(TASK1), + 0, true); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); @@ -2205,7 +2442,7 @@ public void testAccessors() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); EasyMock.expect(worker.getPlugins()).andReturn(plugins).anyTimes(); - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT).times(2); WorkerConfigTransformer configTransformer = EasyMock.mock(WorkerConfigTransformer.class); @@ -2214,9 +2451,9 @@ public void testAccessors() throws Exception { EasyMock.replay(configTransformer); ClusterConfigState snapshotWithTransform = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), - TASK_CONFIGS_MAP, Collections.emptySet(), configTransformer); + TASK_CONFIGS_MAP, Collections.emptyMap(), Collections.emptyMap(), Collections.emptySet(), Collections.emptySet(), configTransformer); - expectPostRebalanceCatchup(snapshotWithTransform); + expectConfigRefreshAndSnapshot(snapshotWithTransform); member.wakeup(); @@ -2260,8 +2497,8 @@ public void testAccessors() throws Exception { @Test public void testPutConnectorConfig() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); - expectRebalance(1, Arrays.asList(CONN1), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Arrays.asList(CONN1), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0); Capture> onFirstStart = newCapture(); worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), @@ -2357,7 +2594,7 @@ public void testKeyRotationWhenWorkerBecomesLeader() throws Exception { EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2); expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); // First rebalance: poll indefinitely as no key has been read yet, so expiration doesn't come into play member.poll(Long.MAX_VALUE); EasyMock.expectLastCall(); @@ -2366,13 +2603,13 @@ public void testKeyRotationWhenWorkerBecomesLeader() throws Exception { SessionKey initialKey = new SessionKey(EasyMock.mock(SecretKey.class), 0); ClusterConfigState snapshotWithKey = new ClusterConfigState(2, initialKey, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), - TASK_CONFIGS_MAP, Collections.emptySet()); - expectPostRebalanceCatchup(snapshotWithKey); + TASK_CONFIGS_MAP, Collections.emptyMap(), Collections.emptyMap(), Collections.emptySet(), Collections.emptySet()); + expectConfigRefreshAndSnapshot(snapshotWithKey); // Second rebalance: poll indefinitely as worker is follower, so expiration still doesn't come into play member.poll(Long.MAX_VALUE); EasyMock.expectLastCall(); - expectRebalance(2, Collections.emptyList(), Collections.emptyList(), "member", MEMBER_URL); + expectRebalance(2, Collections.emptyList(), Collections.emptyList(), "member", MEMBER_URL, true); Capture updatedKey = EasyMock.newCapture(); configBackingStore.putSessionKey(EasyMock.capture(updatedKey)); EasyMock.expectLastCall().andAnswer(() -> { @@ -2401,15 +2638,15 @@ public void testKeyRotationDisabledWhenWorkerBecomesFollower() throws Exception EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2); - expectRebalance(1, Collections.emptyList(), Collections.emptyList(), "member", MEMBER_URL); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), "member", MEMBER_URL, true); SecretKey initialSecretKey = EasyMock.mock(SecretKey.class); EasyMock.expect(initialSecretKey.getAlgorithm()).andReturn(DistributedConfig.INTER_WORKER_KEY_GENERATION_ALGORITHM_DEFAULT).anyTimes(); EasyMock.expect(initialSecretKey.getEncoded()).andReturn(new byte[32]).anyTimes(); SessionKey initialKey = new SessionKey(initialSecretKey, time.milliseconds()); ClusterConfigState snapshotWithKey = new ClusterConfigState(1, initialKey, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), - TASK_CONFIGS_MAP, Collections.emptySet()); - expectPostRebalanceCatchup(snapshotWithKey); + TASK_CONFIGS_MAP, Collections.emptyMap(), Collections.emptyMap(), Collections.emptySet(), Collections.emptySet()); + expectConfigRefreshAndSnapshot(snapshotWithKey); // First rebalance: poll for a limited time as worker is leader and must wake up for key expiration Capture firstPollTimeout = EasyMock.newCapture(); member.poll(EasyMock.captureLong(firstPollTimeout)); @@ -2540,8 +2777,8 @@ public void testFailedToWriteSessionKey() throws Exception { // session key to the config topic, and fail EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2); - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); configBackingStore.putSessionKey(anyObject(SessionKey.class)); EasyMock.expectLastCall().andThrow(new ConnectException("Oh no!")); @@ -2549,7 +2786,7 @@ public void testFailedToWriteSessionKey() throws Exception { // then ensure we're still active in the group // then try a second time to write a new session key, // then finally begin polling for group activity - expectPostRebalanceCatchup(SNAPSHOT); + expectConfigRefreshAndSnapshot(SNAPSHOT); member.ensureActive(); PowerMock.expectLastCall(); configBackingStore.putSessionKey(anyObject(SessionKey.class)); @@ -2573,7 +2810,7 @@ public void testFailedToReadBackNewlyWrittenSessionKey() throws Exception { SessionKey sessionKey = new SessionKey(secretKey, time.milliseconds()); ClusterConfigState snapshotWithSessionKey = new ClusterConfigState(1, sessionKey, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), - TASK_CONFIGS_MAP, Collections.emptySet()); + TASK_CONFIGS_MAP, Collections.emptyMap(), Collections.emptyMap(), Collections.emptySet(), Collections.emptySet()); // First tick -- after joining the group, we try to write a new session key to // the config topic, and fail (in this case, we're trying to simulate that we've @@ -2582,8 +2819,8 @@ public void testFailedToReadBackNewlyWrittenSessionKey() throws Exception { // to write the key) EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2); - expectRebalance(1, Collections.emptyList(), Collections.emptyList()); - expectPostRebalanceCatchup(SNAPSHOT); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); configBackingStore.putSessionKey(anyObject(SessionKey.class)); EasyMock.expectLastCall().andThrow(new ConnectException("Oh no!")); @@ -2611,6 +2848,566 @@ public void testFailedToReadBackNewlyWrittenSessionKey() throws Exception { PowerMock.verifyAll(); } + @Test + public void testFenceZombiesInvalidSignature() { + // Don't have to run the whole gamut of scenarios (invalid signature, missing signature, earlier protocol that doesn't require signatures) + // since the task config tests cover that pretty well. One sanity check to ensure that this method is guarded should be sufficient. + Callback taskConfigCb = EasyMock.mock(Callback.class); + Capture errorCapture = Capture.newInstance(); + taskConfigCb.onCompletion(capture(errorCapture), EasyMock.eq(null)); + EasyMock.expectLastCall().once(); + + EasyMock.expect(member.currentProtocolVersion()).andReturn(CONNECT_PROTOCOL_V2).anyTimes(); + + InternalRequestSignature signature = EasyMock.mock(InternalRequestSignature.class); + EasyMock.expect(signature.keyAlgorithm()).andReturn("HmacSHA256").anyTimes(); + EasyMock.expect(signature.isValid(EasyMock.anyObject())).andReturn(false).anyTimes(); + + PowerMock.replayAll(taskConfigCb, signature); + + herder.fenceZombies(CONN1, taskConfigCb, signature); + + PowerMock.verifyAll(); + assertTrue(errorCapture.getValue() instanceof ConnectRestException); + assertEquals(FORBIDDEN.getStatusCode(), ((ConnectRestException) errorCapture.getValue()).statusCode()); + } + + @Test + public void testTaskRequestedZombieFencingForwardedToLeader() throws Exception { + testTaskRequestedZombieFencingForwardingToLeader(true); + } + + @Test + public void testTaskRequestedZombieFencingFailedForwardToLeader() throws Exception { + testTaskRequestedZombieFencingForwardingToLeader(false); + } + + private void testTaskRequestedZombieFencingForwardingToLeader(boolean succeed) throws Exception { + expectHerderStartup(); + ExecutorService forwardRequestExecutor = EasyMock.mock(ExecutorService.class); + herder.forwardRequestExecutor = forwardRequestExecutor; + + EasyMock.expect(member.memberId()).andStubReturn("member"); + EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2); + expectConfigRefreshAndSnapshot(SNAPSHOT); + + expectRebalance(1, Collections.emptyList(), Collections.emptyList()); + + expectAnyTicks(); + + member.wakeup(); + EasyMock.expectLastCall(); + + PowerMock.mockStatic(RestClient.class); + + org.easymock.IExpectationSetters> expectRequest = EasyMock.expect( + RestClient.httpRequest( + anyObject(), EasyMock.eq("PUT"), EasyMock.isNull(), EasyMock.isNull(), EasyMock.isNull(), anyObject(), anyObject(), anyObject() + )); + if (succeed) { + expectRequest.andReturn(null); + } else { + expectRequest.andThrow(new ConnectRestException(409, "Rebalance :(")); + } + + Capture forwardRequest = EasyMock.newCapture(); + forwardRequestExecutor.execute(EasyMock.capture(forwardRequest)); + EasyMock.expectLastCall().andAnswer(() -> { + forwardRequest.getValue().run(); + return null; + }); + + expectHerderShutdown(true); + forwardRequestExecutor.shutdown(); + EasyMock.expectLastCall(); + EasyMock.expect(forwardRequestExecutor.awaitTermination(anyLong(), anyObject())).andReturn(true); + + PowerMock.replayAll(forwardRequestExecutor); + + + startBackgroundHerder(); + + FutureCallback fencing = new FutureCallback<>(); + herder.fenceZombies(TASK1, fencing); + + if (!succeed) { + ExecutionException fencingException = + assertThrows(ExecutionException.class, () -> fencing.get(10, TimeUnit.SECONDS)); + assertTrue(fencingException.getCause() instanceof ConnectException); + } else { + fencing.get(10, TimeUnit.SECONDS); + } + + stopBackgroundHerder(); + + PowerMock.verifyAll(); + } + + @Test + public void testExternalZombieFencingRequestForAlreadyFencedConnector() throws Exception { + ClusterConfigState configState = exactlyOnceSnapshot( + expectNewSessionKey(), + TASK_CONFIGS_MAP, + Collections.singletonMap(CONN1, 12), + Collections.singletonMap(CONN1, 5), + Collections.emptySet() + ); + testExternalZombieFencingRequestThatRequiresNoPhysicalFencing(configState, false); + } + + @Test + public void testExternalZombieFencingRequestForSingleTaskConnector() throws Exception { + ClusterConfigState configState = exactlyOnceSnapshot( + expectNewSessionKey(), + Collections.singletonMap(TASK1, TASK_CONFIG), + Collections.singletonMap(CONN1, 1), + Collections.singletonMap(CONN1, 5), + Collections.singleton(CONN1) + ); + testExternalZombieFencingRequestThatRequiresNoPhysicalFencing(configState, true); + } + + @Test + public void testExternalZombieFencingRequestForFreshConnector() throws Exception { + ClusterConfigState configState = exactlyOnceSnapshot( + expectNewSessionKey(), + TASK_CONFIGS_MAP, + Collections.emptyMap(), + Collections.singletonMap(CONN1, 5), + Collections.singleton(CONN1) + ); + testExternalZombieFencingRequestThatRequiresNoPhysicalFencing(configState, true); + } + + private void testExternalZombieFencingRequestThatRequiresNoPhysicalFencing( + ClusterConfigState configState, boolean expectTaskCountRecord + ) throws Exception { + expectHerderStartup(); + + EasyMock.expect(member.memberId()).andStubReturn("leader"); + EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2); + expectConfigRefreshAndSnapshot(SNAPSHOT); + + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + + expectAnyTicks(); + + member.wakeup(); + EasyMock.expectLastCall().anyTimes(); + + expectConfigRefreshAndSnapshot(configState); + + if (expectTaskCountRecord) { + configBackingStore.putTaskCountRecord(CONN1, 1); + EasyMock.expectLastCall(); + } + + expectHerderShutdown(false); + + PowerMock.replayAll(); + + + startBackgroundHerder(); + + FutureCallback fencing = new FutureCallback<>(); + herder.fenceZombies(CONN1, fencing); + + fencing.get(10, TimeUnit.SECONDS); + + stopBackgroundHerder(); + + PowerMock.verifyAll(); + } + + /** + * Tests zombie fencing that completes extremely quickly, and causes all callback-related logic to be invoked + * effectively as soon as it's put into place. This is not likely to occur in practice, but the test is valuable all the + * same especially since it may shed light on potential deadlocks when the unlikely-but-not-impossible happens. + */ + @Test + public void testExternalZombieFencingRequestImmediateCompletion() throws Exception { + expectHerderStartup(); + EasyMock.expect(member.memberId()).andStubReturn("leader"); + EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2); + expectConfigRefreshAndSnapshot(SNAPSHOT); + + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + SessionKey sessionKey = expectNewSessionKey(); + + expectAnyTicks(); + + member.wakeup(); + EasyMock.expectLastCall(); + + ClusterConfigState configState = exactlyOnceSnapshot( + sessionKey, + TASK_CONFIGS_MAP, + Collections.singletonMap(CONN1, 2), + Collections.singletonMap(CONN1, 5), + Collections.singleton(CONN1) + ); + expectConfigRefreshAndSnapshot(configState); + + // The future returned by Worker::fenceZombies + KafkaFuture workerFencingFuture = EasyMock.mock(KafkaFuture.class); + // The future tracked by the herder (which tracks the fencing performed by the worker and the possible followup write to the config topic) + KafkaFuture herderFencingFuture = EasyMock.mock(KafkaFuture.class); + + // Immediately invoke callbacks that the herder sets up for when the worker fencing and writes to the config topic have completed + for (int i = 0; i < 2; i++) { + Capture> herderFencingCallback = EasyMock.newCapture(); + EasyMock.expect(herderFencingFuture.whenComplete(EasyMock.capture(herderFencingCallback))).andAnswer(() -> { + herderFencingCallback.getValue().accept(null, null); + return null; + }); + } + + Capture> fencingFollowup = EasyMock.newCapture(); + EasyMock.expect(workerFencingFuture.thenApply(EasyMock.capture(fencingFollowup))).andAnswer(() -> { + fencingFollowup.getValue().apply(null); + return herderFencingFuture; + }); + EasyMock.expect(worker.fenceZombies(EasyMock.eq(CONN1), EasyMock.eq(2), EasyMock.eq(CONN1_CONFIG))) + .andReturn(workerFencingFuture); + + expectConfigRefreshAndSnapshot(configState); + + configBackingStore.putTaskCountRecord(CONN1, 1); + EasyMock.expectLastCall(); + + expectHerderShutdown(true); + + PowerMock.replayAll(workerFencingFuture, herderFencingFuture); + + + startBackgroundHerder(); + + FutureCallback fencing = new FutureCallback<>(); + herder.fenceZombies(CONN1, fencing); + + fencing.get(10, TimeUnit.SECONDS); + + stopBackgroundHerder(); + + PowerMock.verifyAll(); + } + + /** + * The herder tries to perform a round of fencing, but fails synchronously while invoking Worker::fenceZombies + */ + @Test + public void testExternalZombieFencingRequestSynchronousFailure() throws Exception { + expectHerderStartup(); + EasyMock.expect(member.memberId()).andStubReturn("leader"); + EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2); + expectConfigRefreshAndSnapshot(SNAPSHOT); + + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + SessionKey sessionKey = expectNewSessionKey(); + + expectAnyTicks(); + + member.wakeup(); + EasyMock.expectLastCall(); + + ClusterConfigState configState = exactlyOnceSnapshot( + sessionKey, + TASK_CONFIGS_MAP, + Collections.singletonMap(CONN1, 2), + Collections.singletonMap(CONN1, 5), + Collections.singleton(CONN1) + ); + expectConfigRefreshAndSnapshot(configState); + + Exception fencingException = new KafkaException("whoops!"); + EasyMock.expect(worker.fenceZombies(EasyMock.eq(CONN1), EasyMock.eq(2), EasyMock.eq(CONN1_CONFIG))) + .andThrow(fencingException); + + expectHerderShutdown(true); + + PowerMock.replayAll(); + + + startBackgroundHerder(); + + FutureCallback fencing = new FutureCallback<>(); + herder.fenceZombies(CONN1, fencing); + + ExecutionException exception = assertThrows(ExecutionException.class, () -> fencing.get(10, TimeUnit.SECONDS)); + assertEquals(fencingException, exception.getCause()); + + stopBackgroundHerder(); + + PowerMock.verifyAll(); + } + + /** + * The herder tries to perform a round of fencing and is able to retrieve a future from worker::fenceZombies, but the attempt + * fails at a later point. + */ + @Test + public void testExternalZombieFencingRequestAsynchronousFailure() throws Exception { + expectHerderStartup(); + EasyMock.expect(member.memberId()).andStubReturn("leader"); + EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2); + expectConfigRefreshAndSnapshot(SNAPSHOT); + + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + SessionKey sessionKey = expectNewSessionKey(); + + expectAnyTicks(); + + member.wakeup(); + EasyMock.expectLastCall(); + + ClusterConfigState configState = exactlyOnceSnapshot( + sessionKey, + TASK_CONFIGS_MAP, + Collections.singletonMap(CONN1, 2), + Collections.singletonMap(CONN1, 5), + Collections.singleton(CONN1) + ); + expectConfigRefreshAndSnapshot(configState); + + // The future returned by Worker::fenceZombies + KafkaFuture workerFencingFuture = EasyMock.mock(KafkaFuture.class); + // The future tracked by the herder (which tracks the fencing performed by the worker and the possible followup write to the config topic) + KafkaFuture herderFencingFuture = EasyMock.mock(KafkaFuture.class); + // The callbacks that the herder has accrued for outstanding fencing futures + Capture> herderFencingCallbacks = EasyMock.newCapture(CaptureType.ALL); + + EasyMock.expect(worker.fenceZombies(EasyMock.eq(CONN1), EasyMock.eq(2), EasyMock.eq(CONN1_CONFIG))) + .andReturn(workerFencingFuture); + + EasyMock.expect(workerFencingFuture.thenApply(EasyMock.>anyObject())) + .andReturn(herderFencingFuture); + + CountDownLatch callbacksInstalled = new CountDownLatch(2); + for (int i = 0; i < 2; i++) { + EasyMock.expect(herderFencingFuture.whenComplete(EasyMock.capture(herderFencingCallbacks))).andAnswer(() -> { + callbacksInstalled.countDown(); + return null; + }); + } + + expectHerderShutdown(true); + + PowerMock.replayAll(workerFencingFuture, herderFencingFuture); + + + startBackgroundHerder(); + + FutureCallback fencing = new FutureCallback<>(); + herder.fenceZombies(CONN1, fencing); + + assertTrue(callbacksInstalled.await(10, TimeUnit.SECONDS)); + + Exception fencingException = new AuthorizationException("you didn't say the magic word"); + herderFencingCallbacks.getValues().forEach(cb -> cb.accept(null, fencingException)); + + ExecutionException exception = assertThrows(ExecutionException.class, () -> fencing.get(10, TimeUnit.SECONDS)); + assertTrue(exception.getCause() instanceof ConnectException); + + stopBackgroundHerder(); + + PowerMock.verifyAll(); + } + + /** + * Issues multiple rapid fencing requests for a handful of connectors, each of which takes a little while to complete. + * This mimics what might happen when a few connectors are reconfigured in quick succession and each task for the + * connector needs to hit the leader with a fencing request during its preflight check. + */ + @Test + public void testExternalZombieFencingRequestDelayedCompletion() throws Exception { + final String conn3 = "SourceC"; + final Map tasksPerConnector = new HashMap<>(); + tasksPerConnector.put(CONN1, 5); + tasksPerConnector.put(CONN2, 3); + tasksPerConnector.put(conn3, 12); + + expectHerderStartup(); + EasyMock.expect(member.memberId()).andStubReturn("leader"); + EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2); + expectConfigRefreshAndSnapshot(SNAPSHOT); + + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + SessionKey sessionKey = expectNewSessionKey(); + + expectAnyTicks(); + + // We invoke the herder's fenceZombies method repeatedly, which adds a new request to the queue. + // If the queue is empty, the member is woken up; however, if two or more requests are issued in rapid + // succession, the member won't be woken up. We allow the member to be woken up any number of times + // here since it's not critical to the testing logic and it's difficult to mock things in order to lead to an + // exact number of wakeups. + member.wakeup(); + EasyMock.expectLastCall().anyTimes(); + + Map taskCountRecords = new HashMap<>(); + taskCountRecords.put(CONN1, 2); + taskCountRecords.put(CONN2, 3); + taskCountRecords.put(conn3, 5); + Map taskConfigGenerations = new HashMap<>(); + taskConfigGenerations.put(CONN1, 3); + taskConfigGenerations.put(CONN2, 4); + taskConfigGenerations.put(conn3, 2); + Set pendingFencing = new HashSet<>(Arrays.asList(CONN1, CONN2, conn3)); + ClusterConfigState configState = exactlyOnceSnapshot( + sessionKey, + TASK_CONFIGS_MAP, + taskCountRecords, + taskConfigGenerations, + pendingFencing, + tasksPerConnector + ); + tasksPerConnector.keySet().forEach(c -> expectConfigRefreshAndSnapshot(configState)); + + // The callbacks that the herder has accrued for outstanding fencing futures, which will be completed after + // a successful round of fencing and a task record write to the config topic + Map>> herderFencingCallbacks = new HashMap<>(); + // The callbacks that the herder has installed for after a successful round of zombie fencing, but before writing + // a task record to the config topic + Map>> workerFencingFollowups = new HashMap<>(); + + Map callbacksInstalled = new HashMap<>(); + tasksPerConnector.forEach((connector, numStackedRequests) -> { + // The future returned by Worker::fenceZombies + KafkaFuture workerFencingFuture = EasyMock.mock(KafkaFuture.class); + // The future tracked by the herder (which tracks the fencing performed by the worker and the possible followup write to the config topic) + KafkaFuture herderFencingFuture = EasyMock.mock(KafkaFuture.class); + + Capture> herderFencingCallback = EasyMock.newCapture(CaptureType.ALL); + herderFencingCallbacks.put(connector, herderFencingCallback); + + // Don't immediately invoke callbacks that the herder sets up for when the worker fencing and writes to the config topic have completed + // Instead, wait for them to be installed, then invoke them explicitly after the fact on a thread separate from the herder's tick thread + EasyMock.expect(herderFencingFuture.whenComplete(EasyMock.capture(herderFencingCallback))) + .andReturn(null) + .times(numStackedRequests + 1); + + Capture> fencingFollowup = EasyMock.newCapture(); + CountDownLatch callbackInstalled = new CountDownLatch(1); + workerFencingFollowups.put(connector, fencingFollowup); + callbacksInstalled.put(connector, callbackInstalled); + EasyMock.expect(workerFencingFuture.thenApply(EasyMock.capture(fencingFollowup))).andAnswer(() -> { + callbackInstalled.countDown(); + return herderFencingFuture; + }); + + // We should only perform a single physical zombie fencing; all the subsequent requests should be stacked onto the first one + EasyMock.expect(worker.fenceZombies( + EasyMock.eq(connector), EasyMock.eq(taskCountRecords.get(connector)), EasyMock.anyObject()) + ).andReturn(workerFencingFuture); + + for (int i = 0; i < numStackedRequests; i++) { + expectConfigRefreshAndSnapshot(configState); + } + + PowerMock.replay(workerFencingFuture, herderFencingFuture); + }); + + tasksPerConnector.forEach((connector, taskCount) -> { + configBackingStore.putTaskCountRecord(connector, taskCount); + EasyMock.expectLastCall(); + }); + + expectHerderShutdown(false); + + PowerMock.replayAll(); + + + startBackgroundHerder(); + + List> stackedFencingRequests = new ArrayList<>(); + tasksPerConnector.forEach((connector, numStackedRequests) -> { + List> connectorFencingRequests = IntStream.range(0, numStackedRequests) + .mapToObj(i -> new FutureCallback()) + .collect(Collectors.toList()); + + connectorFencingRequests.forEach(fencing -> + herder.fenceZombies(connector, fencing) + ); + + stackedFencingRequests.addAll(connectorFencingRequests); + }); + + callbacksInstalled.forEach((connector, latch) -> { + try { + assertTrue(latch.await(10, TimeUnit.SECONDS)); + workerFencingFollowups.get(connector).getValue().apply(null); + herderFencingCallbacks.get(connector).getValues().forEach(cb -> cb.accept(null, null)); + } catch (InterruptedException e) { + fail("Unexpectedly interrupted"); + } + }); + + for (FutureCallback fencing : stackedFencingRequests) { + fencing.get(10, TimeUnit.SECONDS); + } + + stopBackgroundHerder(); + + PowerMock.verifyAll(); + } + + @Test + public void testVerifyTaskGeneration() throws Exception { + Map taskConfigGenerations = new HashMap<>(); + + ClusterConfigState configState = new ClusterConfigState(1, null, Collections.singletonMap(CONN1, 3), + Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), + TASK_CONFIGS_MAP, Collections.emptyMap(), taskConfigGenerations, Collections.emptySet(), Collections.emptySet()); + + // We'll simulate a successful read to the end 18 times + IntStream.range(0, 18).forEach(i -> EasyMock.expect(configBackingStore.snapshot()).andReturn(configState)); + + Callback verifyCallback = EasyMock.mock(Callback.class); + for (int i = 0; i < 5; i++) { + verifyCallback.onCompletion(null, null); + EasyMock.expectLastCall(); + } + + PowerMock.replayAll(); + + herder.assignment = new ExtendedAssignment( + (short) 2, (short) 0, "leader", "leaderUrl", 0, + Collections.emptySet(), Collections.singleton(TASK1), + Collections.emptySet(), Collections.emptySet(), 0); + + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(TASK1, 0, verifyCallback)); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(TASK1, 1, verifyCallback)); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(TASK1, 2, verifyCallback)); + + taskConfigGenerations.put(CONN1, 0); + herder.verifyTaskGenerationAndOwnership(TASK1, 0, verifyCallback); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(TASK1, 1, verifyCallback)); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(TASK1, 2, verifyCallback)); + + taskConfigGenerations.put(CONN1, 1); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(TASK1, 0, verifyCallback)); + herder.verifyTaskGenerationAndOwnership(TASK1, 1, verifyCallback); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(TASK1, 2, verifyCallback)); + + taskConfigGenerations.put(CONN1, 2); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(TASK1, 0, verifyCallback)); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(TASK1, 1, verifyCallback)); + herder.verifyTaskGenerationAndOwnership(TASK1, 2, verifyCallback); + + taskConfigGenerations.put(CONN1, 3); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(TASK1, 0, verifyCallback)); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(TASK1, 1, verifyCallback)); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(TASK1, 2, verifyCallback)); + + ConnectorTaskId unassignedTask = new ConnectorTaskId(CONN2, 0); + taskConfigGenerations.put(unassignedTask.connector(), 1); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(unassignedTask, 0, verifyCallback)); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(unassignedTask, 1, verifyCallback)); + assertThrows(ConnectException.class, () -> herder.verifyTaskGenerationAndOwnership(unassignedTask, 2, verifyCallback)); + + PowerMock.verifyAll(); + } + @Test public void testKeyExceptionDetection() { assertFalse(herder.isPossibleExpiredKeyException( @@ -2660,16 +3457,22 @@ public void testHerderStopServicesClosesUponShutdown() { private void expectRebalance(final long offset, final List assignedConnectors, final List assignedTasks) { - expectRebalance(Collections.emptyList(), Collections.emptyList(), - ConnectProtocol.Assignment.NO_ERROR, offset, assignedConnectors, assignedTasks, 0); + expectRebalance(offset, assignedConnectors, assignedTasks, false); } private void expectRebalance(final long offset, final List assignedConnectors, final List assignedTasks, - String leader, String leaderUrl) { + final boolean isLeader) { + expectRebalance(Collections.emptyList(), Collections.emptyList(), + ConnectProtocol.Assignment.NO_ERROR, offset, "leader", "leaderUrl", assignedConnectors, assignedTasks, 0, isLeader); + } + + private void expectRebalance(final long offset, + final List assignedConnectors, final List assignedTasks, + String leader, String leaderUrl, boolean isLeader) { expectRebalance(Collections.emptyList(), Collections.emptyList(), - ConnectProtocol.Assignment.NO_ERROR, offset, leader, leaderUrl, assignedConnectors, assignedTasks, 0); + ConnectProtocol.Assignment.NO_ERROR, offset, leader, leaderUrl, assignedConnectors, assignedTasks, 0, isLeader); } // Handles common initial part of rebalance callback. Does not handle instantiation of connectors and tasks. @@ -2690,7 +3493,7 @@ private void expectRebalance(final Collection revokedConnectors, final List assignedConnectors, final List assignedTasks, int delay) { - expectRebalance(revokedConnectors, revokedTasks, error, offset, "leader", "leaderUrl", assignedConnectors, assignedTasks, delay); + expectRebalance(revokedConnectors, revokedTasks, error, offset, "leader", "leaderUrl", assignedConnectors, assignedTasks, delay, false); } // Handles common initial part of rebalance callback. Does not handle instantiation of connectors and tasks. @@ -2702,7 +3505,8 @@ private void expectRebalance(final Collection revokedConnectors, String leaderUrl, final List assignedConnectors, final List assignedTasks, - int delay) { + int delay, + boolean isLeader) { member.ensureActive(); PowerMock.expectLastCall().andAnswer(() -> { ExtendedAssignment assignment; @@ -2725,6 +3529,10 @@ private void expectRebalance(final Collection revokedConnectors, time.sleep(100L); return null; }); + if (isLeader) { + configBackingStore.claimWritePrivileges(); + EasyMock.expectLastCall(); + } if (!revokedConnectors.isEmpty()) { for (String connector : revokedConnectors) { @@ -2747,10 +3555,111 @@ private void expectRebalance(final Collection revokedConnectors, PowerMock.expectLastCall(); } - private void expectPostRebalanceCatchup(final ClusterConfigState readToEndSnapshot) throws TimeoutException { - configBackingStore.refresh(EasyMock.anyLong(), EasyMock.anyObject(TimeUnit.class)); + private ClusterConfigState exactlyOnceSnapshot( + SessionKey sessionKey, + Map> taskConfigs, + Map taskCountRecords, + Map taskConfigGenerations, + Set pendingFencing) { + + Set connectors = new HashSet<>(); + connectors.addAll(taskCountRecords.keySet()); + connectors.addAll(taskConfigGenerations.keySet()); + connectors.addAll(pendingFencing); + Map taskCounts = connectors.stream() + .collect(Collectors.toMap(Function.identity(), c -> 1)); + + return exactlyOnceSnapshot(sessionKey, taskConfigs, taskCountRecords, taskConfigGenerations, pendingFencing, taskCounts); + } + + private ClusterConfigState exactlyOnceSnapshot( + SessionKey sessionKey, + Map> taskConfigs, + Map taskCountRecords, + Map taskConfigGenerations, + Set pendingFencing, + Map taskCounts) { + + Set connectors = new HashSet<>(); + connectors.addAll(taskCounts.keySet()); + connectors.addAll(taskCountRecords.keySet()); + connectors.addAll(taskConfigGenerations.keySet()); + connectors.addAll(pendingFencing); + + Map> connectorConfigs = connectors.stream() + .collect(Collectors.toMap(Function.identity(), c -> CONN1_CONFIG)); + + return new ClusterConfigState(1, sessionKey, taskCounts, + connectorConfigs, Collections.singletonMap(CONN1, TargetState.STARTED), + taskConfigs, taskCountRecords, taskConfigGenerations, pendingFencing, Collections.emptySet()); + } + + private void expectAnyTicks() { + member.ensureActive(); + EasyMock.expectLastCall().anyTimes(); + member.poll(EasyMock.anyInt()); + PowerMock.expectLastCall().anyTimes(); + } + + private SessionKey expectNewSessionKey() { + SecretKey secretKey = EasyMock.niceMock(SecretKey.class); + EasyMock.expect(secretKey.getAlgorithm()).andReturn(INTER_WORKER_KEY_GENERATION_ALGORITHM_DEFAULT).anyTimes(); + EasyMock.expect(secretKey.getEncoded()).andReturn(new byte[32]).anyTimes(); + SessionKey sessionKey = new SessionKey(secretKey, time.milliseconds() + TimeUnit.DAYS.toMillis(1)); + configBackingStore.putSessionKey(anyObject(SessionKey.class)); + EasyMock.expectLastCall().andAnswer(() -> { + configUpdateListener.onSessionKeyUpdate(sessionKey); + return null; + }); + EasyMock.replay(secretKey); + return sessionKey; + } + + private void expectConfigRefreshAndSnapshot(final ClusterConfigState readToEndSnapshot) { + try { + configBackingStore.refresh(EasyMock.anyLong(), EasyMock.anyObject(TimeUnit.class)); + EasyMock.expectLastCall(); + EasyMock.expect(configBackingStore.snapshot()).andReturn(readToEndSnapshot); + } catch (TimeoutException e) { + fail("Mocked method should not throw checked exception"); + } + } + + private void startBackgroundHerder() { + herderExecutor = Executors.newSingleThreadExecutor(); + herderExecutor.submit(herder); + } + + private void stopBackgroundHerder() throws Exception { + herder.stop(); + herderExecutor.shutdown(); + herderExecutor.awaitTermination(10, TimeUnit.SECONDS); + } + + private void expectHerderStartup() { + worker.start(); + EasyMock.expectLastCall(); + statusBackingStore.start(); + EasyMock.expectLastCall(); + configBackingStore.start(); + EasyMock.expectLastCall(); + } + + private void expectHerderShutdown(boolean wakeup) { + if (wakeup) { + member.wakeup(); + EasyMock.expectLastCall(); + } + EasyMock.expect(worker.connectorNames()).andReturn(Collections.emptySet()); + EasyMock.expect(worker.taskIds()).andReturn(Collections.emptySet()); + member.stop(); + EasyMock.expectLastCall(); + statusBackingStore.stop(); + EasyMock.expectLastCall(); + configBackingStore.stop(); + EasyMock.expectLastCall(); + worker.stop(); EasyMock.expectLastCall(); - EasyMock.expect(configBackingStore.snapshot()).andReturn(readToEndSnapshot); } private void assertStatistics(int expectedEpoch, int completedRebalances, double rebalanceTime, double millisSinceLastRebalance) { @@ -2851,4 +3760,14 @@ private abstract class BogusSourceConnector extends SourceConnector { private abstract class BogusSourceTask extends SourceTask { } + private DistributedHerder exactlyOnceHerder() { + Map config = new HashMap<>(HERDER_CONFIG); + config.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "enabled"); + return PowerMock.createPartialMock(DistributedHerder.class, + new String[]{"connectorTypeForClass", "updateDeletedConnectorStatus", "updateDeletedTaskStatus", "validateConnectorConfig"}, + new DistributedConfig(config), worker, WORKER_ID, KAFKA_CLUSTER_ID, + statusBackingStore, configBackingStore, member, MEMBER_URL, metrics, time, noneConnectorClientConfigOverridePolicy, + new AutoCloseable[0]); + } + } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index 0fe153132eb93..92c3db960de2f 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.utils.Time; import org.apache.kafka.connect.runtime.TargetState; import org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.ConnectorsAndTasks; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.util.ConnectorTaskId; import org.junit.After; import org.junit.Before; @@ -1282,7 +1283,7 @@ private static ClusterConfigState clusterConfigState(long offset, connectorConfigs(connectorStart, connectorNumEnd), connectorTargetStates(connectorStart, connectorNumEnd, TargetState.STARTED), taskConfigs(0, connectorNum, connectorNum * taskNum), - Collections.emptySet()); + Collections.emptyMap(), Collections.emptyMap(), Collections.emptySet(), Collections.emptySet()); } private static Map memberConfigs(String givenLeader, diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java index 35ba6249d7455..3b2195fb23d6b 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java @@ -26,6 +26,7 @@ import org.apache.kafka.common.requests.RequestTestUtils; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.storage.KafkaConfigBackingStore; import org.apache.kafka.connect.util.ConnectorTaskId; import org.junit.After; diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java index d4ae2b463ea1a..11186a3293cb6 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java @@ -36,6 +36,7 @@ import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.connect.runtime.TargetState; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.storage.KafkaConfigBackingStore; import org.apache.kafka.connect.util.ConnectorTaskId; import org.easymock.EasyMock; @@ -156,6 +157,9 @@ public void setup() { Collections.singletonMap(connectorId1, new HashMap()), Collections.singletonMap(connectorId1, TargetState.STARTED), Collections.singletonMap(taskId1x0, new HashMap()), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptySet(), Collections.emptySet() ); @@ -179,6 +183,9 @@ public void setup() { configState2ConnectorConfigs, configState2TargetStates, configState2TaskConfigs, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptySet(), Collections.emptySet() ); @@ -205,6 +212,9 @@ public void setup() { configStateSingleTaskConnectorsConnectorConfigs, configStateSingleTaskConnectorsTargetStates, configStateSingleTaskConnectorsTaskConfigs, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptySet(), Collections.emptySet() ); } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java index 3a419b8c5ca27..c63e4f9a6abc3 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java @@ -786,8 +786,6 @@ public void testRestartConnectorAndTasksConnectorNotFound() { assertThrows(NotFoundException.class, () -> connectorsResource.restartConnector(CONNECTOR_NAME, NULL_HEADERS, restartRequest.includeTasks(), restartRequest.onlyFailed(), FORWARD) ); - - PowerMock.verifyAll(); } @Test @@ -848,6 +846,70 @@ public void testRestartConnectorAndTasksRequestAccepted() throws Throwable { PowerMock.verifyAll(); } + @Test + public void testFenceZombiesNoInternalRequestSignature() throws Throwable { + final Capture> cb = Capture.newInstance(); + herder.fenceZombies(EasyMock.eq(CONNECTOR_NAME), EasyMock.capture(cb), EasyMock.anyObject(InternalRequestSignature.class)); + expectAndCallbackResult(cb, null); + + PowerMock.replayAll(); + + connectorsResource.fenceZombies(CONNECTOR_NAME, NULL_HEADERS, FORWARD, serializeAsBytes(null)); + + PowerMock.verifyAll(); + } + + @Test + public void testFenceZombiesWithInternalRequestSignature() throws Throwable { + final String signatureAlgorithm = "HmacSHA256"; + final String encodedSignature = "Kv1/OSsxzdVIwvZ4e30avyRIVrngDfhzVUm/kAZEKc4="; + + final Capture> cb = Capture.newInstance(); + final Capture signatureCapture = Capture.newInstance(); + herder.fenceZombies(EasyMock.eq(CONNECTOR_NAME), EasyMock.capture(cb), EasyMock.capture(signatureCapture)); + expectAndCallbackResult(cb, null); + + HttpHeaders headers = EasyMock.mock(HttpHeaders.class); + EasyMock.expect(headers.getHeaderString(InternalRequestSignature.SIGNATURE_ALGORITHM_HEADER)) + .andReturn(signatureAlgorithm) + .once(); + EasyMock.expect(headers.getHeaderString(InternalRequestSignature.SIGNATURE_HEADER)) + .andReturn(encodedSignature) + .once(); + + PowerMock.replayAll(headers); + + connectorsResource.fenceZombies(CONNECTOR_NAME, headers, FORWARD, serializeAsBytes(null)); + + PowerMock.verifyAll(); + InternalRequestSignature expectedSignature = new InternalRequestSignature( + serializeAsBytes(null), + Mac.getInstance(signatureAlgorithm), + Base64.getDecoder().decode(encodedSignature) + ); + assertEquals( + expectedSignature, + signatureCapture.getValue() + ); + + PowerMock.verifyAll(); + } + + @Test + public void testFenceZombiesConnectorNotFound() throws Throwable { + final Capture> cb = Capture.newInstance(); + herder.fenceZombies(EasyMock.eq(CONNECTOR_NAME), EasyMock.capture(cb), EasyMock.anyObject(InternalRequestSignature.class)); + + expectAndCallbackException(cb, new NotFoundException("not found")); + + PowerMock.replayAll(); + + assertThrows(NotFoundException.class, + () -> connectorsResource.fenceZombies(CONNECTOR_NAME, NULL_HEADERS, FORWARD, serializeAsBytes(null))); + + PowerMock.verifyAll(); + } + @Test public void testRestartConnectorNotFound() { final Capture> cb = Capture.newInstance(); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java index 2504d98de68f4..88ec7d41ebb1e 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java @@ -41,7 +41,6 @@ import org.apache.kafka.connect.runtime.Worker; import org.apache.kafka.connect.runtime.WorkerConfigTransformer; import org.apache.kafka.connect.runtime.WorkerConnector; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; import org.apache.kafka.connect.runtime.isolation.DelegatingClassLoader; import org.apache.kafka.connect.runtime.isolation.PluginClassLoader; import org.apache.kafka.connect.runtime.isolation.Plugins; @@ -54,6 +53,7 @@ import org.apache.kafka.connect.sink.SinkTask; import org.apache.kafka.connect.source.SourceConnector; import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.storage.MemoryConfigBackingStore; import org.apache.kafka.connect.storage.StatusBackingStore; import org.apache.kafka.connect.util.Callback; @@ -364,9 +364,12 @@ public void testRestartTask() throws Exception { Collections.singletonMap(CONNECTOR_NAME, connectorConfig), Collections.singletonMap(CONNECTOR_NAME, TargetState.STARTED), Collections.singletonMap(taskId, taskConfig(SourceSink.SOURCE)), + Collections.emptyMap(), + Collections.emptyMap(), + new HashSet<>(), new HashSet<>(), transformer); - worker.startTask(taskId, configState, connectorConfig, taskConfig(SourceSink.SOURCE), herder, TargetState.STARTED); + worker.startSourceTask(taskId, configState, connectorConfig, taskConfig(SourceSink.SOURCE), herder, TargetState.STARTED); EasyMock.expectLastCall().andReturn(true); PowerMock.replayAll(); @@ -402,9 +405,12 @@ public void testRestartTaskFailureOnStart() throws Exception { Collections.singletonMap(CONNECTOR_NAME, connectorConfig), Collections.singletonMap(CONNECTOR_NAME, TargetState.STARTED), Collections.singletonMap(new ConnectorTaskId(CONNECTOR_NAME, 0), taskConfig(SourceSink.SOURCE)), + Collections.emptyMap(), + Collections.emptyMap(), + new HashSet<>(), new HashSet<>(), transformer); - worker.startTask(taskId, configState, connectorConfig, taskConfig(SourceSink.SOURCE), herder, TargetState.STARTED); + worker.startSourceTask(taskId, configState, connectorConfig, taskConfig(SourceSink.SOURCE), herder, TargetState.STARTED); EasyMock.expectLastCall().andReturn(false); PowerMock.replayAll(); @@ -572,9 +578,12 @@ public void testRestartConnectorAndTasksOnlyTasks() throws Exception { Collections.singletonMap(CONNECTOR_NAME, connectorConfig), Collections.singletonMap(CONNECTOR_NAME, TargetState.STARTED), Collections.singletonMap(taskId, taskConfig(SourceSink.SINK)), + Collections.emptyMap(), + Collections.emptyMap(), + new HashSet<>(), new HashSet<>(), transformer); - worker.startTask(taskId, configState, connectorConfig, taskConfig(SourceSink.SINK), herder, TargetState.STARTED); + worker.startSinkTask(taskId, configState, connectorConfig, taskConfig(SourceSink.SINK), herder, TargetState.STARTED); EasyMock.expectLastCall().andReturn(true); PowerMock.replayAll(); @@ -635,9 +644,12 @@ public void testRestartConnectorAndTasksBoth() throws Exception { Collections.singletonMap(CONNECTOR_NAME, connectorConfig), Collections.singletonMap(CONNECTOR_NAME, TargetState.STARTED), Collections.singletonMap(taskId, taskConfig(SourceSink.SINK)), + Collections.emptyMap(), + Collections.emptyMap(), + new HashSet<>(), new HashSet<>(), transformer); - worker.startTask(taskId, configState, connectorConfig, taskConfig(SourceSink.SINK), herder, TargetState.STARTED); + worker.startSinkTask(taskId, configState, connectorConfig, taskConfig(SourceSink.SINK), herder, TargetState.STARTED); EasyMock.expectLastCall().andReturn(true); PowerMock.replayAll(); @@ -878,7 +890,6 @@ private void expectAdd(SourceSink sourceSink) { Capture> onStart = EasyMock.newCapture(); worker.startConnector(EasyMock.eq(CONNECTOR_NAME), EasyMock.eq(connectorProps), EasyMock.anyObject(HerderConnectorContext.class), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED), EasyMock.capture(onStart)); - // EasyMock.expectLastCall().andReturn(true); EasyMock.expectLastCall().andAnswer(() -> { onStart.getValue().onCompletion(null, TargetState.STARTED); return true; @@ -902,9 +913,16 @@ private void expectAdd(SourceSink sourceSink) { Collections.singletonMap(CONNECTOR_NAME, connectorConfig(sourceSink)), Collections.singletonMap(CONNECTOR_NAME, TargetState.STARTED), Collections.singletonMap(new ConnectorTaskId(CONNECTOR_NAME, 0), generatedTaskProps), + Collections.emptyMap(), + Collections.emptyMap(), + new HashSet<>(), new HashSet<>(), transformer); - worker.startTask(new ConnectorTaskId(CONNECTOR_NAME, 0), configState, connectorConfig(sourceSink), generatedTaskProps, herder, TargetState.STARTED); + if (sourceSink.equals(SourceSink.SOURCE)) { + worker.startSourceTask(new ConnectorTaskId(CONNECTOR_NAME, 0), configState, connectorConfig(sourceSink), generatedTaskProps, herder, TargetState.STARTED); + } else { + worker.startSinkTask(new ConnectorTaskId(CONNECTOR_NAME, 0), configState, connectorConfig(sourceSink), generatedTaskProps, herder, TargetState.STARTED); + } EasyMock.expectLastCall().andReturn(true); EasyMock.expect(herder.connectorTypeForClass(BogusSourceConnector.class.getName())) diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java index 18d92bf16bad7..609061ec87758 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java @@ -20,7 +20,10 @@ import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.IsolationLevel; +import org.apache.kafka.common.errors.ProducerFencedException; import org.apache.kafka.common.header.internals.RecordHeaders; import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.common.config.ConfigException; @@ -30,7 +33,6 @@ import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.runtime.RestartRequest; import org.apache.kafka.connect.runtime.TargetState; -import org.apache.kafka.connect.runtime.distributed.ClusterConfigState; import org.apache.kafka.connect.runtime.distributed.DistributedConfig; import org.apache.kafka.connect.util.Callback; import org.apache.kafka.connect.util.ConnectUtils; @@ -50,22 +52,30 @@ import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.reflect.Whitebox; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import static org.apache.kafka.clients.consumer.ConsumerConfig.ISOLATION_LEVEL_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.TRANSACTIONAL_ID_CONFIG; +import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.GROUP_ID_CONFIG; import static org.apache.kafka.connect.storage.KafkaConfigBackingStore.INCLUDE_TASKS_FIELD_NAME; import static org.apache.kafka.connect.storage.KafkaConfigBackingStore.ONLY_FAILED_FIELD_NAME; import static org.apache.kafka.connect.storage.KafkaConfigBackingStore.RESTART_KEY; +import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -96,6 +106,7 @@ public class KafkaConfigBackingStoreTest { private static final List CONNECTOR_CONFIG_KEYS = Arrays.asList("connector-connector1", "connector-connector2"); private static final List COMMIT_TASKS_CONFIG_KEYS = Arrays.asList("commit-connector1", "commit-connector2"); private static final List TARGET_STATE_KEYS = Arrays.asList("target-state-connector1", "target-state-connector2"); + private static final List CONNECTOR_TASK_COUNT_RECORD_KEYS = Arrays.asList("tasks-count-connector1", "tasks-count-connector2"); private static final String CONNECTOR_1_NAME = "connector1"; private static final String CONNECTOR_2_NAME = "connector2"; @@ -124,6 +135,10 @@ public class KafkaConfigBackingStoreTest { new Struct(KafkaConfigBackingStore.TASK_CONFIGURATION_V0).put("properties", SAMPLE_CONFIGS.get(0)), new Struct(KafkaConfigBackingStore.TASK_CONFIGURATION_V0).put("properties", SAMPLE_CONFIGS.get(1)) ); + private static final List CONNECTOR_TASK_COUNT_RECORD_STRUCTS = Arrays.asList( + new Struct(KafkaConfigBackingStore.TASK_COUNT_RECORD_V0).put("tasks", 6), + new Struct(KafkaConfigBackingStore.TASK_COUNT_RECORD_V0).put("tasks", 9) + ); private static final Struct TARGET_STATE_PAUSED = new Struct(KafkaConfigBackingStore.TARGET_STATE_V0).put("state", "PAUSED"); private static final Struct TASKS_COMMIT_STRUCT_TWO_TASK_CONNECTOR @@ -152,6 +167,8 @@ public class KafkaConfigBackingStoreTest { private ConfigBackingStore.UpdateListener configUpdateListener; @Mock KafkaBasedLog storeLog; + @Mock + Producer fencableProducer; private KafkaConfigBackingStore configStorage; private Capture capturedTopic = EasyMock.newCapture(); @@ -163,15 +180,22 @@ public class KafkaConfigBackingStoreTest { private long logOffset = 0; + private void createStore(DistributedConfig config, KafkaBasedLog storeLog) { + configStorage = PowerMock.createPartialMock( + KafkaConfigBackingStore.class, + new String[]{"createKafkaBasedLog", "createFencableProducer"}, + converter, config, null); + Whitebox.setInternalState(configStorage, "configLog", storeLog); + configStorage.setUpdateListener(configUpdateListener); + } + @Before public void setUp() { PowerMock.mockStaticPartial(ConnectUtils.class, "lookupKafkaClusterId"); EasyMock.expect(ConnectUtils.lookupKafkaClusterId(EasyMock.anyObject())).andReturn("test-cluster").anyTimes(); PowerMock.replay(ConnectUtils.class); - configStorage = PowerMock.createPartialMock(KafkaConfigBackingStore.class, new String[]{"createKafkaBasedLog"}, converter, DEFAULT_DISTRIBUTED_CONFIG, null); - Whitebox.setInternalState(configStorage, "configLog", storeLog); - configStorage.setUpdateListener(configUpdateListener); + createStore(DEFAULT_DISTRIBUTED_CONFIG, storeLog); } @Test @@ -204,6 +228,32 @@ public void testStartStop() throws Exception { PowerMock.verifyAll(); } + @Test + public void testSnapshotCannotMutateInternalState() throws Exception { + expectConfigure(); + expectStart(Collections.emptyList(), Collections.emptyMap()); + expectPartitionCount(1); + PowerMock.replayAll(); + + Map settings = new HashMap<>(DEFAULT_CONFIG_STORAGE_PROPS); + settings.put("config.storage.min.insync.replicas", "3"); + settings.put("config.storage.max.message.bytes", "1001"); + configStorage.setupAndCreateKafkaBasedLog(TOPIC, new DistributedConfig(settings)); + + configStorage.start(); + ClusterConfigState snapshot = configStorage.snapshot(); + assertNotSame(snapshot.connectorTaskCounts, configStorage.connectorTaskCounts); + assertNotSame(snapshot.connectorConfigs, configStorage.connectorConfigs); + assertNotSame(snapshot.connectorTargetStates, configStorage.connectorTargetStates); + assertNotSame(snapshot.taskConfigs, configStorage.taskConfigs); + assertNotSame(snapshot.connectorTaskCountRecords, configStorage.connectorTaskCountRecords); + assertNotSame(snapshot.connectorTaskConfigGenerations, configStorage.connectorTaskConfigGenerations); + assertNotSame(snapshot.connectorsPendingFencing, configStorage.connectorsPendingFencing); + assertNotSame(snapshot.inconsistentConnectors, configStorage.inconsistent); + + PowerMock.verifyAll(); + } + @Test public void testPutConnectorConfig() throws Exception { expectConfigure(); @@ -267,6 +317,172 @@ public void testPutConnectorConfig() throws Exception { PowerMock.verifyAll(); } + @Test + public void testWritePrivileges() throws Exception { + // With exactly.once.source.support = preparing (or also, "enabled"), we need to use a transactional producer + // to write some types of messages to the config topic + Map workerProps = new HashMap<>(DEFAULT_CONFIG_STORAGE_PROPS); + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "preparing"); + DistributedConfig config = new DistributedConfig(workerProps); + createStore(config, storeLog); + + expectConfigure(); + expectStart(Collections.emptyList(), Collections.emptyMap()); + + // Try and fail to write a task count record to the config topic without write privileges + expectConvert(KafkaConfigBackingStore.TASK_COUNT_RECORD_V0, CONNECTOR_TASK_COUNT_RECORD_STRUCTS.get(0), CONFIGS_SERIALIZED.get(0)); + // Claim write privileges + expectFencableProducer(); + // And write the task count record successfully + expectConvert(KafkaConfigBackingStore.TASK_COUNT_RECORD_V0, CONNECTOR_TASK_COUNT_RECORD_STRUCTS.get(0), CONFIGS_SERIALIZED.get(0)); + fencableProducer.beginTransaction(); + EasyMock.expectLastCall(); + EasyMock.expect(fencableProducer.send(EasyMock.anyObject())).andReturn(null); + fencableProducer.commitTransaction(); + EasyMock.expectLastCall(); + expectRead(CONNECTOR_TASK_COUNT_RECORD_KEYS.get(0), CONFIGS_SERIALIZED.get(0), CONNECTOR_TASK_COUNT_RECORD_STRUCTS.get(0)); + + // Try to write a connector config + expectConvert(KafkaConfigBackingStore.CONNECTOR_CONFIGURATION_V0, CONNECTOR_CONFIG_STRUCTS.get(0), CONFIGS_SERIALIZED.get(1)); + fencableProducer.beginTransaction(); + EasyMock.expectLastCall(); + EasyMock.expect(fencableProducer.send(EasyMock.anyObject())).andReturn(null); + // Get fenced out + fencableProducer.commitTransaction(); + EasyMock.expectLastCall().andThrow(new ProducerFencedException("Better luck next time")); + // And fail when trying to write again without reclaiming write privileges + expectConvert(KafkaConfigBackingStore.CONNECTOR_CONFIGURATION_V0, CONNECTOR_CONFIG_STRUCTS.get(0), CONFIGS_SERIALIZED.get(1)); + + // In the meantime, write a target state (which doesn't require write privileges) + expectConvert(KafkaConfigBackingStore.TARGET_STATE_V0, TARGET_STATE_PAUSED, CONFIGS_SERIALIZED.get(1)); + storeLog.send("target-state-" + CONNECTOR_IDS.get(1), CONFIGS_SERIALIZED.get(1)); + PowerMock.expectLastCall(); + + // Reclaim write privileges + expectFencableProducer(); + // And successfully write the config + expectConvert(KafkaConfigBackingStore.CONNECTOR_CONFIGURATION_V0, CONNECTOR_CONFIG_STRUCTS.get(0), CONFIGS_SERIALIZED.get(1)); + fencableProducer.beginTransaction(); + EasyMock.expectLastCall(); + EasyMock.expect(fencableProducer.send(EasyMock.anyObject())).andReturn(null); + fencableProducer.commitTransaction(); + EasyMock.expectLastCall(); + expectConvertRead(CONNECTOR_CONFIG_KEYS.get(1), CONNECTOR_CONFIG_STRUCTS.get(0), CONFIGS_SERIALIZED.get(2)); + configUpdateListener.onConnectorConfigUpdate(CONNECTOR_IDS.get(1)); + EasyMock.expectLastCall(); + + expectPartitionCount(1); + expectStop(); + fencableProducer.close(Duration.ZERO); + EasyMock.expectLastCall(); + + PowerMock.replayAll(); + + + configStorage.setupAndCreateKafkaBasedLog(TOPIC, DEFAULT_DISTRIBUTED_CONFIG); + configStorage.start(); + + // Should fail the first time since we haven't claimed write privileges + assertThrows(IllegalStateException.class, () -> configStorage.putTaskCountRecord(CONNECTOR_IDS.get(0), 6)); + // Should succeed now + configStorage.claimWritePrivileges(); + configStorage.putTaskCountRecord(CONNECTOR_IDS.get(0), 6); + + // Should fail again when we get fenced out + assertThrows(ProducerFencedException.class, () -> configStorage.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(0))); + // Should fail if we retry without reclaiming write privileges + assertThrows(IllegalStateException.class, () -> configStorage.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(0))); + + // Should succeed even without write privileges (target states can be written by anyone) + configStorage.putTargetState(CONNECTOR_IDS.get(1), TargetState.PAUSED); + + // Should succeed if we re-claim write privileges + configStorage.claimWritePrivileges(); + configStorage.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(0)); + + configStorage.stop(); + + PowerMock.verifyAll(); + } + + @Test + public void testTaskCountRecordsAndGenerations() throws Exception { + expectConfigure(); + expectStart(Collections.emptyList(), Collections.emptyMap()); + + // Task configs should read to end, write to the log, read to end, write root, then read to end again + expectReadToEnd(new LinkedHashMap<>()); + expectConvertWriteRead( + TASK_CONFIG_KEYS.get(0), KafkaConfigBackingStore.TASK_CONFIGURATION_V0, CONFIGS_SERIALIZED.get(0), + "properties", SAMPLE_CONFIGS.get(0)); + expectConvertWriteRead( + TASK_CONFIG_KEYS.get(1), KafkaConfigBackingStore.TASK_CONFIGURATION_V0, CONFIGS_SERIALIZED.get(1), + "properties", SAMPLE_CONFIGS.get(1)); + expectReadToEnd(new LinkedHashMap<>()); + expectConvertWriteRead( + COMMIT_TASKS_CONFIG_KEYS.get(0), KafkaConfigBackingStore.CONNECTOR_TASKS_COMMIT_V0, CONFIGS_SERIALIZED.get(2), + "tasks", 2); // Starts with 0 tasks, after update has 2 + // As soon as root is rewritten, we should see a callback notifying us that we reconfigured some tasks + configUpdateListener.onTaskConfigUpdate(Arrays.asList(TASK_IDS.get(0), TASK_IDS.get(1))); + EasyMock.expectLastCall(); + + // Records to be read by consumer as it reads to the end of the log + LinkedHashMap serializedConfigs = new LinkedHashMap<>(); + serializedConfigs.put(TASK_CONFIG_KEYS.get(0), CONFIGS_SERIALIZED.get(0)); + serializedConfigs.put(TASK_CONFIG_KEYS.get(1), CONFIGS_SERIALIZED.get(1)); + serializedConfigs.put(COMMIT_TASKS_CONFIG_KEYS.get(0), CONFIGS_SERIALIZED.get(2)); + expectReadToEnd(serializedConfigs); + + // Task count records are read back after writing as well + expectConvertWriteRead( + CONNECTOR_TASK_COUNT_RECORD_KEYS.get(0), KafkaConfigBackingStore.TASK_COUNT_RECORD_V0, CONFIGS_SERIALIZED.get(3), + "tasks", 4); + serializedConfigs = new LinkedHashMap<>(); + serializedConfigs.put(CONNECTOR_TASK_COUNT_RECORD_KEYS.get(0), CONFIGS_SERIALIZED.get(3)); + expectReadToEnd(serializedConfigs); + + expectPartitionCount(1); + expectStop(); + + PowerMock.replayAll(); + + configStorage.setupAndCreateKafkaBasedLog(TOPIC, DEFAULT_DISTRIBUTED_CONFIG); + configStorage.start(); + + // Bootstrap as if we had already added the connector, but no tasks had been added yet + whiteboxAddConnector(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0), Collections.emptyList()); + + // Before anything is written + String connectorName = CONNECTOR_IDS.get(0); + ClusterConfigState configState = configStorage.snapshot(); + assertFalse(configState.pendingFencing(connectorName)); + assertNull(configState.taskCountRecord(connectorName)); + assertNull(configState.taskConfigGeneration(connectorName)); + + // Writing task configs should block until all the writes have been performed and the root record update + // has completed + List> taskConfigs = Arrays.asList(SAMPLE_CONFIGS.get(0), SAMPLE_CONFIGS.get(1)); + configStorage.putTaskConfigs("connector1", taskConfigs); + + configState = configStorage.snapshot(); + assertEquals(3, configState.offset()); + assertTrue(configState.pendingFencing(connectorName)); + assertNull(configState.taskCountRecord(connectorName)); + assertEquals(0, (long) configState.taskConfigGeneration(connectorName)); + + configStorage.putTaskCountRecord(connectorName, 4); + + configState = configStorage.snapshot(); + assertEquals(4, configState.offset()); + assertFalse(configState.pendingFencing(connectorName)); + assertEquals(4, (long) configState.taskCountRecord(connectorName)); + assertEquals(0, (long) configState.taskConfigGeneration(connectorName)); + + configStorage.stop(); + + PowerMock.verifyAll(); + } + @Test public void testPutTaskConfigs() throws Exception { expectConfigure(); @@ -684,30 +900,36 @@ public void testRestore() throws Exception { expectConfigure(); // Overwrite each type at least once to ensure we see the latest data after loading List> existingRecords = Arrays.asList( - new ConsumerRecord<>(TOPIC, 0, 0, 0L, TimestampType.CREATE_TIME, 0, 0, CONNECTOR_CONFIG_KEYS.get(0), + new ConsumerRecord<>(TOPIC, 0, 0, 0L, TimestampType.CREATE_TIME, 0, 0, CONNECTOR_TASK_COUNT_RECORD_KEYS.get(0), CONFIGS_SERIALIZED.get(0), new RecordHeaders(), Optional.empty()), - new ConsumerRecord<>(TOPIC, 0, 1, 0L, TimestampType.CREATE_TIME, 0, 0, TASK_CONFIG_KEYS.get(0), + new ConsumerRecord<>(TOPIC, 0, 1, 0L, TimestampType.CREATE_TIME, 0, 0, CONNECTOR_CONFIG_KEYS.get(0), CONFIGS_SERIALIZED.get(1), new RecordHeaders(), Optional.empty()), - new ConsumerRecord<>(TOPIC, 0, 2, 0L, TimestampType.CREATE_TIME, 0, 0, TASK_CONFIG_KEYS.get(1), + new ConsumerRecord<>(TOPIC, 0, 2, 0L, TimestampType.CREATE_TIME, 0, 0, TASK_CONFIG_KEYS.get(0), CONFIGS_SERIALIZED.get(2), new RecordHeaders(), Optional.empty()), - new ConsumerRecord<>(TOPIC, 0, 3, 0L, TimestampType.CREATE_TIME, 0, 0, CONNECTOR_CONFIG_KEYS.get(0), + new ConsumerRecord<>(TOPIC, 0, 3, 0L, TimestampType.CREATE_TIME, 0, 0, TASK_CONFIG_KEYS.get(1), CONFIGS_SERIALIZED.get(3), new RecordHeaders(), Optional.empty()), - new ConsumerRecord<>(TOPIC, 0, 4, 0L, TimestampType.CREATE_TIME, 0, 0, COMMIT_TASKS_CONFIG_KEYS.get(0), + new ConsumerRecord<>(TOPIC, 0, 4, 0L, TimestampType.CREATE_TIME, 0, 0, CONNECTOR_CONFIG_KEYS.get(0), CONFIGS_SERIALIZED.get(4), new RecordHeaders(), Optional.empty()), - // Connector after root update should make it through, task update shouldn't - new ConsumerRecord<>(TOPIC, 0, 5, 0L, TimestampType.CREATE_TIME, 0, 0, CONNECTOR_CONFIG_KEYS.get(0), + new ConsumerRecord<>(TOPIC, 0, 5, 0L, TimestampType.CREATE_TIME, 0, 0, COMMIT_TASKS_CONFIG_KEYS.get(0), CONFIGS_SERIALIZED.get(5), new RecordHeaders(), Optional.empty()), - new ConsumerRecord<>(TOPIC, 0, 6, 0L, TimestampType.CREATE_TIME, 0, 0, TASK_CONFIG_KEYS.get(0), - CONFIGS_SERIALIZED.get(6), new RecordHeaders(), Optional.empty())); + new ConsumerRecord<>(TOPIC, 0, 6, 0L, TimestampType.CREATE_TIME, 0, 0, CONNECTOR_TASK_COUNT_RECORD_KEYS.get(1), + CONFIGS_SERIALIZED.get(6), new RecordHeaders(), Optional.empty()), + // Connector after root update should make it through, task update shouldn't + new ConsumerRecord<>(TOPIC, 0, 7, 0L, TimestampType.CREATE_TIME, 0, 0, CONNECTOR_CONFIG_KEYS.get(0), + CONFIGS_SERIALIZED.get(7), new RecordHeaders(), Optional.empty()), + new ConsumerRecord<>(TOPIC, 0, 8, 0L, TimestampType.CREATE_TIME, 0, 0, TASK_CONFIG_KEYS.get(0), + CONFIGS_SERIALIZED.get(8), new RecordHeaders(), Optional.empty())); LinkedHashMap deserialized = new LinkedHashMap<>(); - deserialized.put(CONFIGS_SERIALIZED.get(0), CONNECTOR_CONFIG_STRUCTS.get(0)); - deserialized.put(CONFIGS_SERIALIZED.get(1), TASK_CONFIG_STRUCTS.get(0)); + deserialized.put(CONFIGS_SERIALIZED.get(0), CONNECTOR_TASK_COUNT_RECORD_STRUCTS.get(0)); + deserialized.put(CONFIGS_SERIALIZED.get(1), CONNECTOR_CONFIG_STRUCTS.get(0)); deserialized.put(CONFIGS_SERIALIZED.get(2), TASK_CONFIG_STRUCTS.get(0)); - deserialized.put(CONFIGS_SERIALIZED.get(3), CONNECTOR_CONFIG_STRUCTS.get(1)); - deserialized.put(CONFIGS_SERIALIZED.get(4), TASKS_COMMIT_STRUCT_TWO_TASK_CONNECTOR); - deserialized.put(CONFIGS_SERIALIZED.get(5), CONNECTOR_CONFIG_STRUCTS.get(2)); - deserialized.put(CONFIGS_SERIALIZED.get(6), TASK_CONFIG_STRUCTS.get(1)); - logOffset = 7; + deserialized.put(CONFIGS_SERIALIZED.get(3), TASK_CONFIG_STRUCTS.get(0)); + deserialized.put(CONFIGS_SERIALIZED.get(4), CONNECTOR_CONFIG_STRUCTS.get(1)); + deserialized.put(CONFIGS_SERIALIZED.get(5), TASKS_COMMIT_STRUCT_TWO_TASK_CONNECTOR); + deserialized.put(CONFIGS_SERIALIZED.get(6), CONNECTOR_TASK_COUNT_RECORD_STRUCTS.get(1)); + deserialized.put(CONFIGS_SERIALIZED.get(7), CONNECTOR_CONFIG_STRUCTS.get(2)); + deserialized.put(CONFIGS_SERIALIZED.get(8), TASK_CONFIG_STRUCTS.get(1)); + logOffset = 9; expectStart(existingRecords, deserialized); expectPartitionCount(1); @@ -722,7 +944,7 @@ public void testRestore() throws Exception { // Should see a single connector and its config should be the last one seen anywhere in the log ClusterConfigState configState = configStorage.snapshot(); - assertEquals(7, configState.offset()); // Should always be next to be read, even if uncommitted + assertEquals(logOffset, configState.offset()); // Should always be next to be read, even if uncommitted assertEquals(Arrays.asList(CONNECTOR_IDS.get(0)), new ArrayList<>(configState.connectors())); assertEquals(TargetState.STARTED, configState.targetState(CONNECTOR_IDS.get(0))); // CONNECTOR_CONFIG_STRUCTS[2] -> SAMPLE_CONFIGS[2] @@ -732,7 +954,9 @@ public void testRestore() throws Exception { // Both TASK_CONFIG_STRUCTS[0] -> SAMPLE_CONFIGS[0] assertEquals(SAMPLE_CONFIGS.get(0), configState.taskConfig(TASK_IDS.get(0))); assertEquals(SAMPLE_CONFIGS.get(0), configState.taskConfig(TASK_IDS.get(1))); + assertEquals(9, (int) configState.taskCountRecord(CONNECTOR_IDS.get(1))); assertEquals(Collections.EMPTY_SET, configState.inconsistentConnectors()); + assertEquals(Collections.singleton("connector1"), configState.connectorsPendingFencing); configStorage.stop(); @@ -1067,6 +1291,127 @@ public void testExceptionOnStartWhenConfigTopicHasMultiplePartitions() throws Ex PowerMock.verifyAll(); } + @Test + public void testFencableProducerPropertiesInsertedByDefault() throws Exception { + Map workerProps = new HashMap<>(DEFAULT_CONFIG_STORAGE_PROPS); + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "preparing"); + String groupId = "my-connect-cluster"; + workerProps.put(GROUP_ID_CONFIG, groupId); + workerProps.remove(TRANSACTIONAL_ID_CONFIG); + workerProps.remove(ENABLE_IDEMPOTENCE_CONFIG); + DistributedConfig config = new DistributedConfig(workerProps); + createStore(config, storeLog); + + PowerMock.replayAll(); + + Map fencableProducerProperties = configStorage.fencableProducerProps(config); + assertEquals("connect-cluster-" + groupId, fencableProducerProperties.get(TRANSACTIONAL_ID_CONFIG)); + assertEquals("true", fencableProducerProperties.get(ENABLE_IDEMPOTENCE_CONFIG)); + + PowerMock.verifyAll(); + } + + @Test + public void testFencableProducerPropertiesOverrideUserSuppliedValues() throws Exception { + Map workerProps = new HashMap<>(DEFAULT_CONFIG_STORAGE_PROPS); + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "preparing"); + String groupId = "my-other-connect-cluster"; + workerProps.put(GROUP_ID_CONFIG, groupId); + workerProps.put(TRANSACTIONAL_ID_CONFIG, "my-custom-transactional-id"); + workerProps.put(ENABLE_IDEMPOTENCE_CONFIG, "false"); + DistributedConfig config = new DistributedConfig(workerProps); + createStore(config, storeLog); + + PowerMock.replayAll(); + + Map fencableProducerProperties = configStorage.fencableProducerProps(config); + assertEquals("connect-cluster-" + groupId, fencableProducerProperties.get(TRANSACTIONAL_ID_CONFIG)); + assertEquals("true", fencableProducerProperties.get(ENABLE_IDEMPOTENCE_CONFIG)); + + PowerMock.verifyAll(); + } + + @Test + public void testConsumerPropertiesInsertedByDefaultWithExactlyOnceSourceEnabled() throws Exception { + Map workerProps = new HashMap<>(DEFAULT_CONFIG_STORAGE_PROPS); + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "enabled"); + workerProps.remove(ISOLATION_LEVEL_CONFIG); + DistributedConfig config = new DistributedConfig(workerProps); + createStore(config, storeLog); + + expectConfigure(); + PowerMock.replayAll(); + + configStorage.setupAndCreateKafkaBasedLog(TOPIC, config); + + assertEquals( + IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT), + capturedConsumerProps.getValue().get(ISOLATION_LEVEL_CONFIG) + ); + + PowerMock.verifyAll(); + } + + @Test + public void testConsumerPropertiesOverrideUserSuppliedValuesWithExactlyOnceSourceEnabled() throws Exception { + Map workerProps = new HashMap<>(DEFAULT_CONFIG_STORAGE_PROPS); + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "enabled"); + workerProps.put(ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_UNCOMMITTED.name().toLowerCase(Locale.ROOT)); + DistributedConfig config = new DistributedConfig(workerProps); + createStore(config, storeLog); + + expectConfigure(); + PowerMock.replayAll(); + + configStorage.setupAndCreateKafkaBasedLog(TOPIC, config); + + assertEquals( + IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT), + capturedConsumerProps.getValue().get(ISOLATION_LEVEL_CONFIG) + ); + + PowerMock.verifyAll(); + } + + @Test + public void testConsumerPropertiesNotInsertedByDefaultWithoutExactlyOnceSourceEnabled() throws Exception { + Map workerProps = new HashMap<>(DEFAULT_CONFIG_STORAGE_PROPS); + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "preparing"); + workerProps.remove(ISOLATION_LEVEL_CONFIG); + DistributedConfig config = new DistributedConfig(workerProps); + createStore(config, storeLog); + + expectConfigure(); + PowerMock.replayAll(); + + configStorage.setupAndCreateKafkaBasedLog(TOPIC, config); + + assertNull(capturedConsumerProps.getValue().get(ISOLATION_LEVEL_CONFIG)); + + PowerMock.verifyAll(); + } + + @Test + public void testConsumerPropertiesDoNotOverrideUserSuppliedValuesWithoutExactlyOnceSourceEnabled() throws Exception { + Map workerProps = new HashMap<>(DEFAULT_CONFIG_STORAGE_PROPS); + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "preparing"); + workerProps.put(ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_UNCOMMITTED.name().toLowerCase(Locale.ROOT)); + DistributedConfig config = new DistributedConfig(workerProps); + createStore(config, storeLog); + + expectConfigure(); + PowerMock.replayAll(); + + configStorage.setupAndCreateKafkaBasedLog(TOPIC, config); + + assertEquals( + IsolationLevel.READ_UNCOMMITTED.name().toLowerCase(Locale.ROOT), + capturedConsumerProps.getValue().get(ISOLATION_LEVEL_CONFIG) + ); + + PowerMock.verifyAll(); + } + private void expectConfigure() throws Exception { PowerMock.expectPrivate(configStorage, "createKafkaBasedLog", EasyMock.capture(capturedTopic), EasyMock.capture(capturedProducerProps), @@ -1075,6 +1420,13 @@ private void expectConfigure() throws Exception { .andReturn(storeLog); } + private void expectFencableProducer() throws Exception { + fencableProducer.initTransactions(); + EasyMock.expectLastCall(); + PowerMock.expectPrivate(configStorage, "createFencableProducer") + .andReturn(fencableProducer); + } + private void expectPartitionCount(int partitionCount) { EasyMock.expect(storeLog.partitionCount()) .andReturn(partitionCount); @@ -1117,6 +1469,11 @@ private void expectRead(final String key, final byte[] serializedValue, Struct d expectRead(serializedData, Collections.singletonMap(key, deserializedValue)); } + private void expectConvert(Schema valueSchema, Struct valueStruct, byte[] serialized) { + EasyMock.expect(converter.fromConnectData(EasyMock.eq(TOPIC), EasyMock.eq(valueSchema), EasyMock.eq(valueStruct))) + .andReturn(serialized); + } + // Expect a conversion & write to the underlying log, followed by a subsequent read when the data is consumed back // from the log. Validate the data that is captured when the conversion is performed matches the specified data // (by checking a single field's value) @@ -1137,6 +1494,14 @@ private void expectConvertWriteRead(final String configKey, final Schema valueSc }); } + private void expectConvertRead(final String configKey, final Struct struct, final byte[] serialized) { + EasyMock.expect(converter.toConnectData(EasyMock.eq(TOPIC), EasyMock.aryEq(serialized))) + .andAnswer(() -> new SchemaAndValue(null, serialized == null ? null : structToMap(struct))); + LinkedHashMap recordsToRead = new LinkedHashMap<>(); + recordsToRead.put(configKey, serialized); + expectReadToEnd(recordsToRead); + } + // This map needs to maintain ordering private void expectReadToEnd(final LinkedHashMap serializedConfigs) { EasyMock.expect(storeLog.readToEnd()) diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java index cdce2a186a2d4..bec4df5f9440e 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaOffsetBackingStoreTest.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.header.internals.RecordHeaders; import org.apache.kafka.common.record.TimestampType; @@ -46,6 +47,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.concurrent.ExecutionException; @@ -54,6 +56,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; +import static org.apache.kafka.clients.consumer.ConsumerConfig.ISOLATION_LEVEL_CONFIG; +import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -385,6 +389,87 @@ public void testSetFailure() throws Exception { PowerMock.verifyAll(); } + @Test + public void testConsumerPropertiesInsertedByDefaultWithExactlyOnceSourceEnabled() throws Exception { + Map workerProps = new HashMap<>(DEFAULT_PROPS); + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "enabled"); + workerProps.remove(ISOLATION_LEVEL_CONFIG); + DistributedConfig config = new DistributedConfig(workerProps); + + expectConfigure(); + expectClusterId(); + PowerMock.replayAll(); + + store.configure(config); + + assertEquals( + IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT), + capturedConsumerProps.getValue().get(ISOLATION_LEVEL_CONFIG) + ); + + PowerMock.verifyAll(); + } + + @Test + public void testConsumerPropertiesOverrideUserSuppliedValuesWithExactlyOnceSourceEnabled() throws Exception { + Map workerProps = new HashMap<>(DEFAULT_PROPS); + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "enabled"); + workerProps.put(ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_UNCOMMITTED.name().toLowerCase(Locale.ROOT)); + DistributedConfig config = new DistributedConfig(workerProps); + + expectConfigure(); + expectClusterId(); + PowerMock.replayAll(); + + store.configure(config); + + assertEquals( + IsolationLevel.READ_COMMITTED.name().toLowerCase(Locale.ROOT), + capturedConsumerProps.getValue().get(ISOLATION_LEVEL_CONFIG) + ); + + PowerMock.verifyAll(); + } + + @Test + public void testConsumerPropertiesNotInsertedByDefaultWithoutExactlyOnceSourceEnabled() throws Exception { + Map workerProps = new HashMap<>(DEFAULT_PROPS); + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "disabled"); + workerProps.remove(ISOLATION_LEVEL_CONFIG); + DistributedConfig config = new DistributedConfig(workerProps); + + expectConfigure(); + expectClusterId(); + PowerMock.replayAll(); + + store.configure(config); + + assertNull(capturedConsumerProps.getValue().get(ISOLATION_LEVEL_CONFIG)); + + PowerMock.verifyAll(); + } + + @Test + public void testConsumerPropertiesDoNotOverrideUserSuppliedValuesWithoutExactlyOnceSourceEnabled() throws Exception { + Map workerProps = new HashMap<>(DEFAULT_PROPS); + workerProps.put(EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG, "disabled"); + workerProps.put(ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_UNCOMMITTED.name().toLowerCase(Locale.ROOT)); + DistributedConfig config = new DistributedConfig(workerProps); + + expectConfigure(); + expectClusterId(); + PowerMock.replayAll(); + + store.configure(config); + + assertEquals( + IsolationLevel.READ_UNCOMMITTED.name().toLowerCase(Locale.ROOT), + capturedConsumerProps.getValue().get(ISOLATION_LEVEL_CONFIG) + ); + + PowerMock.verifyAll(); + } + private void expectConfigure() throws Exception { PowerMock.expectPrivate(store, "createKafkaBasedLog", EasyMock.capture(capturedTopic), EasyMock.capture(capturedProducerProps), EasyMock.capture(capturedConsumerProps), EasyMock.capture(capturedConsumedCallback), diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/OffsetStorageReaderImplTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/OffsetStorageReaderImplTest.java new file mode 100644 index 0000000000000..709beef30f119 --- /dev/null +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/OffsetStorageReaderImplTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.storage; + +import org.apache.kafka.connect.data.SchemaAndValue; +import org.easymock.EasyMock; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.api.easymock.annotation.Mock; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +@RunWith(PowerMockRunner.class) +public class OffsetStorageReaderImplTest { + + private static final String NAMESPACE = "reddit-source"; + private static final Map PARTITION1 = Collections.singletonMap("subreddit", "apachekafka"); + private static final Map PARTITION2 = Collections.singletonMap("subreddit", "CatsStandingUp"); + + private static final Map OFFSET1 = Collections.singletonMap("timestamp", "4761"); + private static final Map OFFSET2 = Collections.singletonMap("timestamp", "2112"); + + @Mock private Converter keyConverter; + @Mock private Converter valueConverter; + @Mock private ConnectorOffsetBackingStore store; + + private OffsetStorageReaderImpl offsetReader; + + @Before + public void setup() { + offsetReader = new OffsetStorageReaderImpl(store, NAMESPACE, keyConverter, valueConverter); + } + + @Test + public void testSingleRead() { + expectOffsetsForPartitions(Collections.singletonMap(PARTITION1, OFFSET1)); + + PowerMock.replayAll(); + + assertEquals( + OFFSET1, + offsetReader.offset(PARTITION1) + ); + } + + @Test + public void testMultipleReads() { + Map, Map> expectedOffsets = new HashMap<>(); + expectedOffsets.put(PARTITION1, OFFSET1); + expectedOffsets.put(PARTITION2, OFFSET2); + expectOffsetsForPartitions(expectedOffsets); + + PowerMock.replayAll(); + + assertEquals( + expectedOffsets, + offsetReader.offsets(expectedOffsets.keySet()) + ); + } + + private void expectOffsetsForPartitions(Map, Map> offsetsAndPartitions) { + Map, byte[]> serializedPartitions = offsetsAndPartitions.keySet().stream() + .collect(Collectors.toMap(Function.identity(), this::serialize)); + + serializedPartitions.forEach((partition, serializedPartition) -> + EasyMock.expect(keyConverter.fromConnectData(NAMESPACE, null, Arrays.asList(NAMESPACE, partition))) + .andReturn(serializedPartition) + ); + + Map, byte[]> serializedOffsets = offsetsAndPartitions.values().stream() + .collect(Collectors.toMap(Function.identity(), this::serialize)); + + Map serializedOffsetsAndPartitions = serializedPartitions.entrySet().stream() + .collect(Collectors.toMap(e -> ByteBuffer.wrap(e.getValue()), e -> ByteBuffer.wrap(serializedOffsets.get(offsetsAndPartitions.get(e.getKey()))))); + + EasyMock.expect(store.get(EasyMock.anyObject())).andReturn(CompletableFuture.completedFuture(serializedOffsetsAndPartitions)); + + serializedOffsets.forEach((offset, serializedOffset) -> + EasyMock.expect(valueConverter.toConnectData(NAMESPACE, serializedOffset)).andReturn(new SchemaAndValue(null, offset)) + ); + } + + private byte[] serialize(Map partitionOrOffset) { + return ByteBuffer.allocate(4).putInt(partitionOrOffset.hashCode()).array(); + } +} diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/OffsetStorageWriterTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/OffsetStorageWriterTest.java index b6eb0f6a48763..8972c418360a9 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/OffsetStorageWriterTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/OffsetStorageWriterTest.java @@ -229,4 +229,4 @@ private void expectStore(Map key, byte[] keySerialized, }); } -} +} \ No newline at end of file diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/ConnectUtilsTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/ConnectUtilsTest.java index 62a01b706a832..d1330277e8e45 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/ConnectUtilsTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/ConnectUtilsTest.java @@ -26,9 +26,11 @@ import org.junit.Test; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -101,4 +103,73 @@ public void testAddMetricsContextPropertiesStandalone() { assertEquals("cluster-1", prop.get(CommonClientConfigs.METRICS_CONTEXT_PREFIX + WorkerConfig.CONNECT_KAFKA_CLUSTER_ID)); } + + @Test + public void testNoOverrideWarning() { + Map props = new HashMap<>(); + assertEquals( + Optional.empty(), + ConnectUtils.ensurePropertyAndGetWarning(props, "key", "value", "because i say so", true) + ); + assertEquals("value", props.get("key")); + + props.clear(); + assertEquals( + Optional.empty(), + ConnectUtils.ensurePropertyAndGetWarning(props, "key", "value", "because i say so", false) + ); + assertEquals("value", props.get("key")); + + props.clear(); + props.put("key", "value"); + assertEquals( + Optional.empty(), + ConnectUtils.ensurePropertyAndGetWarning(props, "key", "value", "because i say so", true) + ); + assertEquals("value", props.get("key")); + + props.clear(); + props.put("key", "VALUE"); + assertEquals( + Optional.empty(), + ConnectUtils.ensurePropertyAndGetWarning(props, "key", "value", "because i say so", false) + ); + assertEquals("VALUE", props.get("key")); + } + + @Test + public void testOverrideWarning() { + Map props = new HashMap<>(); + props.put("\u1984", "little brother"); + String expectedWarning = "The value 'little brother' for the '\u1984' property will be ignored as it cannot be overridden " + + "thanks to newly-introduced federal legislation. " + + "The value 'big brother' will be used instead."; + assertEquals( + Optional.of(expectedWarning), + ConnectUtils.ensurePropertyAndGetWarning( + props, + "\u1984", + "big brother", + "thanks to newly-introduced federal legislation", + false) + ); + assertEquals(Collections.singletonMap("\u1984", "big brother"), props); + + props.clear(); + props.put("\u1984", "BIG BROTHER"); + expectedWarning = "The value 'BIG BROTHER' for the '\u1984' property will be ignored as it cannot be overridden " + + "thanks to newly-introduced federal legislation. " + + "The value 'big brother' will be used instead."; + assertEquals( + Optional.of(expectedWarning), + ConnectUtils.ensurePropertyAndGetWarning( + props, + "\u1984", + "big brother", + "thanks to newly-introduced federal legislation", + true) + ); + assertEquals(Collections.singletonMap("\u1984", "big brother"), props); + } + } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java index adcde378bbe95..4b097c7e43157 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java @@ -18,6 +18,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.kafka.common.network.ListenerName; +import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.utils.Exit; import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.runtime.isolation.Plugins; @@ -80,6 +82,7 @@ public class EmbeddedConnectCluster { private final Set connectCluster; private final EmbeddedKafkaCluster kafkaCluster; + private final ListenerName brokerListener; private final Map workerProps; private final String connectClusterName; private final int numBrokers; @@ -93,12 +96,13 @@ public class EmbeddedConnectCluster { private final ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader(); private EmbeddedConnectCluster(String name, Map workerProps, int numWorkers, - int numBrokers, Properties brokerProps, + int numBrokers, Properties brokerProps, ListenerName brokerListener, boolean maskExitProcedures) { this.workerProps = workerProps; this.connectClusterName = name; this.numBrokers = numBrokers; this.kafkaCluster = new EmbeddedKafkaCluster(numBrokers, brokerProps); + this.brokerListener = brokerListener; this.connectCluster = new LinkedHashSet<>(); this.numInitialWorkers = numWorkers; this.maskExitProcedures = maskExitProcedures; @@ -240,7 +244,7 @@ public boolean allWorkersRunning() { public void startConnect() { log.info("Starting Connect cluster '{}' with {} workers", connectClusterName, numInitialWorkers); - workerProps.put(BOOTSTRAP_SERVERS_CONFIG, kafka().bootstrapServers()); + workerProps.put(BOOTSTRAP_SERVERS_CONFIG, kafka().bootstrapServers(brokerListener)); // use a random available port workerProps.put(LISTENERS_CONFIG, "HTTP://" + REST_HOST_NAME + ":0"); @@ -795,6 +799,7 @@ public static class Builder { private Map workerProps = new HashMap<>(); private int numWorkers = DEFAULT_NUM_WORKERS; private int numBrokers = DEFAULT_NUM_BROKERS; + private ListenerName brokerListener = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT); private Properties brokerProps = DEFAULT_BROKER_CONFIG; private boolean maskExitProcedures = true; @@ -818,6 +823,11 @@ public Builder numBrokers(int numBrokers) { return this; } + public Builder brokerListener(ListenerName brokerListener) { + this.brokerListener = brokerListener; + return this; + } + public Builder brokerProps(Properties brokerProps) { this.brokerProps = brokerProps; return this; @@ -842,7 +852,7 @@ public Builder maskExitProcedures(boolean mask) { public EmbeddedConnectCluster build() { return new EmbeddedConnectCluster(name, workerProps, numWorkers, numBrokers, - brokerProps, maskExitProcedures); + brokerProps, brokerListener, maskExitProcedures); } } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedKafkaCluster.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedKafkaCluster.java index f1a63a4615caa..5ac4ebd1b06b2 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedKafkaCluster.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedKafkaCluster.java @@ -75,6 +75,8 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; /** * Setup an embedded Kafka cluster with specified number of brokers and specified broker properties. To be used for @@ -95,7 +97,7 @@ public class EmbeddedKafkaCluster { private final boolean hasListenerConfig; private EmbeddedZookeeper zookeeper = null; - private ListenerName listenerName = new ListenerName("PLAINTEXT"); + private ListenerName defaultListenerName = new ListenerName("PLAINTEXT"); private KafkaProducer producer; public EmbeddedKafkaCluster(final int numBrokers, @@ -139,20 +141,18 @@ private void doStart() { putIfAbsent(brokerConfig, KafkaConfig.LogCleanerDedupeBufferSizeProp(), 2 * 1024 * 1024L); Object listenerConfig = brokerConfig.get(KafkaConfig.InterBrokerListenerNameProp()); - if (listenerConfig == null) - listenerConfig = brokerConfig.get(KafkaConfig.InterBrokerSecurityProtocolProp()); - if (listenerConfig == null) - listenerConfig = "PLAINTEXT"; - listenerName = new ListenerName(listenerConfig.toString()); + if (listenerConfig != null) { + defaultListenerName = new ListenerName(listenerConfig.toString()); + } for (int i = 0; i < brokers.length; i++) { brokerConfig.put(KafkaConfig.BrokerIdProp(), i); currentBrokerLogDirs[i] = currentBrokerLogDirs[i] == null ? createLogDir() : currentBrokerLogDirs[i]; brokerConfig.put(KafkaConfig.LogDirProp(), currentBrokerLogDirs[i]); if (!hasListenerConfig) - brokerConfig.put(KafkaConfig.ListenersProp(), listenerName.value() + "://localhost:" + currentBrokerPorts[i]); + brokerConfig.put(KafkaConfig.ListenersProp(), defaultListenerName.value() + "://localhost:" + currentBrokerPorts[i]); brokers[i] = TestUtils.createServer(new KafkaConfig(brokerConfig, true), time); - currentBrokerPorts[i] = brokers[i].boundPort(listenerName); + currentBrokerPorts[i] = brokers[i].boundPort(defaultListenerName); } Map producerProps = new HashMap<>(); @@ -187,7 +187,7 @@ private void stop(boolean deleteLogDirs, boolean stopZK) { try { broker.shutdown(); } catch (Throwable t) { - String msg = String.format("Could not shutdown broker at %s", address(broker)); + String msg = String.format("Could not shutdown broker at %s", address(broker, defaultListenerName)); log.error(msg, t); throw new RuntimeException(msg, t); } @@ -200,7 +200,7 @@ private void stop(boolean deleteLogDirs, boolean stopZK) { CoreUtils.delete(broker.config().logDirs()); } catch (Throwable t) { String msg = String.format("Could not clean up log dirs for broker at %s", - address(broker)); + address(broker, defaultListenerName)); log.error(msg, t); throw new RuntimeException(msg, t); } @@ -234,13 +234,19 @@ private String createLogDir() { } public String bootstrapServers() { + return bootstrapServers(defaultListenerName); + } + + public String bootstrapServers(ListenerName listenerName) { return Arrays.stream(brokers) - .map(this::address) + .map(broker -> address(broker, listenerName)) .collect(Collectors.joining(",")); } - public String address(KafkaServer server) { - final EndPoint endPoint = server.advertisedListeners().head(); + public String address(KafkaServer server, ListenerName listenerName) { + final EndPoint endPoint = server.advertisedListeners() + .find(endpoint -> listenerName.equals(endpoint.listenerName())) + .get(); return endPoint.host() + ":" + endPoint.port(); } @@ -416,7 +422,7 @@ public void produce(String topic, Integer partition, String key, String value) { } public Admin createAdminClient(Properties adminClientConfig) { - adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()); + adminClientConfig.putIfAbsent(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()); final Object listeners = brokerConfig.get(KafkaConfig.ListenersProp()); if (listeners != null && listeners.toString().contains("SSL")) { adminClientConfig.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, brokerConfig.get(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)); @@ -439,9 +445,23 @@ public Admin createAdminClient() { * @return a {@link ConsumerRecords} collection containing at least n records. */ public ConsumerRecords consume(int n, long maxDuration, String... topics) { + return consume(n, maxDuration, Collections.emptyMap(), topics); + } + + /** + * Consume at least n records in a given duration or throw an exception. + * + * @param n the number of expected records in this topic. + * @param maxDuration the max duration to wait for these records (in milliseconds). + * @param topics the topics to subscribe and consume records from. + * @param consumerProps overrides to the default properties the consumer is constructed with; + * may not be null + * @return a {@link ConsumerRecords} collection containing at least n records. + */ + public ConsumerRecords consume(int n, long maxDuration, Map consumerProps, String... topics) { Map>> records = new HashMap<>(); int consumedRecords = 0; - try (KafkaConsumer consumer = createConsumerAndSubscribeTo(Collections.emptyMap(), topics)) { + try (KafkaConsumer consumer = createConsumerAndSubscribeTo(consumerProps, topics)) { final long startMillis = System.currentTimeMillis(); long allowedDuration = maxDuration; while (allowedDuration > 0) { @@ -495,6 +515,28 @@ public KafkaConsumer createConsumerAndSubscribeTo(Map createProducer(Map producerProps) { + Map props = new HashMap<>(producerProps); + + putIfAbsent(props, BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()); + putIfAbsent(props, ENABLE_AUTO_COMMIT_CONFIG, "false"); + putIfAbsent(props, AUTO_OFFSET_RESET_CONFIG, "earliest"); + putIfAbsent(props, KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); + putIfAbsent(props, VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); + if (sslEnabled()) { + putIfAbsent(props, SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, brokerConfig.get(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)); + putIfAbsent(props, SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, brokerConfig.get(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)); + putIfAbsent(props, CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); + } + KafkaProducer producer; + try { + producer = new KafkaProducer<>(props); + } catch (Throwable t) { + throw new ConnectException("Failed to create producer", t); + } + return producer; + } + private static void putIfAbsent(final Map props, final String propertyKey, final Object propertyValue) { if (!props.containsKey(propertyKey)) { props.put(propertyKey, propertyValue); diff --git a/connect/runtime/src/test/resources/log4j.properties b/connect/runtime/src/test/resources/log4j.properties index 176692deb7b2b..9fb0613015988 100644 --- a/connect/runtime/src/test/resources/log4j.properties +++ b/connect/runtime/src/test/resources/log4j.properties @@ -33,3 +33,5 @@ log4j.logger.kafka=WARN log4j.logger.org.apache.kafka.connect=DEBUG log4j.logger.org.apache.kafka.connect.runtime.distributed=DEBUG log4j.logger.org.apache.kafka.connect.integration=DEBUG +# Can get really noisy at the end of tests +log4j.logger.org.apache.kafka.connect.runtime.distributed.WorkerCoordinator=INFO diff --git a/gradle/spotbugs-exclude.xml b/gradle/spotbugs-exclude.xml index 98588117f04dd..103b49b6c4320 100644 --- a/gradle/spotbugs-exclude.xml +++ b/gradle/spotbugs-exclude.xml @@ -279,7 +279,7 @@ For a detailed description of spotbugs bug categories, see https://spotbugs.read - + diff --git a/tests/kafkatest/services/connect.py b/tests/kafkatest/services/connect.py index 26c0d927dccd5..3682ce943377d 100644 --- a/tests/kafkatest/services/connect.py +++ b/tests/kafkatest/services/connect.py @@ -432,13 +432,14 @@ class VerifiableSource(VerifiableConnector): Helper class for running a verifiable source connector on a Kafka Connect cluster and analyzing the output. """ - def __init__(self, cc, name="verifiable-source", tasks=1, topic="verifiable", throughput=1000): + def __init__(self, cc, name="verifiable-source", tasks=1, topic="verifiable", throughput=1000, complete_records=False): self.cc = cc self.logger = self.cc.logger self.name = name self.tasks = tasks self.topic = topic self.throughput = throughput + self.complete_records = complete_records def committed_messages(self): return list(filter(lambda m: 'committed' in m and m['committed'], self.messages())) @@ -453,7 +454,8 @@ def start(self): 'connector.class': 'org.apache.kafka.connect.tools.VerifiableSourceConnector', 'tasks.max': self.tasks, 'topic': self.topic, - 'throughput': self.throughput + 'throughput': self.throughput, + 'complete.record.data': self.complete_records }) diff --git a/tests/kafkatest/tests/connect/connect_distributed_test.py b/tests/kafkatest/tests/connect/connect_distributed_test.py index 6bc52b0d35f49..7479e7789d194 100644 --- a/tests/kafkatest/tests/connect/connect_distributed_test.py +++ b/tests/kafkatest/tests/connect/connect_distributed_test.py @@ -54,6 +54,7 @@ class ConnectDistributedTest(Test): STATUS_TOPIC = "connect-status" STATUS_REPLICATION_FACTOR = "1" STATUS_PARTITIONS = "1" + EXACTLY_ONCE_SOURCE_SUPPORT = "disabled" SCHEDULED_REBALANCE_MAX_DELAY_MS = "60000" CONNECT_PROTOCOL="sessioned" @@ -84,7 +85,11 @@ def setup_services(self, security_protocol=SecurityConfig.PLAINTEXT, timestamp_t self.kafka = KafkaService(self.test_context, self.num_brokers, self.zk, security_protocol=security_protocol, interbroker_security_protocol=security_protocol, topics=self.topics, version=broker_version, - server_prop_overrides=[["auto.create.topics.enable", str(auto_create_topics)]]) + server_prop_overrides=[ + ["auto.create.topics.enable", str(auto_create_topics)], + ["transaction.state.log.replication.factor", str(self.num_brokers)], + ["transaction.state.log.min.isr", str(self.num_brokers)] + ]) if timestamp_type is not None: for node in self.kafka.nodes: node.config[config_property.MESSAGE_TIMESTAMP_TYPE] = timestamp_type @@ -158,27 +163,32 @@ def task_is_running(self, connector, task_id, node=None): return self._task_has_state(task_id, status, 'RUNNING') @cluster(num_nodes=5) - @matrix(connect_protocol=['sessioned', 'compatible', 'eager']) - def test_restart_failed_connector(self, connect_protocol): + @matrix(exactly_once_source=[True, False], connect_protocol=['sessioned', 'compatible', 'eager']) + def test_restart_failed_connector(self, exactly_once_source, connect_protocol): + self.EXACTLY_ONCE_SOURCE_SUPPORT = 'enabled' if exactly_once_source else 'disabled' self.CONNECT_PROTOCOL = connect_protocol self.setup_services() self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node)) self.cc.start() - self.sink = MockSink(self.cc, self.topics.keys(), mode='connector-failure', delay_sec=5) - self.sink.start() + if exactly_once_source: + self.connector = MockSource(self.cc, mode='connector-failure', delay_sec=5) + else: + self.connector = MockSink(self.cc, self.topics.keys(), mode='connector-failure', delay_sec=5) + self.connector.start() - wait_until(lambda: self.connector_is_failed(self.sink), timeout_sec=15, + wait_until(lambda: self.connector_is_failed(self.connector), timeout_sec=15, err_msg="Failed to see connector transition to the FAILED state") - self.cc.restart_connector(self.sink.name) + self.cc.restart_connector(self.connector.name) - wait_until(lambda: self.connector_is_running(self.sink), timeout_sec=10, + wait_until(lambda: self.connector_is_running(self.connector), timeout_sec=10, err_msg="Failed to see connector transition to the RUNNING state") @cluster(num_nodes=5) - @matrix(connector_type=['source', 'sink'], connect_protocol=['sessioned', 'compatible', 'eager']) + @matrix(connector_type=['source', 'exactly-once source', 'sink'], connect_protocol=['sessioned', 'compatible', 'eager']) def test_restart_failed_task(self, connector_type, connect_protocol): + self.EXACTLY_ONCE_SOURCE_SUPPORT = 'enabled' if connector_type == 'exactly-once source' else 'disabled' self.CONNECT_PROTOCOL = connect_protocol self.setup_services() self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node)) @@ -246,13 +256,14 @@ def test_restart_connector_and_tasks_failed_task(self, connector_type, connect_p err_msg="Failed to see task transition to the RUNNING state") @cluster(num_nodes=5) - @matrix(connect_protocol=['sessioned', 'compatible', 'eager']) - def test_pause_and_resume_source(self, connect_protocol): + @matrix(exactly_once_source=[True, False], connect_protocol=['sessioned', 'compatible', 'eager']) + def test_pause_and_resume_source(self, exactly_once_source, connect_protocol): """ Verify that source connectors stop producing records when paused and begin again after being resumed. """ + self.EXACTLY_ONCE_SOURCE_SUPPORT = 'enabled' if exactly_once_source else 'disabled' self.CONNECT_PROTOCOL = connect_protocol self.setup_services() self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node)) @@ -335,12 +346,13 @@ def test_pause_and_resume_sink(self, connect_protocol): err_msg="Failed to consume messages after resuming sink connector") @cluster(num_nodes=5) - @matrix(connect_protocol=['sessioned', 'compatible', 'eager']) - def test_pause_state_persistent(self, connect_protocol): + @matrix(exactly_once_source=[True, False], connect_protocol=['sessioned', 'compatible', 'eager']) + def test_pause_state_persistent(self, exactly_once_source, connect_protocol): """ Verify that paused state is preserved after a cluster restart. """ + self.EXACTLY_ONCE_SOURCE_SUPPORT = 'enabled' if exactly_once_source else 'disabled' self.CONNECT_PROTOCOL = connect_protocol self.setup_services() self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node)) @@ -362,13 +374,14 @@ def test_pause_state_persistent(self, connect_protocol): err_msg="Failed to see connector startup in PAUSED state") @cluster(num_nodes=6) - @matrix(security_protocol=[SecurityConfig.PLAINTEXT, SecurityConfig.SASL_SSL], connect_protocol=['sessioned', 'compatible', 'eager']) - def test_file_source_and_sink(self, security_protocol, connect_protocol): + @matrix(security_protocol=[SecurityConfig.PLAINTEXT, SecurityConfig.SASL_SSL], exactly_once_source=[True, False], connect_protocol=['sessioned', 'compatible', 'eager']) + def test_file_source_and_sink(self, security_protocol, exactly_once_source, connect_protocol): """ Tests that a basic file connector works across clean rolling bounces. This validates that the connector is correctly created, tasks instantiated, and as nodes restart the work is rebalanced across nodes. """ + self.EXACTLY_ONCE_SOURCE_SUPPORT = 'enabled' if exactly_once_source else 'disabled' self.CONNECT_PROTOCOL = connect_protocol self.setup_services(security_protocol=security_protocol) self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node)) @@ -452,7 +465,7 @@ def test_bounce(self, clean, connect_protocol): self.cc.stop() # Validate at least once delivery of everything that was reported as written since we should have flushed and - # cleanly exited. Currently this only tests at least once delivery because the sink task may not have consumed + # cleanly exited. Currently this only tests at least once delivery for sinks because the task may not have consumed # all the messages generated by the source task. This needs to be done per-task since seqnos are not unique across # tasks. success = True @@ -518,6 +531,91 @@ def test_bounce(self, clean, connect_protocol): assert success, "Found validation errors:\n" + "\n ".join(errors) + @cluster(num_nodes=6) + @matrix(clean=[True, False], connect_protocol=['sessioned', 'compatible', 'eager']) + def test_exactly_once_source(self, clean, connect_protocol): + """ + Validates that source and sink tasks that run continuously and produce a predictable sequence of messages + run correctly and deliver messages exactly once when Kafka Connect workers undergo clean rolling bounces. + """ + num_tasks = 3 + + self.EXACTLY_ONCE_SOURCE_SUPPORT = 'enabled' + self.CONNECT_PROTOCOL = connect_protocol + self.setup_services() + self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node)) + self.cc.start() + + self.source = VerifiableSource(self.cc, topic=self.TOPIC, tasks=num_tasks, throughput=100, complete_records=True) + self.source.start() + + for _ in range(3): + for node in self.cc.nodes: + started = time.time() + self.logger.info("%s bouncing Kafka Connect on %s", clean and "Clean" or "Hard", str(node.account)) + self.cc.stop_node(node, clean_shutdown=clean) + with node.account.monitor_log(self.cc.LOG_FILE) as monitor: + self.cc.start_node(node) + monitor.wait_until("Starting connectors and tasks using config offset", timeout_sec=90, + err_msg="Kafka Connect worker didn't successfully join group and start work") + self.logger.info("Bounced Kafka Connect on %s and rejoined in %f seconds", node.account, time.time() - started) + + # Give additional time for the consumer groups to recover. Even if it is not a hard bounce, there are + # some cases where a restart can cause a rebalance to take the full length of the session timeout + # (e.g. if the client shuts down before it has received the memberId from its initial JoinGroup). + # If we don't give enough time for the group to stabilize, the next bounce may cause consumers to + # be shut down before they have any time to process data and we can end up with zero data making it + # through the test. + time.sleep(15) + + # Wait at least scheduled.rebalance.max.delay.ms to expire and rebalance + time.sleep(60) + + # Allow the connectors to startup, recover, and exit cleanly before + # ending the test. It's possible for the source connector to make + # uncommitted progress, and for the sink connector to read messages that + # have not been committed yet, and fail a later assertion. + wait_until(lambda: self.is_running(self.source), timeout_sec=30, + err_msg="Failed to see connector transition to the RUNNING state") + time.sleep(15) + self.source.stop() + self.cc.stop() + + consumer = ConsoleConsumer(self.test_context, 1, self.kafka, self.source.topic, message_validator=json.loads, consumer_timeout_ms=1000, isolation_level="read_committed") + consumer.run() + src_messages = consumer.messages_consumed[1] + + success = True + errors = [] + for task in range(num_tasks): + # Validate source messages + src_seqnos = [msg['payload']['seqno'] for msg in src_messages if msg['payload']['task'] == task] + # Every seqno up to the largest one we ever saw should appear. Each seqno should only appear once because clean + # bouncing should commit on rebalance. + src_seqno_max = max(src_seqnos) if len(src_seqnos) else 0 + self.logger.debug("Max source seqno: %d", src_seqno_max) + src_seqno_counts = Counter(src_seqnos) + missing_src_seqnos = sorted(set(range(src_seqno_max)).difference(set(src_seqnos))) + duplicate_src_seqnos = sorted(seqno for seqno,count in src_seqno_counts.items() if count > 1) + + if missing_src_seqnos: + self.logger.error("Missing source sequence numbers for task " + str(task)) + errors.append("Found missing source sequence numbers for task %d: %s" % (task, missing_src_seqnos)) + success = False + if duplicate_src_seqnos: + self.logger.error("Duplicate source sequence numbers for task " + str(task)) + errors.append("Found duplicate source sequence numbers for task %d: %s" % (task, duplicate_src_seqnos)) + success = False + + if not success: + self.mark_for_collect(self.cc) + # Also collect the data in the topic to aid in debugging + consumer_validator = ConsoleConsumer(self.test_context, 1, self.kafka, self.source.topic, consumer_timeout_ms=1000, print_key=True) + consumer_validator.run() + self.mark_for_collect(consumer_validator, "consumer_stdout") + + assert success, "Found validation errors:\n" + "\n ".join(errors) + @cluster(num_nodes=6) @matrix(connect_protocol=['sessioned', 'compatible', 'eager']) def test_transformations(self, connect_protocol): @@ -576,41 +674,50 @@ def test_transformations(self, connect_protocol): assert obj['payload'][ts_fieldname] == ts @cluster(num_nodes=5) - @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned') - @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned') - @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned') - @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned') - @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='sessioned') - @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') - @parametrize(broker_version=str(LATEST_2_3), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') - @parametrize(broker_version=str(LATEST_2_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') - @parametrize(broker_version=str(LATEST_2_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') - @parametrize(broker_version=str(LATEST_2_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') - @parametrize(broker_version=str(LATEST_1_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') - @parametrize(broker_version=str(LATEST_1_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') - @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') - @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') - @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') - @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='compatible') - @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') - @parametrize(broker_version=str(LATEST_2_3), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') - @parametrize(broker_version=str(LATEST_2_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') - @parametrize(broker_version=str(LATEST_2_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') - @parametrize(broker_version=str(LATEST_2_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') - @parametrize(broker_version=str(LATEST_1_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') - @parametrize(broker_version=str(LATEST_1_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') - @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') - @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') - @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') - @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, security_protocol=SecurityConfig.PLAINTEXT, connect_protocol='eager') - def test_broker_compatibility(self, broker_version, auto_create_topics, security_protocol, connect_protocol): + @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, exactly_once_source=True, connect_protocol='sessioned') + @parametrize(broker_version=str(LATEST_2_3), auto_create_topics=False, exactly_once_source=True, connect_protocol='sessioned') + @parametrize(broker_version=str(LATEST_2_2), auto_create_topics=False, exactly_once_source=True, connect_protocol='sessioned') + @parametrize(broker_version=str(LATEST_2_1), auto_create_topics=False, exactly_once_source=True, connect_protocol='sessioned') + @parametrize(broker_version=str(LATEST_2_0), auto_create_topics=False, exactly_once_source=True, connect_protocol='sessioned') + @parametrize(broker_version=str(LATEST_1_1), auto_create_topics=False, exactly_once_source=True, connect_protocol='sessioned') + @parametrize(broker_version=str(LATEST_1_0), auto_create_topics=False, exactly_once_source=True, connect_protocol='sessioned') + @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, exactly_once_source=True, connect_protocol='sessioned') + @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, exactly_once_source=False, connect_protocol='sessioned') + @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, exactly_once_source=False, connect_protocol='sessioned') + @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, exactly_once_source=False, connect_protocol='sessioned') + @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, exactly_once_source=False, connect_protocol='sessioned') + @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, exactly_once_source=False, connect_protocol='sessioned') + @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, exactly_once_source=False, connect_protocol='compatible') + @parametrize(broker_version=str(LATEST_2_3), auto_create_topics=False, exactly_once_source=False, connect_protocol='compatible') + @parametrize(broker_version=str(LATEST_2_2), auto_create_topics=False, exactly_once_source=False, connect_protocol='compatible') + @parametrize(broker_version=str(LATEST_2_1), auto_create_topics=False, exactly_once_source=False, connect_protocol='compatible') + @parametrize(broker_version=str(LATEST_2_0), auto_create_topics=False, exactly_once_source=False, connect_protocol='compatible') + @parametrize(broker_version=str(LATEST_1_1), auto_create_topics=False, exactly_once_source=False, connect_protocol='compatible') + @parametrize(broker_version=str(LATEST_1_0), auto_create_topics=False, exactly_once_source=False, connect_protocol='compatible') + @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, exactly_once_source=False, connect_protocol='compatible') + @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, exactly_once_source=False, connect_protocol='compatible') + @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, exactly_once_source=False, connect_protocol='compatible') + @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, exactly_once_source=False, connect_protocol='compatible') + @parametrize(broker_version=str(DEV_BRANCH), auto_create_topics=False, exactly_once_source=False, connect_protocol='eager') + @parametrize(broker_version=str(LATEST_2_3), auto_create_topics=False, exactly_once_source=False, connect_protocol='eager') + @parametrize(broker_version=str(LATEST_2_2), auto_create_topics=False, exactly_once_source=False, connect_protocol='eager') + @parametrize(broker_version=str(LATEST_2_1), auto_create_topics=False, exactly_once_source=False, connect_protocol='eager') + @parametrize(broker_version=str(LATEST_2_0), auto_create_topics=False, exactly_once_source=False, connect_protocol='eager') + @parametrize(broker_version=str(LATEST_1_1), auto_create_topics=False, exactly_once_source=False, connect_protocol='eager') + @parametrize(broker_version=str(LATEST_1_0), auto_create_topics=False, exactly_once_source=False, connect_protocol='eager') + @parametrize(broker_version=str(LATEST_0_11_0), auto_create_topics=False, exactly_once_source=False, connect_protocol='eager') + @parametrize(broker_version=str(LATEST_0_10_2), auto_create_topics=False, exactly_once_source=False, connect_protocol='eager') + @parametrize(broker_version=str(LATEST_0_10_1), auto_create_topics=False, exactly_once_source=False, connect_protocol='eager') + @parametrize(broker_version=str(LATEST_0_10_0), auto_create_topics=True, exactly_once_source=False, connect_protocol='eager') + def test_broker_compatibility(self, broker_version, auto_create_topics, exactly_once_source, connect_protocol): """ Verify that Connect will start up with various broker versions with various configurations. When Connect distributed starts up, it either creates internal topics (v0.10.1.0 and after) or relies upon the broker to auto-create the topics (v0.10.0.x and before). """ + self.EXACTLY_ONCE_SOURCE_SUPPORT = 'enabled' if exactly_once_source else 'disabled' self.CONNECT_PROTOCOL = connect_protocol - self.setup_services(broker_version=KafkaVersion(broker_version), auto_create_topics=auto_create_topics, security_protocol=security_protocol) + self.setup_services(broker_version=KafkaVersion(broker_version), auto_create_topics=auto_create_topics, security_protocol=SecurityConfig.PLAINTEXT) self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node)) self.cc.start() diff --git a/tests/kafkatest/tests/connect/templates/connect-distributed.properties b/tests/kafkatest/tests/connect/templates/connect-distributed.properties index 6d2d5e28d1035..c407766feda85 100644 --- a/tests/kafkatest/tests/connect/templates/connect-distributed.properties +++ b/tests/kafkatest/tests/connect/templates/connect-distributed.properties @@ -20,6 +20,8 @@ bootstrap.servers={{ kafka.bootstrap_servers(kafka.security_config.security_prot group.id={{ group|default("connect-cluster") }} +exactly.once.source.support={{ EXACTLY_ONCE_SOURCE_SUPPORT|default("disabled") }} + connect.protocol={{ CONNECT_PROTOCOL|default("sessioned") }} scheduled.rebalance.max.delay.ms={{ SCHEDULED_REBALANCE_MAX_DELAY_MS|default(60000) }}