From 1d2999fc8f255ba671c98ceffa309e2eaf71e9ec Mon Sep 17 00:00:00 2001 From: David Arthur Date: Thu, 16 Mar 2023 13:42:20 -0400 Subject: [PATCH 01/27] Add pre-migration state for the KRaft controller --- .../kafka/zk/ZkMigrationIntegrationTest.scala | 7 + .../controller/MigrationControlManager.java | 53 +++- .../controller/ProducerIdControlManager.java | 2 +- .../kafka/controller/QuorumController.java | 229 +++++++++++++----- .../migration/KRaftMigrationDriver.java | 40 ++- .../metadata/migration/ZkMigrationState.java | 6 + .../metadata/migration/ZkRecordConsumer.java | 3 +- .../controller/QuorumControllerTest.java | 35 +++ .../migration/KRaftMigrationDriverTest.java | 6 +- 9 files changed, 308 insertions(+), 73 deletions(-) diff --git a/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala b/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala index 231bf01c91743..0bf5a9c1f36eb 100644 --- a/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala @@ -34,6 +34,7 @@ import org.apache.kafka.metadata.migration.ZkMigrationLeadershipState import org.apache.kafka.raft.RaftConfig import org.apache.kafka.server.common.{ApiMessageAndVersion, MetadataVersion, ProducerIdsBlock} import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertNotNull, assertTrue} +import org.junit.jupiter.api.Timeout import org.junit.jupiter.api.extension.ExtendWith import org.slf4j.LoggerFactory @@ -44,6 +45,7 @@ import scala.collection.Seq import scala.jdk.CollectionConverters._ @ExtendWith(value = Array(classOf[ClusterTestExtensions])) +@Timeout(120) class ZkMigrationIntegrationTest { val log = LoggerFactory.getLogger(classOf[ZkMigrationIntegrationTest]) @@ -64,6 +66,11 @@ class ZkMigrationIntegrationTest { } } + /** + * Test ZkMigrationClient against a real ZooKeeper-backed Kafka cluster. This test creates a ZK cluster + * and modifies data using AdminClient. The ZkMigrationClient is then used to read the metadata from ZK + * as would happen during a migration. The generated records are then verified. + */ @ClusterTest(brokers = 3, clusterType = Type.ZK, metadataVersion = MetadataVersion.IBP_3_4_IV0) def testMigrate(clusterInstance: ClusterInstance): Unit = { val admin = clusterInstance.createAdminClient() diff --git a/metadata/src/main/java/org/apache/kafka/controller/MigrationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/MigrationControlManager.java index 7f477785129c1..e07bbd9826c0a 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/MigrationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/MigrationControlManager.java @@ -17,22 +17,69 @@ package org.apache.kafka.controller; import org.apache.kafka.common.metadata.ZkMigrationStateRecord; +import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.metadata.migration.ZkMigrationState; import org.apache.kafka.timeline.SnapshotRegistry; import org.apache.kafka.timeline.TimelineObject; +import org.slf4j.Logger; + +import java.util.Optional; public class MigrationControlManager { private final TimelineObject zkMigrationState; - MigrationControlManager(SnapshotRegistry snapshotRegistry) { - zkMigrationState = new TimelineObject<>(snapshotRegistry, ZkMigrationState.NONE); + private final Logger log; + + MigrationControlManager( + SnapshotRegistry snapshotRegistry, + LogContext logContext + ) { + zkMigrationState = new TimelineObject<>(snapshotRegistry, ZkMigrationState.UNINITIALIZED); + log = logContext.logger(MigrationControlManager.class); } ZkMigrationState zkMigrationState() { return zkMigrationState.get(); } + boolean inPreMigrationMode() { + return zkMigrationState.get() == ZkMigrationState.PRE_MIGRATION; + } + void replay(ZkMigrationStateRecord record) { - zkMigrationState.set(ZkMigrationState.of(record.zkMigrationState())); + ZkMigrationState recordState = ZkMigrationState.of(record.zkMigrationState()); + // From uninitialized, only allow PRE_MIGRATION + if (zkMigrationState.get().equals(ZkMigrationState.UNINITIALIZED)) { + if (recordState.equals(ZkMigrationState.PRE_MIGRATION) || recordState.equals(ZkMigrationState.NONE)) { + log.info("Initializing ZK migration state as {}", recordState); + zkMigrationState.set(recordState); + } else { + throw new IllegalStateException("The first migration state seen can only be PRE_MIGRATION or NONE, not " + recordState.name()); + } + } else { + switch (zkMigrationState.get()) { + case NONE: + throw new IllegalStateException("Cannot ever change migration state from NONE"); + case PRE_MIGRATION: + if (recordState.equals(ZkMigrationState.MIGRATION)) { + log.info("Transitioning ZK migration state to {}", recordState); + zkMigrationState.set(recordState); + } else { + throw new IllegalStateException("Cannot change migration state from PRE_MIGRATION to " + recordState); + } + break; + case MIGRATION: + if (recordState.equals(ZkMigrationState.POST_MIGRATION)) { + log.info("Transitioning ZK migration state to {}", recordState); + zkMigrationState.set(recordState); + } else { + throw new IllegalStateException("Cannot change migration state from MIGRATION to " + recordState); + } + break; + case POST_MIGRATION: + throw new IllegalStateException("Cannot ever change migration state from POST_MIGRATION"); + } + } + } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ProducerIdControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ProducerIdControlManager.java index 2c23bba9dd77f..932af0c2e754e 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ProducerIdControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ProducerIdControlManager.java @@ -69,7 +69,7 @@ void replay(ProducerIdsRecord record) { ProducerIdsBlock nextBlock = nextProducerBlock.get(); if (nextBlock != ProducerIdsBlock.EMPTY && record.nextProducerId() <= nextBlock.firstProducerId()) { throw new RuntimeException("Next Producer ID from replayed record (" + record.nextProducerId() + ")" + - " is not greater than current next Producer ID (" + nextBlock.firstProducerId() + ")"); + " is not greater than current next Producer ID in block (" + nextBlock + ")"); } else { nextProducerBlock.set(new ProducerIdsBlock(record.brokerId(), record.nextProducerId(), ProducerIdsBlock.PRODUCER_ID_BLOCK_SIZE)); brokerEpoch.set(record.brokerEpoch()); diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 71a1dbc4f729b..cf0c8ca1ccb73 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -110,6 +110,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map.Entry; @@ -130,6 +131,8 @@ import static java.util.concurrent.TimeUnit.MICROSECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.apache.kafka.controller.QuorumController.ControllerOperationFlag.DOES_NOT_UPDATE_QUEUE_TIME; +import static org.apache.kafka.controller.QuorumController.ControllerOperationFlag.RUNS_IN_PREMIGRATION; /** @@ -573,6 +576,11 @@ ConfigurationControlManager configurationControl() { return configurationControl; } + // Visible for testing + MigrationControlManager migrationControl() { + return zkMigrationControlManager; + } + public ZkRecordConsumer zkRecordConsumer() { return zkRecordConsumer; } @@ -591,6 +599,26 @@ CompletableFuture appendReadEvent( return event.future(); } + enum ControllerOperationFlag { + /** + * A flag that signifies that this operation should not update the event queue time metric. + * We use this when the event was not appended to the queue. + */ + DOES_NOT_UPDATE_QUEUE_TIME, + + /** + * A flag that signifies that this operation can be processed when in pre-migration mode. + * Operations without this flag will always return NOT_CONTROLLER when invoked in premigration + * mode. + *

+ * In pre-migration mode, we are still waiting to load the metadata from Apache + * ZooKeeper into the metadata log. Therefore, the metadata log is mostly empty, + * even though the cluster really does have metadata. Very few operations should + * use this flag. + */ + RUNS_IN_PREMIGRATION + } + interface ControllerWriteOperation { /** * Generate the metadata records needed to implement this controller write @@ -628,19 +656,19 @@ class ControllerWriteEvent implements EventQueue.Event, DeferredEvent { private final CompletableFuture future; private final ControllerWriteOperation op; private final long eventCreatedTimeNs = time.nanoseconds(); - private final boolean deferred; + private final EnumSet flags; private OptionalLong startProcessingTimeNs = OptionalLong.empty(); private ControllerResultAndOffset resultAndOffset; - ControllerWriteEvent(String name, ControllerWriteOperation op) { - this(name, op, false); - } - - ControllerWriteEvent(String name, ControllerWriteOperation op, boolean deferred) { + ControllerWriteEvent( + String name, + ControllerWriteOperation op, + EnumSet flags + ) { this.name = name; this.future = new CompletableFuture(); this.op = op; - this.deferred = deferred; + this.flags = flags; this.resultAndOffset = null; } @@ -651,13 +679,18 @@ CompletableFuture future() { @Override public void run() throws Exception { long now = time.nanoseconds(); - if (!deferred) { + if (!flags.contains(DOES_NOT_UPDATE_QUEUE_TIME)) { // We exclude deferred events from the event queue time metric to prevent // incorrectly including the deferral time in the queue time. controllerMetrics.updateEventQueueTime(NANOSECONDS.toMillis(now - eventCreatedTimeNs)); } int controllerEpoch = curClaimEpoch; - if (!isActiveController()) { + if (!isActiveController(controllerEpoch)) { + throw newNotControllerException(); + } + if (zkMigrationControlManager.inPreMigrationMode() && !flags.contains(RUNS_IN_PREMIGRATION)) { + log.info("Cannot run write operation {} in premigration mode. Returning " + + "NOT_CONTROLLER.", name); throw newNotControllerException(); } startProcessingTimeNs = OptionalLong.of(now); @@ -818,10 +851,21 @@ static long appendRecords( } } - CompletableFuture appendWriteEvent(String name, - OptionalLong deadlineNs, - ControllerWriteOperation op) { - ControllerWriteEvent event = new ControllerWriteEvent<>(name, op); + CompletableFuture appendWriteEvent( + String name, + OptionalLong deadlineNs, + ControllerWriteOperation op + ) { + return appendWriteEvent(name, deadlineNs, op, EnumSet.noneOf(ControllerOperationFlag.class)); + } + + CompletableFuture appendWriteEvent( + String name, + OptionalLong deadlineNs, + ControllerWriteOperation op, + EnumSet flags + ) { + ControllerWriteEvent event = new ControllerWriteEvent<>(name, op, flags); if (deadlineNs.isPresent()) { queue.appendWithDeadline(deadlineNs.getAsLong(), event); } else { @@ -850,15 +894,8 @@ public void processBatchEndOffset(long offset) { } @Override public void beginMigration() { + log.info("Starting ZK Migration"); // TODO use KIP-868 transaction - ControllerWriteEvent event = new ControllerWriteEvent<>("Begin ZK Migration", - new MigrationWriteOperation( - Collections.singletonList( - new ApiMessageAndVersion( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()), - ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION) - ))); - queue.append(event); } @Override @@ -869,19 +906,30 @@ public CompletableFuture acceptBatch(List recordBatch) return future; } ControllerWriteEvent batchEvent = new ControllerWriteEvent<>("ZK Migration Batch", - new MigrationWriteOperation(recordBatch)); + new MigrationWriteOperation(recordBatch), EnumSet.of(RUNS_IN_PREMIGRATION)); queue.append(batchEvent); return batchEvent.future; } @Override - public OffsetAndEpoch completeMigration() { - // TODO write migration record, use KIP-868 transaction - return highestMigrationRecordOffset; + public CompletableFuture completeMigration() { + log.info("Completing ZK Migration"); + // TODO use KIP-868 transaction + ControllerWriteEvent event = new ControllerWriteEvent<>("Complete ZK Migration", + new MigrationWriteOperation( + Collections.singletonList( + new ApiMessageAndVersion( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()), + ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION) + )), + EnumSet.of(RUNS_IN_PREMIGRATION)); + queue.append(event); + return event.future.thenApply(__ -> highestMigrationRecordOffset); } @Override public void abortMigration() { + log.error("Aborting ZK Migration"); // TODO use KIP-868 transaction } } @@ -1088,7 +1136,11 @@ private void maybeCompleteAuthorizerInitialLoad() { } private boolean isActiveController() { - return curClaimEpoch != -1; + return isActiveController(curClaimEpoch); + } + + private boolean isActiveController(int claimEpoch) { + return claimEpoch != -1; } private void updateWriteOffset(long offset) { @@ -1127,7 +1179,8 @@ private void claim(int epoch) { // of the queue rather than the end (hence prepend rather than append). It's also // important not to use prepend for anything else, to preserve the ordering here. queue.prepend(new ControllerWriteEvent<>("completeActivation[" + epoch + "]", - new CompleteActivationEvent())); + new CompleteActivationEvent(), + EnumSet.of(DOES_NOT_UPDATE_QUEUE_TIME, RUNS_IN_PREMIGRATION))); } catch (Throwable e) { fatalFaultHandler.handleFault("exception while claiming leadership", e); } @@ -1145,9 +1198,55 @@ public ControllerResult generateRecordsAndResult() throws Exception { "at metadata.version {} from {}.", bootstrapMetadata.records().size(), bootstrapMetadata.metadataVersion(), bootstrapMetadata.source()); records.addAll(bootstrapMetadata.records()); - } else if (featureControl.metadataVersion().equals(MetadataVersion.MINIMUM_KRAFT_VERSION)) { - log.info("No metadata.version feature level record was found in the log. " + - "Treating the log as version {}.", MetadataVersion.MINIMUM_KRAFT_VERSION); + + // Initialize the log with a ZkMigrationState + if (bootstrapMetadata.metadataVersion().isMigrationSupported()) { + if (zkMigrationEnabled) { + log.info("Writing ZkMigrationState of PRE_MIGRATION to the log, since migrations are enabled."); + records.add(new ApiMessageAndVersion( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()), + ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION + )); + } else { + log.debug("Writing ZkMigrationState of NONE to the log, since migrations are not enabled."); + records.add(new ApiMessageAndVersion( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()), + ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION + )); + } + } + } else { + // Logs have been replayed. We need to initialize some things here if upgrading from older KRaft versions + if (featureControl.metadataVersion().equals(MetadataVersion.MINIMUM_KRAFT_VERSION)) { + log.info("No metadata.version feature level record was found in the log. " + + "Treating the log as version {}.", MetadataVersion.MINIMUM_KRAFT_VERSION); + } + + if (featureControl.metadataVersion().isMigrationSupported()) { + // If there are records in the log, we cannot be in the middle of a migration + switch (zkMigrationControlManager.zkMigrationState()) { + case UNINITIALIZED: + log.debug("Writing a ZkMigrationState of NONE to the log to indicate this cluster was not migrated from ZK."); + records.add(new ApiMessageAndVersion( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()), + ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION + )); + break; + case NONE: + log.debug("Read a ZkMigrationState of NONE from the log indicating this cluster was never migrated from ZK."); + if (zkMigrationEnabled) { + log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); + } + break; + case POST_MIGRATION: + log.debug("Read a ZkMigrationState of POST_MIGRATION from the log indicating this cluster was previously migrated from ZK."); + break; + default: + // Once we have support for metadata transactions, this failure case will only exist for + // errant ZkMigrationStateRecord that are encountered outside a transaction. + throw fatalFaultHandler.handleFault("Detected an in-progress migration during startup, cannot continue."); + } + } } return ControllerResult.atomicOf(records, null); } @@ -1206,9 +1305,16 @@ private void renounce() { } } - private void scheduleDeferredWriteEvent(String name, long deadlineNs, - ControllerWriteOperation op) { - ControllerWriteEvent event = new ControllerWriteEvent<>(name, op, true); + private void scheduleDeferredWriteEvent( + String name, + long deadlineNs, + ControllerWriteOperation op, + EnumSet flags + ) { + if (!flags.contains(DOES_NOT_UPDATE_QUEUE_TIME)) { + throw new RuntimeException("deferred events should not update the queue time."); + } + ControllerWriteEvent event = new ControllerWriteEvent<>(name, op, flags); queue.scheduleDeferred(name, new EarliestDeadlineFunction(deadlineNs), event); event.future.exceptionally(e -> { if (e instanceof UnknownServerException && e.getCause() != null && @@ -1224,7 +1330,7 @@ private void scheduleDeferredWriteEvent(String name, long deadlineNs, log.error("Unexpected exception while executing deferred write event {}. " + "Rescheduling for a minute from now.", name, e); scheduleDeferredWriteEvent(name, - deadlineNs + NANOSECONDS.convert(1, TimeUnit.MINUTES), op); + deadlineNs + NANOSECONDS.convert(1, TimeUnit.MINUTES), op, flags); return null; }); } @@ -1237,13 +1343,15 @@ private void rescheduleMaybeFenceStaleBrokers() { cancelMaybeFenceReplicas(); return; } - scheduleDeferredWriteEvent(MAYBE_FENCE_REPLICAS, nextCheckTimeNs, () -> { - ControllerResult result = replicationControl.maybeFenceOneStaleBroker(); - // This following call ensures that if there are multiple brokers that - // are currently stale, then fencing for them is scheduled immediately - rescheduleMaybeFenceStaleBrokers(); - return result; - }); + scheduleDeferredWriteEvent(MAYBE_FENCE_REPLICAS, nextCheckTimeNs, + () -> { + ControllerResult result = replicationControl.maybeFenceOneStaleBroker(); + // This following call ensures that if there are multiple brokers that + // are currently stale, then fencing for them is scheduled immediately + rescheduleMaybeFenceStaleBrokers(); + return result; + }, + EnumSet.of(DOES_NOT_UPDATE_QUEUE_TIME, RUNS_IN_PREMIGRATION)); } private void cancelMaybeFenceReplicas() { @@ -1255,7 +1363,8 @@ private void cancelMaybeFenceReplicas() { private void maybeScheduleNextBalancePartitionLeaders() { if (imbalancedScheduled != ImbalanceSchedule.SCHEDULED && leaderImbalanceCheckIntervalNs.isPresent() && - replicationControl.arePartitionLeadersImbalanced()) { + replicationControl.arePartitionLeadersImbalanced() && + zkMigrationControlManager.inPreMigrationMode()) { log.debug( "Scheduling write event for {} because scheduled ({}), checkIntervalNs ({}) and isImbalanced ({})", @@ -1281,7 +1390,7 @@ private void maybeScheduleNextBalancePartitionLeaders() { // generated by a ControllerWriteEvent have been applied. return result; - }, true); + }, EnumSet.of(DOES_NOT_UPDATE_QUEUE_TIME)); long delayNs = time.nanoseconds(); if (imbalancedScheduled == ImbalanceSchedule.DEFERRED) { @@ -1329,7 +1438,7 @@ private void maybeScheduleNextWriteNoOpRecord() { null ); }, - true + EnumSet.of(DOES_NOT_UPDATE_QUEUE_TIME, RUNS_IN_PREMIGRATION) ); long delayNs = time.nanoseconds() + maxIdleIntervalNs.getAsLong(); @@ -1423,7 +1532,7 @@ private void replay(ApiMessage message, Optional snapshotId, lon // NoOpRecord is an empty record and doesn't need to be replayed break; case ZK_MIGRATION_STATE_RECORD: - // TODO handle this + zkMigrationControlManager.replay((ZkMigrationStateRecord) message); break; default: throw new RuntimeException("Unhandled record type " + type); @@ -1556,6 +1665,8 @@ private void resetToEmptyState() { */ private final AclControlManager aclControlManager; + private final MigrationControlManager zkMigrationControlManager; + /** * Tracks replaying the log. * This must be accessed only by the event queue thread. @@ -1659,6 +1770,11 @@ private enum ImbalanceSchedule { */ private final int maxRecordsPerBatch; + /** + * True if the controller was configured to allow Zookeeper migrations. + */ + private final boolean zkMigrationEnabled; + private QuorumController( FaultHandler fatalFaultHandler, LogContext logContext, @@ -1760,6 +1876,8 @@ private QuorumController( this.curClaimEpoch = -1; this.needToCompleteAuthorizerLoad = authorizer.isPresent(); this.zkRecordConsumer = new MigrationRecordConsumer(); + this.zkMigrationEnabled = zkMigrationEnabled; + this.zkMigrationControlManager = new MigrationControlManager(snapshotRegistry, logContext); updateWriteOffset(-1); resetToEmptyState(); @@ -1811,7 +1929,7 @@ public CompletableFuture unregisterBroker( int brokerId ) { return appendWriteEvent("unregisterBroker", context.deadlineNs(), - () -> replicationControl.unregisterBroker(brokerId)); + () -> replicationControl.unregisterBroker(brokerId), EnumSet.of(RUNS_IN_PREMIGRATION)); } @Override @@ -1986,7 +2104,8 @@ public void processBatchEndOffset(long offset) { maybeUpdateControlledShutdownOffset(brokerId, offset); } } - }); + }, + EnumSet.of(RUNS_IN_PREMIGRATION)); } @Override @@ -1994,13 +2113,15 @@ public CompletableFuture registerBroker( ControllerRequestContext context, BrokerRegistrationRequestData request ) { - return appendWriteEvent("registerBroker", context.deadlineNs(), () -> { - ControllerResult result = clusterControl. - registerBroker(request, writeOffset + 1, featureControl. - finalizedFeatures(Long.MAX_VALUE)); - rescheduleMaybeFenceStaleBrokers(); - return result; - }); + return appendWriteEvent("registerBroker", context.deadlineNs(), + () -> { + ControllerResult result = clusterControl. + registerBroker(request, writeOffset + 1, featureControl. + finalizedFeatures(Long.MAX_VALUE)); + rescheduleMaybeFenceStaleBrokers(); + return result; + }, + EnumSet.of(RUNS_IN_PREMIGRATION)); } @Override diff --git a/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java b/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java index b29edc25807f7..5590880f7caba 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java +++ b/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java @@ -33,8 +33,9 @@ import org.apache.kafka.raft.OffsetAndEpoch; import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.fault.FaultHandler; +import org.apache.kafka.server.util.Deadline; +import org.apache.kafka.server.util.FutureUtils; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.Collections; @@ -42,8 +43,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; @@ -59,7 +60,15 @@ public class KRaftMigrationDriver implements MetadataPublisher { private final static Consumer NO_OP_HANDLER = ex -> { }; + /** + * When waiting for the metadata layer to commit batches, we block the migration driver thread for this + * amount of time. A large value is selected to avoid timeouts in the common case, but prevent us from + * blocking indefinitely. + */ + private final static int METADATA_COMMIT_MAX_WAIT_MS = 300_000; + private final Time time; + private final LogContext logContext; private final Logger log; private final int nodeId; private final MigrationClient zkMigrationClient; @@ -90,10 +99,11 @@ public KRaftMigrationDriver( this.zkMigrationClient = zkMigrationClient; this.propagator = propagator; this.time = Time.SYSTEM; - this.log = LoggerFactory.getLogger(KRaftMigrationDriver.class); + this.logContext = new LogContext(String.format("[KRaftMigrationDriver nodeId=%d] ", nodeId)); + this.log = this.logContext.logger(KRaftMigrationDriver.class); this.migrationState = MigrationDriverState.UNINITIALIZED; this.migrationLeadershipState = ZkMigrationLeadershipState.EMPTY; - this.eventQueue = new KafkaEventQueue(Time.SYSTEM, new LogContext("KRaftMigrationDriver"), "kraft-migration"); + this.eventQueue = new KafkaEventQueue(Time.SYSTEM, logContext, "kraft-migration"); this.image = MetadataImage.EMPTY; this.leaderAndEpoch = LeaderAndEpoch.UNKNOWN; this.initialZkLoadHandler = initialZkLoadHandler; @@ -344,7 +354,11 @@ public void run() throws Exception { break; default: if (!isActive) { - apply("KRaftLeaderEvent is not active", state -> ZkMigrationLeadershipState.EMPTY); + apply("KRaftLeaderEvent is not active", state -> + state.withNewKRaftController( + leaderAndEpoch.leaderId().orElse(ZkMigrationLeadershipState.EMPTY.kraftControllerId()), + leaderAndEpoch.epoch()) + ); transitionTo(MigrationDriverState.INACTIVE); } else { // Apply the new KRaft state @@ -435,15 +449,19 @@ public void run() throws Exception { log.info("Migrating {} records from ZK", batch.size()); } CompletableFuture future = zkRecordConsumer.acceptBatch(batch); + FutureUtils.waitWithLogging(KRaftMigrationDriver.this.log, KRaftMigrationDriver.this.logContext.logPrefix(), + "the metadata layer to commit migration record batch", + future, Deadline.fromDelay(time, METADATA_COMMIT_MAX_WAIT_MS, TimeUnit.MILLISECONDS), time); count.addAndGet(batch.size()); - future.get(); - } catch (InterruptedException e) { + } catch (Throwable e) { throw new RuntimeException(e); - } catch (ExecutionException e) { - throw new RuntimeException(e.getCause()); } }, brokersInMetadata::add); - OffsetAndEpoch offsetAndEpochAfterMigration = zkRecordConsumer.completeMigration(); + CompletableFuture completeMigrationFuture = zkRecordConsumer.completeMigration(); + OffsetAndEpoch offsetAndEpochAfterMigration = FutureUtils.waitWithLogging( + KRaftMigrationDriver.this.log, KRaftMigrationDriver.this.logContext.logPrefix(), + "the metadata layer to complete the migration", + completeMigrationFuture, Deadline.fromDelay(time, METADATA_COMMIT_MAX_WAIT_MS, TimeUnit.MILLISECONDS), time); log.info("Completed migration of metadata from Zookeeper to KRaft. A total of {} metadata records were " + "generated. The current metadata offset is now {} with an epoch of {}. Saw {} brokers in the " + "migrated metadata {}.", @@ -455,7 +473,7 @@ public void run() throws Exception { ZkMigrationLeadershipState newState = migrationLeadershipState.withKRaftMetadataOffsetAndEpoch( offsetAndEpochAfterMigration.offset(), offsetAndEpochAfterMigration.epoch()); - apply("Migrate metadata from Zk", state -> zkMigrationClient.setMigrationRecoveryState(newState)); + apply("Finished migrating ZK data", state -> zkMigrationClient.setMigrationRecoveryState(newState)); transitionTo(MigrationDriverState.KRAFT_CONTROLLER_TO_BROKER_COMM); } catch (Throwable t) { zkRecordConsumer.abortMigration(); diff --git a/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkMigrationState.java b/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkMigrationState.java index f8404cfe5d141..8debe053b8db8 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkMigrationState.java +++ b/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkMigrationState.java @@ -27,6 +27,12 @@ * @see org.apache.kafka.common.metadata.ZkMigrationStateRecord */ public enum ZkMigrationState { + /** + * This is a synthetic value used internally by the controller to indicate that no decision has + * been made about the state of a ZK migration. This value should _not_ be written into the log. + */ + UNINITIALIZED((byte) -1), + /** * The cluster was created in KRaft mode. A cluster that was created in ZK mode can never attain * this state; the endpoint of migration is POST_MIGRATION, instead. diff --git a/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkRecordConsumer.java b/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkRecordConsumer.java index 4e35b719d14ba..312c06971dbee 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkRecordConsumer.java +++ b/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkRecordConsumer.java @@ -21,10 +21,11 @@ import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; public interface ZkRecordConsumer { void beginMigration(); CompletableFuture acceptBatch(List recordBatch); - OffsetAndEpoch completeMigration(); + CompletableFuture completeMigration(); void abortMigration(); } diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java index f85b1f921da56..9f1963afa6acc 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -90,6 +90,7 @@ import org.apache.kafka.metadata.PartitionRegistration; import org.apache.kafka.metadata.authorizer.StandardAuthorizer; import org.apache.kafka.metadata.bootstrap.BootstrapMetadata; +import org.apache.kafka.metadata.migration.ZkMigrationState; import org.apache.kafka.metadata.util.BatchFileWriter; import org.apache.kafka.metalog.LocalLogManager; import org.apache.kafka.metalog.LocalLogManagerTestEnv; @@ -1320,4 +1321,38 @@ public void testAppendRecordsAtomically() { 2, appender)).getMessage()); } + + @Test + public void testBootstrapZkMigrationRecord() throws Exception { + assertEquals(ZkMigrationState.PRE_MIGRATION, + checkBootstrapZkMigrationRecord(MetadataVersion.IBP_3_4_IV0, true)); + + assertEquals(ZkMigrationState.NONE, + checkBootstrapZkMigrationRecord(MetadataVersion.IBP_3_4_IV0, false)); + + assertEquals(ZkMigrationState.UNINITIALIZED, + checkBootstrapZkMigrationRecord(MetadataVersion.IBP_3_3_IV0, true)); + + assertEquals(ZkMigrationState.UNINITIALIZED, + checkBootstrapZkMigrationRecord(MetadataVersion.IBP_3_3_IV0, false)); + } + + public ZkMigrationState checkBootstrapZkMigrationRecord( + MetadataVersion metadataVersion, + boolean migrationEnabled + ) throws Exception { + try ( + LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv.Builder(1).build(); + QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv.Builder(logEnv). + setControllerBuilderInitializer(controllerBuilder -> { + controllerBuilder.setZkMigrationEnabled(migrationEnabled); + }). + setBootstrapMetadata(BootstrapMetadata.fromVersion(metadataVersion, "test")). + build(); + ) { + QuorumController active = controlEnv.activeController(); + return active.appendReadEvent("read migration state", OptionalLong.empty(), + () -> active.migrationControl().zkMigrationState()).get(30, TimeUnit.SECONDS); + } + } } diff --git a/metadata/src/test/java/org/apache/kafka/metadata/migration/KRaftMigrationDriverTest.java b/metadata/src/test/java/org/apache/kafka/metadata/migration/KRaftMigrationDriverTest.java index 01b749ebc9a1e..a4a5eaf341ef6 100644 --- a/metadata/src/test/java/org/apache/kafka/metadata/migration/KRaftMigrationDriverTest.java +++ b/metadata/src/test/java/org/apache/kafka/metadata/migration/KRaftMigrationDriverTest.java @@ -57,12 +57,12 @@ public void beginMigration() { @Override public CompletableFuture acceptBatch(List recordBatch) { - return null; + return CompletableFuture.completedFuture(null); } @Override - public OffsetAndEpoch completeMigration() { - return new OffsetAndEpoch(100, 1); + public CompletableFuture completeMigration() { + return CompletableFuture.completedFuture(new OffsetAndEpoch(100, 1)); } @Override From 64e9df99608e23cd3f878188d56c840eb1cfedd0 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Thu, 16 Mar 2023 14:29:34 -0400 Subject: [PATCH 02/27] Add integration test for pre-migration --- .../kafka/zk/ZkMigrationIntegrationTest.scala | 26 ++++++++++++++++--- .../BrokerRegistrationRequestTest.scala | 3 ++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala b/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala index 0bf5a9c1f36eb..91b2d46199d73 100644 --- a/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala @@ -23,24 +23,25 @@ import kafka.test.junit.ClusterTestExtensions import kafka.test.junit.ZkClusterInvocationContext.ZkClusterInstance import kafka.testkit.{KafkaClusterTestKit, TestKitNodes} import kafka.utils.TestUtils -import org.apache.kafka.clients.admin.{Admin, AlterClientQuotasResult, AlterConfigOp, AlterConfigsResult, ConfigEntry, NewTopic} +import org.apache.kafka.clients.admin.{Admin, AlterClientQuotasResult, AlterConfigOp, AlterConfigsResult, ConfigEntry, CreateTopicsOptions, NewTopic} import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} import org.apache.kafka.common.Uuid import org.apache.kafka.common.config.{ConfigResource, TopicConfig} +import org.apache.kafka.common.errors.TimeoutException import org.apache.kafka.common.quota.{ClientQuotaAlteration, ClientQuotaEntity} import org.apache.kafka.common.serialization.StringSerializer import org.apache.kafka.image.{MetadataDelta, MetadataImage, MetadataProvenance} import org.apache.kafka.metadata.migration.ZkMigrationLeadershipState import org.apache.kafka.raft.RaftConfig import org.apache.kafka.server.common.{ApiMessageAndVersion, MetadataVersion, ProducerIdsBlock} -import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertNotNull, assertTrue} +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertNotNull, assertThrows, assertTrue} import org.junit.jupiter.api.Timeout import org.junit.jupiter.api.extension.ExtendWith import org.slf4j.LoggerFactory import java.util import java.util.Properties -import java.util.concurrent.TimeUnit +import java.util.concurrent.{ExecutionException, TimeUnit} import scala.collection.Seq import scala.jdk.CollectionConverters._ @@ -263,4 +264,23 @@ class ZkMigrationIntegrationTest { assertTrue(firstProducerIdBlock.firstProducerId() < producerIdBlock.firstProducerId()) } } + + /** + * Start a KRaft cluster with migrations enabled, verify that the controller does not accept metadata changes + * through the RPCs + */ + @ClusterTest(clusterType = Type.KRAFT, brokers = 0, controllers = 1, metadataVersion = MetadataVersion.IBP_3_4_IV0, + serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))) + def testPreMigrationMode(clusterInstance: ClusterInstance): Unit = { + val adminClient = clusterInstance.createAdminClient() + val result = adminClient.createTopics( + java.util.Collections.singleton(new NewTopic("test", 10, 1.asInstanceOf[Short])), + new CreateTopicsOptions().timeoutMs(1000) + ) + + assertEquals( + classOf[TimeoutException], + assertThrows(classOf[ExecutionException], () => result.all().get(5, TimeUnit.MINUTES)).getCause.getClass + ) + } } diff --git a/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala b/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala index 09c74d4e15587..5add636f58931 100644 --- a/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala @@ -22,6 +22,7 @@ import kafka.test.ClusterInstance import kafka.test.annotation.{ClusterConfigProperty, ClusterTest, Type} import kafka.test.junit.ClusterTestExtensions import org.apache.kafka.clients.ClientResponse +import org.apache.kafka.common.errors.TimeoutException import org.apache.kafka.common.message.{BrokerRegistrationRequestData, BrokerRegistrationResponseData} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.ListenerName @@ -35,7 +36,7 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.extension.ExtendWith import org.junit.jupiter.api.{Tag, Timeout} -import java.util.concurrent.{CompletableFuture, TimeUnit, TimeoutException} +import java.util.concurrent.{CompletableFuture, TimeUnit} /** * This test simulates a broker registering with the KRaft quorum under different configurations. From 4d1a9cbc858d6712e75af1c0506686527177481b Mon Sep 17 00:00:00 2001 From: David Arthur Date: Thu, 16 Mar 2023 15:04:59 -0400 Subject: [PATCH 03/27] Add migration state to MetadataImage --- .../unit/kafka/server/MetadataCacheTest.scala | 5 +- .../kafka/server/ReplicaManagerTest.scala | 3 +- .../kafka/controller/QuorumController.java | 15 ++++-- .../org/apache/kafka/image/MetadataDelta.java | 25 ++++++++- .../org/apache/kafka/image/MetadataImage.java | 24 +++++++-- .../apache/kafka/image/ZkMigrationDelta.java | 26 +++++++++ .../apache/kafka/image/ZkMigrationImage.java | 54 +++++++++++++++++++ .../apache/kafka/image/MetadataImageTest.java | 6 ++- 8 files changed, 141 insertions(+), 17 deletions(-) create mode 100644 metadata/src/main/java/org/apache/kafka/image/ZkMigrationDelta.java create mode 100644 metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java diff --git a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala index 9685a613970ce..34a3fb1cbabb9 100644 --- a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala +++ b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala @@ -32,7 +32,7 @@ import org.apache.kafka.common.requests.UpdateMetadataRequest import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.metadata.{BrokerRegistrationChangeRecord, PartitionRecord, RegisterBrokerRecord, RemoveTopicRecord, TopicRecord} import org.apache.kafka.common.metadata.RegisterBrokerRecord.{BrokerEndpoint, BrokerEndpointCollection} -import org.apache.kafka.image.{ClusterImage, MetadataDelta, MetadataImage, MetadataProvenance} +import org.apache.kafka.image.{ClusterImage, MetadataDelta, MetadataImage, MetadataProvenance, ZkMigrationImage} import org.apache.kafka.server.common.MetadataVersion import org.junit.jupiter.api.Assertions._ @@ -72,7 +72,8 @@ object MetadataCacheTest { image.clientQuotas(), image.producerIds(), image.acls(), - image.scram()) + image.scram(), + ZkMigrationImage.EMPTY) val delta = new MetadataDelta.Builder().setImage(partialImage).build() def toRecord(broker: UpdateMetadataBroker): RegisterBrokerRecord = { diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 047c5a5ac842a..bbda8d5a86306 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -4363,7 +4363,8 @@ class ReplicaManagerTest { ClientQuotasImage.EMPTY, ProducerIdsImage.EMPTY, AclsImage.EMPTY, - ScramImage.EMPTY + ScramImage.EMPTY, + ZkMigrationImage.EMPTY ) } diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index cf0c8ca1ccb73..566d8c34b2315 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -1199,8 +1199,8 @@ public ControllerResult generateRecordsAndResult() throws Exception { bootstrapMetadata.metadataVersion(), bootstrapMetadata.source()); records.addAll(bootstrapMetadata.records()); - // Initialize the log with a ZkMigrationState if (bootstrapMetadata.metadataVersion().isMigrationSupported()) { + // Initialize the log with a ZkMigrationState if (zkMigrationEnabled) { log.info("Writing ZkMigrationState of PRE_MIGRATION to the log, since migrations are enabled."); records.add(new ApiMessageAndVersion( @@ -1219,26 +1219,31 @@ public ControllerResult generateRecordsAndResult() throws Exception { // Logs have been replayed. We need to initialize some things here if upgrading from older KRaft versions if (featureControl.metadataVersion().equals(MetadataVersion.MINIMUM_KRAFT_VERSION)) { log.info("No metadata.version feature level record was found in the log. " + - "Treating the log as version {}.", MetadataVersion.MINIMUM_KRAFT_VERSION); + "Treating the log as version {}.", MetadataVersion.MINIMUM_KRAFT_VERSION); } if (featureControl.metadataVersion().isMigrationSupported()) { - // If there are records in the log, we cannot be in the middle of a migration switch (zkMigrationControlManager.zkMigrationState()) { case UNINITIALIZED: + // No ZkMigrationState record seen, put a NONE in the log log.debug("Writing a ZkMigrationState of NONE to the log to indicate this cluster was not migrated from ZK."); + if (zkMigrationEnabled) { + log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); + } records.add(new ApiMessageAndVersion( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()), - ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()), + ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION )); break; case NONE: + // This is a non-migrated KRaft cluster log.debug("Read a ZkMigrationState of NONE from the log indicating this cluster was never migrated from ZK."); if (zkMigrationEnabled) { log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); } break; case POST_MIGRATION: + // This is a migrated KRaft cluster log.debug("Read a ZkMigrationState of POST_MIGRATION from the log indicating this cluster was previously migrated from ZK."); break; default: diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java b/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java index 05fc879bfe997..205a1d2b10fec 100644 --- a/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java @@ -35,6 +35,7 @@ import org.apache.kafka.common.metadata.UnfenceBrokerRecord; import org.apache.kafka.common.metadata.UnregisterBrokerRecord; import org.apache.kafka.common.metadata.UserScramCredentialRecord; +import org.apache.kafka.common.metadata.ZkMigrationStateRecord; import org.apache.kafka.common.protocol.ApiMessage; import org.apache.kafka.server.common.MetadataVersion; @@ -76,6 +77,8 @@ public MetadataDelta build() { private ScramDelta scramDelta = null; + private ZkMigrationDelta migrationDelta = null; + public MetadataDelta(MetadataImage image) { this.image = image; } @@ -158,6 +161,11 @@ public ScramDelta getOrCreateScramDelta() { return scramDelta; } + public ZkMigrationDelta getOrCreateZkMigrationDelta() { + if (migrationDelta == null) migrationDelta = new ZkMigrationDelta(image.migration()); + return migrationDelta; + } + public Optional metadataVersionChanged() { if (featuresDelta == null) { return Optional.empty(); @@ -226,7 +234,7 @@ public void replay(ApiMessage record) { */ break; case ZK_MIGRATION_STATE_RECORD: - // TODO handle this + replay((ZkMigrationStateRecord) record); break; default: throw new RuntimeException("Unknown metadata record type " + type); @@ -312,6 +320,10 @@ public void replay(RemoveUserScramCredentialRecord record) { getOrCreateScramDelta().replay(record); } + public void replay(ZkMigrationStateRecord record) { + getOrCreateZkMigrationDelta().replay(record); + } + /** * Create removal deltas for anything which was in the base image, but which was not * referenced in the snapshot records we just applied. @@ -325,6 +337,7 @@ public void finishSnapshot() { getOrCreateProducerIdsDelta().finishSnapshot(); getOrCreateAclsDelta().finishSnapshot(); getOrCreateScramDelta().finishSnapshot(); + getOrCreateZkMigrationDelta().finishSnapshot(); } public MetadataImage apply(MetadataProvenance provenance) { @@ -376,6 +389,12 @@ public MetadataImage apply(MetadataProvenance provenance) { } else { newScram = scramDelta.apply(); } + ZkMigrationImage newMigration; + if (migrationDelta == null) { + newMigration = image.migration(); + } else { + newMigration = migrationDelta.apply(); + } return new MetadataImage( provenance, newFeatures, @@ -385,7 +404,8 @@ public MetadataImage apply(MetadataProvenance provenance) { newClientQuotas, newProducerIds, newAcls, - newScram + newScram, + newMigration ); } @@ -400,6 +420,7 @@ public String toString() { ", producerIdsDelta=" + producerIdsDelta + ", aclsDelta=" + aclsDelta + ", scramDelta=" + scramDelta + + ", migrationDelta=" + migrationDelta + ')'; } } diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java b/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java index 8643a23e03c58..da4c5fedeea61 100644 --- a/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java @@ -39,7 +39,8 @@ public final class MetadataImage { ClientQuotasImage.EMPTY, ProducerIdsImage.EMPTY, AclsImage.EMPTY, - ScramImage.EMPTY); + ScramImage.EMPTY, + ZkMigrationImage.EMPTY); private final MetadataProvenance provenance; @@ -59,6 +60,8 @@ public final class MetadataImage { private final ScramImage scram; + private final ZkMigrationImage migration; + public MetadataImage( MetadataProvenance provenance, FeaturesImage features, @@ -68,7 +71,8 @@ public MetadataImage( ClientQuotasImage clientQuotas, ProducerIdsImage producerIds, AclsImage acls, - ScramImage scram + ScramImage scram, + ZkMigrationImage migration ) { this.provenance = provenance; this.features = features; @@ -79,6 +83,7 @@ public MetadataImage( this.producerIds = producerIds; this.acls = acls; this.scram = scram; + this.migration = migration; } public boolean isEmpty() { @@ -89,7 +94,8 @@ public boolean isEmpty() { clientQuotas.isEmpty() && producerIds.isEmpty() && acls.isEmpty() && - scram.isEmpty(); + scram.isEmpty() && + migration.isEmpty(); } public MetadataProvenance provenance() { @@ -136,6 +142,10 @@ public ScramImage scram() { return scram; } + public ZkMigrationImage migration() { + return migration; + } + public void write(ImageWriter writer, ImageWriterOptions options) { // Features should be written out first so we can include the metadata.version at the beginning of the // snapshot @@ -147,6 +157,7 @@ public void write(ImageWriter writer, ImageWriterOptions options) { producerIds.write(writer, options); acls.write(writer, options); scram.write(writer, options); + migration.write(writer, options); writer.close(true); } @@ -162,7 +173,8 @@ public boolean equals(Object o) { clientQuotas.equals(other.clientQuotas) && producerIds.equals(other.producerIds) && acls.equals(other.acls) && - scram.equals(other.scram); + scram.equals(other.scram) && + migration.equals(other.migration); } @Override @@ -176,7 +188,8 @@ public int hashCode() { clientQuotas, producerIds, acls, - scram); + scram, + migration); } @Override @@ -191,6 +204,7 @@ public String toString() { ", producerIdsImage=" + producerIds + ", acls=" + acls + ", scram=" + scram + + ", migration=" + migration + ")"; } } diff --git a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationDelta.java b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationDelta.java new file mode 100644 index 0000000000000..ac20cf95910d7 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationDelta.java @@ -0,0 +1,26 @@ +package org.apache.kafka.image; + +import org.apache.kafka.common.metadata.ZkMigrationStateRecord; +import org.apache.kafka.metadata.migration.ZkMigrationState; + +public class ZkMigrationDelta { + private final ZkMigrationImage image; + + private ZkMigrationState updatedState; + + public ZkMigrationDelta(ZkMigrationImage image) { + this.image = image; + } + + public void replay(ZkMigrationStateRecord record) { + this.updatedState = ZkMigrationState.of(record.zkMigrationState()); + } + + public void finishSnapshot() { + // no-op + } + + public ZkMigrationImage apply() { + return new ZkMigrationImage(updatedState); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java new file mode 100644 index 0000000000000..134e39f8155d5 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java @@ -0,0 +1,54 @@ +package org.apache.kafka.image; + +import org.apache.kafka.common.metadata.ZkMigrationStateRecord; +import org.apache.kafka.image.writer.ImageWriter; +import org.apache.kafka.image.writer.ImageWriterOptions; +import org.apache.kafka.metadata.migration.ZkMigrationState; + +import java.util.Objects; + +public class ZkMigrationImage { + public static final ZkMigrationImage EMPTY = new ZkMigrationImage(ZkMigrationState.UNINITIALIZED); + + private final ZkMigrationState state; + + ZkMigrationImage(ZkMigrationState state) { + this.state = state; + } + + public ZkMigrationState migrationState() { + return state; + } + + public boolean isEmpty() { + return this.equals(ZkMigrationImage.EMPTY); + } + + public void write(ImageWriter writer, ImageWriterOptions options) { + if (options.metadataVersion().isMigrationSupported()) { + writer.write(0, new ZkMigrationStateRecord().setZkMigrationState(state.value())); + } else { + options.handleLoss("the ZK Migration state"); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ZkMigrationImage that = (ZkMigrationImage) o; + return state == that.state; + } + + @Override + public int hashCode() { + return Objects.hash(state); + } + + @Override + public String toString() { + return "ZkMigrationImage{" + + "state=" + state + + '}'; + } +} diff --git a/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java b/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java index 2a6e3e6f3e556..a3e5133b70ec9 100644 --- a/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java @@ -44,7 +44,8 @@ public class MetadataImageTest { ClientQuotasImageTest.IMAGE1, ProducerIdsImageTest.IMAGE1, AclsImageTest.IMAGE1, - ScramImageTest.IMAGE1); + ScramImageTest.IMAGE1, + ZkMigrationImage.EMPTY); DELTA1 = new MetadataDelta.Builder(). setImage(IMAGE1). @@ -67,7 +68,8 @@ public class MetadataImageTest { ClientQuotasImageTest.IMAGE2, ProducerIdsImageTest.IMAGE2, AclsImageTest.IMAGE2, - ScramImageTest.IMAGE2); + ScramImageTest.IMAGE2, + ZkMigrationImage.EMPTY); } @Test From 078fcdf5db8a3bff862e753bfd05bea4802d1dc9 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Thu, 16 Mar 2023 17:22:25 -0400 Subject: [PATCH 04/27] checkstyle --- checkstyle/suppressions.xml | 2 +- .../kafka/controller/QuorumController.java | 6 +++--- ...ager.java => ZkMigrationControlManager.java} | 7 +++---- .../apache/kafka/image/ZkMigrationDelta.java | 17 +++++++++++++++++ .../apache/kafka/image/ZkMigrationImage.java | 17 +++++++++++++++++ .../metadata/migration/ZkRecordConsumer.java | 1 - 6 files changed, 41 insertions(+), 9 deletions(-) rename metadata/src/main/java/org/apache/kafka/controller/{MigrationControlManager.java => ZkMigrationControlManager.java} (95%) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 93ae7c30d0f3e..294e069aa561d 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -310,7 +310,7 @@ + files="(ClientQuotasImage|KafkaEventQueue|MetadataDelta|ReplicationControlManager|FeatureControlManager|KRaftMigrationDriver).java"/> zkMigrationState; private final Logger log; - MigrationControlManager( + ZkMigrationControlManager( SnapshotRegistry snapshotRegistry, LogContext logContext ) { zkMigrationState = new TimelineObject<>(snapshotRegistry, ZkMigrationState.UNINITIALIZED); - log = logContext.logger(MigrationControlManager.class); + log = logContext.logger(ZkMigrationControlManager.class); } ZkMigrationState zkMigrationState() { diff --git a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationDelta.java b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationDelta.java index ac20cf95910d7..c197013419cd0 100644 --- a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationDelta.java +++ b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationDelta.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.kafka.image; import org.apache.kafka.common.metadata.ZkMigrationStateRecord; diff --git a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java index 134e39f8155d5..1edf4e4006efd 100644 --- a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java +++ b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.kafka.image; import org.apache.kafka.common.metadata.ZkMigrationStateRecord; diff --git a/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkRecordConsumer.java b/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkRecordConsumer.java index 312c06971dbee..00ffdcb4abba7 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkRecordConsumer.java +++ b/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkRecordConsumer.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; public interface ZkRecordConsumer { void beginMigration(); From c9c3bfc3dc564d515c089cb50cafbae6581b422e Mon Sep 17 00:00:00 2001 From: David Arthur Date: Fri, 17 Mar 2023 09:50:04 -0400 Subject: [PATCH 05/27] Only write ZkMigrationStateRecord if we've seen one --- .../java/org/apache/kafka/image/ZkMigrationImage.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java index 1edf4e4006efd..db041fb6ac137 100644 --- a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java +++ b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java @@ -42,10 +42,12 @@ public boolean isEmpty() { } public void write(ImageWriter writer, ImageWriterOptions options) { - if (options.metadataVersion().isMigrationSupported()) { - writer.write(0, new ZkMigrationStateRecord().setZkMigrationState(state.value())); - } else { - options.handleLoss("the ZK Migration state"); + if (!isEmpty()) { + if (options.metadataVersion().isMigrationSupported()) { + writer.write(0, new ZkMigrationStateRecord().setZkMigrationState(state.value())); + } else { + options.handleLoss("the ZK Migration state"); + } } } From f9608cf8eac6b42bc55f6defab9353e860f7d0f3 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Fri, 17 Mar 2023 09:52:13 -0400 Subject: [PATCH 06/27] Rename some things --- .../unit/kafka/server/MetadataCacheTest.scala | 4 ++-- .../kafka/server/ReplicaManagerTest.scala | 2 +- .../org/apache/kafka/image/MetadataDelta.java | 8 +++---- .../org/apache/kafka/image/MetadataImage.java | 22 +++++++++---------- ...nDelta.java => ZkMigrationStateDelta.java} | 10 ++++----- ...nImage.java => ZkMigrationStateImage.java} | 10 ++++----- .../apache/kafka/image/MetadataImageTest.java | 4 ++-- 7 files changed, 30 insertions(+), 30 deletions(-) rename metadata/src/main/java/org/apache/kafka/image/{ZkMigrationDelta.java => ZkMigrationStateDelta.java} (83%) rename metadata/src/main/java/org/apache/kafka/image/{ZkMigrationImage.java => ZkMigrationStateImage.java} (86%) diff --git a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala index 34a3fb1cbabb9..28e4349387ff3 100644 --- a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala +++ b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala @@ -32,7 +32,7 @@ import org.apache.kafka.common.requests.UpdateMetadataRequest import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.metadata.{BrokerRegistrationChangeRecord, PartitionRecord, RegisterBrokerRecord, RemoveTopicRecord, TopicRecord} import org.apache.kafka.common.metadata.RegisterBrokerRecord.{BrokerEndpoint, BrokerEndpointCollection} -import org.apache.kafka.image.{ClusterImage, MetadataDelta, MetadataImage, MetadataProvenance, ZkMigrationImage} +import org.apache.kafka.image.{ClusterImage, MetadataDelta, MetadataImage, MetadataProvenance, ZkMigrationStateImage} import org.apache.kafka.server.common.MetadataVersion import org.junit.jupiter.api.Assertions._ @@ -73,7 +73,7 @@ object MetadataCacheTest { image.producerIds(), image.acls(), image.scram(), - ZkMigrationImage.EMPTY) + ZkMigrationStateImage.EMPTY) val delta = new MetadataDelta.Builder().setImage(partialImage).build() def toRecord(broker: UpdateMetadataBroker): RegisterBrokerRecord = { diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 0a8ea05b8b9ee..e2c681184973b 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -4365,7 +4365,7 @@ class ReplicaManagerTest { ProducerIdsImage.EMPTY, AclsImage.EMPTY, ScramImage.EMPTY, - ZkMigrationImage.EMPTY + ZkMigrationStateImage.EMPTY ) } diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java b/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java index 205a1d2b10fec..c23603e3b3657 100644 --- a/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java @@ -77,7 +77,7 @@ public MetadataDelta build() { private ScramDelta scramDelta = null; - private ZkMigrationDelta migrationDelta = null; + private ZkMigrationStateDelta migrationDelta = null; public MetadataDelta(MetadataImage image) { this.image = image; @@ -161,8 +161,8 @@ public ScramDelta getOrCreateScramDelta() { return scramDelta; } - public ZkMigrationDelta getOrCreateZkMigrationDelta() { - if (migrationDelta == null) migrationDelta = new ZkMigrationDelta(image.migration()); + public ZkMigrationStateDelta getOrCreateZkMigrationDelta() { + if (migrationDelta == null) migrationDelta = new ZkMigrationStateDelta(image.migration()); return migrationDelta; } @@ -389,7 +389,7 @@ public MetadataImage apply(MetadataProvenance provenance) { } else { newScram = scramDelta.apply(); } - ZkMigrationImage newMigration; + ZkMigrationStateImage newMigration; if (migrationDelta == null) { newMigration = image.migration(); } else { diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java b/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java index da4c5fedeea61..1e6b460174320 100644 --- a/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java @@ -40,7 +40,7 @@ public final class MetadataImage { ProducerIdsImage.EMPTY, AclsImage.EMPTY, ScramImage.EMPTY, - ZkMigrationImage.EMPTY); + ZkMigrationStateImage.EMPTY); private final MetadataProvenance provenance; @@ -60,7 +60,7 @@ public final class MetadataImage { private final ScramImage scram; - private final ZkMigrationImage migration; + private final ZkMigrationStateImage zkMigration; public MetadataImage( MetadataProvenance provenance, @@ -72,7 +72,7 @@ public MetadataImage( ProducerIdsImage producerIds, AclsImage acls, ScramImage scram, - ZkMigrationImage migration + ZkMigrationStateImage migration ) { this.provenance = provenance; this.features = features; @@ -83,7 +83,7 @@ public MetadataImage( this.producerIds = producerIds; this.acls = acls; this.scram = scram; - this.migration = migration; + this.zkMigration = migration; } public boolean isEmpty() { @@ -95,7 +95,7 @@ public boolean isEmpty() { producerIds.isEmpty() && acls.isEmpty() && scram.isEmpty() && - migration.isEmpty(); + zkMigration.isEmpty(); } public MetadataProvenance provenance() { @@ -142,8 +142,8 @@ public ScramImage scram() { return scram; } - public ZkMigrationImage migration() { - return migration; + public ZkMigrationStateImage migration() { + return zkMigration; } public void write(ImageWriter writer, ImageWriterOptions options) { @@ -157,7 +157,7 @@ public void write(ImageWriter writer, ImageWriterOptions options) { producerIds.write(writer, options); acls.write(writer, options); scram.write(writer, options); - migration.write(writer, options); + zkMigration.write(writer, options); writer.close(true); } @@ -174,7 +174,7 @@ public boolean equals(Object o) { producerIds.equals(other.producerIds) && acls.equals(other.acls) && scram.equals(other.scram) && - migration.equals(other.migration); + zkMigration.equals(other.zkMigration); } @Override @@ -189,7 +189,7 @@ public int hashCode() { producerIds, acls, scram, - migration); + zkMigration); } @Override @@ -204,7 +204,7 @@ public String toString() { ", producerIdsImage=" + producerIds + ", acls=" + acls + ", scram=" + scram + - ", migration=" + migration + + ", migration=" + zkMigration + ")"; } } diff --git a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationDelta.java b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateDelta.java similarity index 83% rename from metadata/src/main/java/org/apache/kafka/image/ZkMigrationDelta.java rename to metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateDelta.java index c197013419cd0..4ee138bf8c513 100644 --- a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationDelta.java +++ b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateDelta.java @@ -20,12 +20,12 @@ import org.apache.kafka.common.metadata.ZkMigrationStateRecord; import org.apache.kafka.metadata.migration.ZkMigrationState; -public class ZkMigrationDelta { - private final ZkMigrationImage image; +public class ZkMigrationStateDelta { + private final ZkMigrationStateImage image; private ZkMigrationState updatedState; - public ZkMigrationDelta(ZkMigrationImage image) { + public ZkMigrationStateDelta(ZkMigrationStateImage image) { this.image = image; } @@ -37,7 +37,7 @@ public void finishSnapshot() { // no-op } - public ZkMigrationImage apply() { - return new ZkMigrationImage(updatedState); + public ZkMigrationStateImage apply() { + return new ZkMigrationStateImage(updatedState); } } diff --git a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateImage.java similarity index 86% rename from metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java rename to metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateImage.java index db041fb6ac137..f7aaf59dded48 100644 --- a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationImage.java +++ b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateImage.java @@ -24,12 +24,12 @@ import java.util.Objects; -public class ZkMigrationImage { - public static final ZkMigrationImage EMPTY = new ZkMigrationImage(ZkMigrationState.UNINITIALIZED); +public class ZkMigrationStateImage { + public static final ZkMigrationStateImage EMPTY = new ZkMigrationStateImage(ZkMigrationState.UNINITIALIZED); private final ZkMigrationState state; - ZkMigrationImage(ZkMigrationState state) { + ZkMigrationStateImage(ZkMigrationState state) { this.state = state; } @@ -38,7 +38,7 @@ public ZkMigrationState migrationState() { } public boolean isEmpty() { - return this.equals(ZkMigrationImage.EMPTY); + return this.equals(ZkMigrationStateImage.EMPTY); } public void write(ImageWriter writer, ImageWriterOptions options) { @@ -55,7 +55,7 @@ public void write(ImageWriter writer, ImageWriterOptions options) { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - ZkMigrationImage that = (ZkMigrationImage) o; + ZkMigrationStateImage that = (ZkMigrationStateImage) o; return state == that.state; } diff --git a/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java b/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java index a3e5133b70ec9..89780e3eac30f 100644 --- a/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java @@ -45,7 +45,7 @@ public class MetadataImageTest { ProducerIdsImageTest.IMAGE1, AclsImageTest.IMAGE1, ScramImageTest.IMAGE1, - ZkMigrationImage.EMPTY); + ZkMigrationStateImage.EMPTY); DELTA1 = new MetadataDelta.Builder(). setImage(IMAGE1). @@ -69,7 +69,7 @@ public class MetadataImageTest { ProducerIdsImageTest.IMAGE2, AclsImageTest.IMAGE2, ScramImageTest.IMAGE2, - ZkMigrationImage.EMPTY); + ZkMigrationStateImage.EMPTY); } @Test From e57f996d859f73d9f0bc78b698e972357036776e Mon Sep 17 00:00:00 2001 From: David Arthur Date: Fri, 17 Mar 2023 15:02:43 -0400 Subject: [PATCH 07/27] Make sure we don't start a migration when we're in KRaft mode --- .../migration/KRaftMigrationDriver.java | 103 +++++++++--------- tests/docker/Dockerfile | 2 + .../tests/core/zookeeper_migration_test.py | 80 +++++++++++++- 3 files changed, 132 insertions(+), 53 deletions(-) diff --git a/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java b/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java index 0cbb959e0c238..c9ec4be659b1e 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java +++ b/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java @@ -54,7 +54,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; /** - * This class orchestrates and manages the state related to a ZK to KRaft migration. An event thread is used to + * This class orchestrates and manages the state related to a ZK to KRaft migration. A single event thread is used to * serialize events coming from various threads and listeners. */ public class KRaftMigrationDriver implements MetadataPublisher { @@ -127,19 +127,18 @@ CompletableFuture migrationState() { return stateFuture; } - private void initializeMigrationState() { - log.info("Recovering migration state"); + private void recoverMigrationStateFromZK() { + log.info("Recovering migration state from ZK"); apply("Recovery", zkMigrationClient::getOrCreateMigrationRecoveryState); String maybeDone = migrationLeadershipState.zkMigrationComplete() ? "done" : "not done"; log.info("Recovered migration state {}. ZK migration is {}.", migrationLeadershipState, maybeDone); + + // Once we've recovered the migration state from ZK, install this class as a metadata published + // by calling the initialZkLoadHandler. initialZkLoadHandler.accept(this); - // Let's transition to INACTIVE state and wait for leadership events. - transitionTo(MigrationDriverState.INACTIVE); - } - private boolean isControllerQuorumReadyForMigration() { - // TODO implement this - return true; + // Transition to INACTIVE state and wait for leadership events. + transitionTo(MigrationDriverState.INACTIVE); } private boolean imageDoesNotContainAllBrokers(MetadataImage image, Set brokerIds) { @@ -323,11 +322,11 @@ class PollEvent extends MigrationEvent { public void run() throws Exception { switch (migrationState) { case UNINITIALIZED: - initializeMigrationState(); + recoverMigrationStateFromZK(); break; case INACTIVE: - // Nothing to do when the driver is inactive. We need to wait on the - // controller node's state to move forward. + // Nothing to do when the driver is inactive. We must wait until a KRaftLeaderEvent + // tells informs us that we are the leader. break; case WAIT_FOR_CONTROLLER_QUORUM: eventQueue.append(new WaitForControllerQuorumEvent()); @@ -359,6 +358,11 @@ public void run() throws Exception { } } + /** + * An event generated by a call to {@link MetadataPublisher#onControllerChange}. This will not be called until + * this class is registered with {@link org.apache.kafka.image.loader.MetadataLoader}. The registration happens + * after the migration state is loaded from ZooKeeper in {@link #recoverMigrationStateFromZK}. + */ class KRaftLeaderEvent extends MigrationEvent { private final LeaderAndEpoch leaderAndEpoch; @@ -371,31 +375,21 @@ public void run() throws Exception { // We can either be the active controller or just resigned from being the controller. KRaftMigrationDriver.this.leaderAndEpoch = leaderAndEpoch; boolean isActive = leaderAndEpoch.isLeader(KRaftMigrationDriver.this.nodeId); - switch (migrationState) { - case UNINITIALIZED: - // Poll and retry after initialization - long deadline = time.nanoseconds() + NANOSECONDS.convert(10, SECONDS); - eventQueue.scheduleDeferred( - "poll", - new EventQueue.DeadlineFunction(deadline), - this); - break; - default: - if (!isActive) { - apply("KRaftLeaderEvent is not active", state -> - state.withNewKRaftController( - leaderAndEpoch.leaderId().orElse(ZkMigrationLeadershipState.EMPTY.kraftControllerId()), - leaderAndEpoch.epoch()) - ); - transitionTo(MigrationDriverState.INACTIVE); - } else { - // Apply the new KRaft state - apply("KRaftLeaderEvent is active", state -> state.withNewKRaftController(nodeId, leaderAndEpoch.epoch())); - // Before becoming the controller fo ZkBrokers, we need to make sure the - // Controller Quorum can handle migration. - transitionTo(MigrationDriverState.WAIT_FOR_CONTROLLER_QUORUM); - } - break; + + if (!isActive) { + apply("KRaftLeaderEvent is not active", state -> + state.withNewKRaftController( + leaderAndEpoch.leaderId().orElse(ZkMigrationLeadershipState.EMPTY.kraftControllerId()), + leaderAndEpoch.epoch()) + ); + transitionTo(MigrationDriverState.INACTIVE); + } else { + // Apply the new KRaft state + apply("KRaftLeaderEvent is active", state -> state.withNewKRaftController(nodeId, leaderAndEpoch.epoch())); + + // Before becoming the controller fo ZkBrokers, we need to make sure the + // Controller Quorum can handle migration. + transitionTo(MigrationDriverState.WAIT_FOR_CONTROLLER_QUORUM); } } } @@ -404,18 +398,29 @@ class WaitForControllerQuorumEvent extends MigrationEvent { @Override public void run() throws Exception { - switch (migrationState) { - case WAIT_FOR_CONTROLLER_QUORUM: - if (isControllerQuorumReadyForMigration()) { - log.debug("Controller Quorum is ready for Zk to KRaft migration"); - // Note that leadership would not change here. Hence we do not need to - // `apply` any leadership state change. - transitionTo(MigrationDriverState.WAIT_FOR_BROKERS); - } - break; - default: - // Ignore the event as we're not trying to become controller anymore. - break; + if (migrationState.equals(MigrationDriverState.WAIT_FOR_CONTROLLER_QUORUM)) { + if (image.isEmpty()) { + log.trace("Waiting until we have received metadata before proceeding with migration"); + return; + } + + if (image.migration().isEmpty()) { + log.trace("Waiting until we have received metadata regarding the ZK migration before proceeding with migration"); + return; + } + + if (image.migration().migrationState().equals(ZkMigrationState.NONE)) { + // This error message is used in zookeeper_migration_test.py::TestMigration.test_pre_migration_mode_3_4 + log.error("The controller's ZkMigrationState is NONE which means this cluster should not be migrated from ZooKeeper. " + + "This controller should not be configured with 'zookeeper.metadata.migration.enable' set to true. " + + "Will not proceed with a migration."); + transitionTo(MigrationDriverState.INACTIVE); + } else { + log.debug("Controller Quorum is ready for Zk to KRaft migration"); + // Note that leadership would not change here. Hence, we do not need to + // `apply` any leadership state change. + transitionTo(MigrationDriverState.WAIT_FOR_BROKERS); + } } } } diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 8f5dba086b7c5..9b5c3c6b1cb94 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -67,6 +67,7 @@ RUN mkdir -p "/opt/kafka-3.0.2" && chmod a+rw /opt/kafka-3.0.2 && curl -s "$KAFK RUN mkdir -p "/opt/kafka-3.1.2" && chmod a+rw /opt/kafka-3.1.2 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.1.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.1.2" RUN mkdir -p "/opt/kafka-3.2.3" && chmod a+rw /opt/kafka-3.2.3 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.2.3.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.2.3" RUN mkdir -p "/opt/kafka-3.3.1" && chmod a+rw /opt/kafka-3.3.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.3.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.3.1" +RUN mkdir -p "/opt/kafka-3.4.0" && chmod a+rw /opt/kafka-3.4.0 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.4.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.4.0" # Streams test dependencies RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.10.0.1-test.jar" -o /opt/kafka-0.10.0.1/libs/kafka-streams-0.10.0.1-test.jar @@ -88,6 +89,7 @@ RUN curl -s "$KAFKA_MIRROR/kafka-streams-3.0.2-test.jar" -o /opt/kafka-3.0.2/lib RUN curl -s "$KAFKA_MIRROR/kafka-streams-3.1.2-test.jar" -o /opt/kafka-3.1.2/libs/kafka-streams-3.1.2-test.jar RUN curl -s "$KAFKA_MIRROR/kafka-streams-3.2.3-test.jar" -o /opt/kafka-3.2.3/libs/kafka-streams-3.2.3-test.jar RUN curl -s "$KAFKA_MIRROR/kafka-streams-3.3.1-test.jar" -o /opt/kafka-3.3.1/libs/kafka-streams-3.3.1-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-3.4.0-test.jar" -o /opt/kafka-3.4.0/libs/kafka-streams-3.4.0-test.jar # The version of Kibosh to use for testing. # If you update this, also update vagrant/base.sh diff --git a/tests/kafkatest/tests/core/zookeeper_migration_test.py b/tests/kafkatest/tests/core/zookeeper_migration_test.py index 257b9f79158e8..d5ad2ec6f66b8 100644 --- a/tests/kafkatest/tests/core/zookeeper_migration_test.py +++ b/tests/kafkatest/tests/core/zookeeper_migration_test.py @@ -17,16 +17,18 @@ import time from ducktape.utils.util import wait_until +from ducktape.mark import parametrize +from ducktape.errors import TimeoutError from kafkatest.services.console_consumer import ConsoleConsumer from kafkatest.services.kafka import KafkaService from kafkatest.services.kafka.config_property import CLUSTER_ID -from kafkatest.services.kafka.quorum import remote_kraft, ServiceQuorumInfo, zk +from kafkatest.services.kafka.quorum import isolated_kraft, ServiceQuorumInfo, zk from kafkatest.services.verifiable_producer import VerifiableProducer from kafkatest.services.zookeeper import ZookeeperService from kafkatest.tests.produce_consume_validate import ProduceConsumeValidateTest from kafkatest.utils import is_int -from kafkatest.version import DEV_BRANCH +from kafkatest.version import DEV_BRANCH, V_3_4_0 class TestMigration(ProduceConsumeValidateTest): @@ -50,10 +52,10 @@ def wait_until_rejoin(self): def do_migration(self): # Start up KRaft controller in migration mode - remote_quorum = partial(ServiceQuorumInfo, remote_kraft) + remote_quorum = partial(ServiceQuorumInfo, isolated_kraft) controller = KafkaService(self.test_context, num_nodes=1, zk=self.zk, version=DEV_BRANCH, allow_zk_with_kraft=True, - remote_kafka=self.kafka, + isolated_kafka=self.kafka, server_prop_overrides=[["zookeeper.connect", self.zk.connect_setting()], ["zookeeper.metadata.migration.enable", "true"]], quorum_info_provider=remote_quorum) @@ -115,3 +117,73 @@ def test_online_migration(self): message_validator=is_int, version=DEV_BRANCH) self.run_produce_consume_validate(core_test_action=self.do_migration) + + @parametrize(metadata_quorum=isolated_kraft) + def test_pre_migration_mode_3_4(self, metadata_quorum): + """ + Start a KRaft quorum in 3.4 without migrations enabled. Since we were not correctly writing + ZkMigrationStateRecord in 3.4, there will be no ZK migration state in the log. + + When upgrading to 3.5+, the controller should see that there are records in the log and + automatically bootstrap a ZkMigrationStateRecord(NONE) into the log (indicating that this + cluster was created in KRaft mode). + + This test ensures that even if we enable migrations after the upgrade to 3.5, that no migration + is able to take place. + """ + self.zk = ZookeeperService(self.test_context, num_nodes=1, version=DEV_BRANCH) + self.zk.start() + + self.kafka = KafkaService(self.test_context, + num_nodes=3, + zk=self.zk, + allow_zk_with_kraft=True, + version=V_3_4_0, + server_prop_overrides=[["zookeeper.metadata.migration.enable", "false"]], + topics={self.topic: {"partitions": self.partitions, + "replication-factor": self.replication_factor, + 'configs': {"min.insync.replicas": 2}}}) + self.kafka.start() + + # Now reconfigure the cluster as if we're trying to do a migration + self.kafka.server_prop_overrides.clear() + self.kafka.server_prop_overrides.extend([ + ["zookeeper.metadata.migration.enable", "true"] + ]) + + self.logger.info("Performing rolling upgrade.") + for node in self.kafka.controller_quorum.nodes: + self.logger.info("Stopping controller node %s" % node.account.hostname) + self.kafka.controller_quorum.stop_node(node) + node.version = DEV_BRANCH + self.logger.info("Restarting controller node %s" % node.account.hostname) + self.kafka.controller_quorum.start_node(node) + self.wait_until_rejoin() + self.logger.info("Successfully restarted controller node %s" % node.account.hostname) + for node in self.kafka.nodes: + self.logger.info("Stopping broker node %s" % node.account.hostname) + self.kafka.stop_node(node) + node.version = DEV_BRANCH + self.logger.info("Restarting broker node %s" % node.account.hostname) + self.kafka.start_node(node) + self.wait_until_rejoin() + self.logger.info("Successfully restarted broker node %s" % node.account.hostname) + + # Check the controller's logs for the error message about the migration state + saw_expected_error = False + for node in self.kafka.controller_quorum.nodes: + with node.account.monitor_log(KafkaService.STDOUT_STDERR_CAPTURE) as monitor: + monitor.offset = 0 + try: + # Shouldn't have to wait too long to see this log message after startup + monitor.wait_until( + "ZkMigrationState is NONE which means this cluster should not be migrated from ZooKeeper", + timeout_sec=10.0, backoff_sec=.25, + err_msg="" + ) + saw_expected_error = True + break + except TimeoutError: + continue + + assert saw_expected_error, "Did not see expected ERROR log for ZkMigrationState = NONE in the controller logs" From da3f9dc5e31bc4738e9a1146842d1ee93f516aa2 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Fri, 17 Mar 2023 15:20:16 -0400 Subject: [PATCH 08/27] Add ZkMigrationState metric --- .../kafka/controller/ControllerMetrics.java | 4 ++++ .../controller/ControllerMetricsManager.java | 10 +++++++++- .../controller/QuorumControllerMetrics.java | 20 ++++++++++++++++++- .../controller/MockControllerMetrics.java | 8 ++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/metadata/src/main/java/org/apache/kafka/controller/ControllerMetrics.java b/metadata/src/main/java/org/apache/kafka/controller/ControllerMetrics.java index baab58540045f..4d3924471b697 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ControllerMetrics.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ControllerMetrics.java @@ -18,6 +18,8 @@ package org.apache.kafka.controller; +import org.apache.kafka.metadata.migration.ZkMigrationState; + public interface ControllerMetrics extends AutoCloseable { void setActive(boolean active); @@ -67,5 +69,7 @@ public interface ControllerMetrics extends AutoCloseable { long lastAppliedRecordTimestamp(); + void setZkMigrationState(ZkMigrationState state); + void close(); } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ControllerMetricsManager.java b/metadata/src/main/java/org/apache/kafka/controller/ControllerMetricsManager.java index 0201677bd5f98..ff255a9a7c3e1 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ControllerMetricsManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ControllerMetricsManager.java @@ -34,8 +34,10 @@ import org.apache.kafka.common.metadata.TopicRecord; import org.apache.kafka.common.metadata.UnfenceBrokerRecord; import org.apache.kafka.common.metadata.UnregisterBrokerRecord; +import org.apache.kafka.common.metadata.ZkMigrationStateRecord; import org.apache.kafka.common.protocol.ApiMessage; import org.apache.kafka.metadata.BrokerRegistrationFencingChange; +import org.apache.kafka.metadata.migration.ZkMigrationState; import org.apache.kafka.server.common.ApiMessageAndVersion; import static org.apache.kafka.metadata.LeaderConstants.NO_LEADER; import static org.apache.kafka.metadata.LeaderConstants.NO_LEADER_CHANGE; @@ -138,6 +140,9 @@ void replay(ApiMessage message) { case REMOVE_TOPIC_RECORD: replay((RemoveTopicRecord) message); break; + case ZK_MIGRATION_STATE_RECORD: + replay((ZkMigrationStateRecord) message); + break; case CONFIG_RECORD: case FEATURE_LEVEL_RECORD: case CLIENT_QUOTA_RECORD: @@ -147,7 +152,6 @@ void replay(ApiMessage message) { case USER_SCRAM_CREDENTIAL_RECORD: case REMOVE_USER_SCRAM_CREDENTIAL_RECORD: case NO_OP_RECORD: - case ZK_MIGRATION_STATE_RECORD: // These record types do not affect metrics break; default: @@ -281,6 +285,10 @@ private void replay(RemoveTopicRecord record) { updateTopicAndPartitionMetrics(); } + private void replay(ZkMigrationStateRecord record) { + controllerMetrics.setZkMigrationState(ZkMigrationState.of(record.zkMigrationState())); + } + private void updateBasedOnPartitionState(TopicIdPartition tp, PartitionState partitionState) { if (partitionState.leader() == NO_LEADER) { offlineTopicPartitions.add(tp); diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumControllerMetrics.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumControllerMetrics.java index 00413c4bd5303..d15a28042ec70 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumControllerMetrics.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumControllerMetrics.java @@ -22,6 +22,7 @@ import com.yammer.metrics.core.MetricName; import com.yammer.metrics.core.MetricsRegistry; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.metadata.migration.ZkMigrationState; import org.apache.kafka.server.metrics.KafkaYammerMetrics; import java.util.Arrays; @@ -58,6 +59,9 @@ public final class QuorumControllerMetrics implements ControllerMetrics { "KafkaController", "LastAppliedRecordTimestamp"); private final static MetricName LAST_APPLIED_RECORD_LAG_MS = getMetricName( "KafkaController", "LastAppliedRecordLagMs"); + private final static MetricName ZK_MIGRATION_STATE = getMetricName( + "KafkaController", "ZkMigrationState"); + private final MetricsRegistry registry; private volatile boolean active; @@ -71,6 +75,7 @@ public final class QuorumControllerMetrics implements ControllerMetrics { private final AtomicLong lastAppliedRecordOffset = new AtomicLong(0); private final AtomicLong lastCommittedRecordOffset = new AtomicLong(0); private final AtomicLong lastAppliedRecordTimestamp = new AtomicLong(0); + private final AtomicInteger zkMigrationState = new AtomicInteger(-1); private final Gauge activeControllerCount; private final Gauge fencedBrokerCountGauge; private final Gauge activeBrokerCountGauge; @@ -83,6 +88,7 @@ public final class QuorumControllerMetrics implements ControllerMetrics { private final Gauge lastCommittedRecordOffsetGauge; private final Gauge lastAppliedRecordTimestampGauge; private final Gauge lastAppliedRecordLagMsGauge; + private final Gauge zkMigrationStateGauge; private final Histogram eventQueueTime; private final Histogram eventQueueProcessingTime; @@ -173,6 +179,13 @@ public Long value() { return time.milliseconds() - lastAppliedRecordTimestamp.get(); } }); + zkMigrationStateGauge = registry.newGauge(ZK_MIGRATION_STATE, new Gauge() { + @Override + public Integer value() { + return zkMigrationState.get(); + } + }); + } @Override @@ -293,6 +306,10 @@ public long lastAppliedRecordTimestamp() { return lastAppliedRecordTimestamp.get(); } + public void setZkMigrationState(ZkMigrationState state) { + zkMigrationState.set(state.value()); + } + @Override public void close() { Arrays.asList( @@ -309,7 +326,8 @@ public void close() { LAST_APPLIED_RECORD_OFFSET, LAST_COMMITTED_RECORD_OFFSET, LAST_APPLIED_RECORD_TIMESTAMP, - LAST_APPLIED_RECORD_LAG_MS + LAST_APPLIED_RECORD_LAG_MS, + ZK_MIGRATION_STATE ).forEach(registry::removeMetric); } diff --git a/metadata/src/test/java/org/apache/kafka/controller/MockControllerMetrics.java b/metadata/src/test/java/org/apache/kafka/controller/MockControllerMetrics.java index 4a0155ed1bde6..691232833cca6 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/MockControllerMetrics.java +++ b/metadata/src/test/java/org/apache/kafka/controller/MockControllerMetrics.java @@ -17,6 +17,8 @@ package org.apache.kafka.controller; +import org.apache.kafka.metadata.migration.ZkMigrationState; + import java.util.concurrent.atomic.AtomicInteger; public final class MockControllerMetrics implements ControllerMetrics { @@ -31,6 +33,7 @@ public final class MockControllerMetrics implements ControllerMetrics { private volatile long lastAppliedRecordOffset = 0; private volatile long lastCommittedRecordOffset = 0; private volatile long lastAppliedRecordTimestamp = 0; + private volatile ZkMigrationState zkMigrationState = ZkMigrationState.UNINITIALIZED; private volatile boolean closed = false; @@ -154,6 +157,11 @@ public long lastAppliedRecordTimestamp() { return lastAppliedRecordTimestamp; } + @Override + public void setZkMigrationState(ZkMigrationState state) { + zkMigrationState = state; + } + @Override public void close() { closed = true; From cef8e6a5e6eaff78c09652d614cd3a7d7a3c5dfa Mon Sep 17 00:00:00 2001 From: David Arthur Date: Mon, 20 Mar 2023 12:37:42 -0400 Subject: [PATCH 09/27] Fix unit tests --- .../kafka/controller/QuorumControllerMetricsTest.java | 3 ++- .../metadata/migration/KRaftMigrationDriverTest.java | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerMetricsTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerMetricsTest.java index 400b860197e53..274fdd9b6e3e3 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerMetricsTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerMetricsTest.java @@ -46,7 +46,8 @@ public void testKafkaControllerMetricNames() { "LastAppliedRecordLagMs", "LastAppliedRecordOffset", "LastAppliedRecordTimestamp", - "LastCommittedRecordOffset" + "LastCommittedRecordOffset", + "ZkMigrationState" ); assertMetricsCreatedAndRemovedUponClose(expectedType, expectedMetricNames); } diff --git a/metadata/src/test/java/org/apache/kafka/metadata/migration/KRaftMigrationDriverTest.java b/metadata/src/test/java/org/apache/kafka/metadata/migration/KRaftMigrationDriverTest.java index b6315f39111cd..3189f54441753 100644 --- a/metadata/src/test/java/org/apache/kafka/metadata/migration/KRaftMigrationDriverTest.java +++ b/metadata/src/test/java/org/apache/kafka/metadata/migration/KRaftMigrationDriverTest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.metadata.BrokerRegistrationChangeRecord; import org.apache.kafka.common.metadata.ConfigRecord; import org.apache.kafka.common.metadata.RegisterBrokerRecord; +import org.apache.kafka.common.metadata.ZkMigrationStateRecord; import org.apache.kafka.image.MetadataDelta; import org.apache.kafka.image.MetadataImage; import org.apache.kafka.image.MetadataProvenance; @@ -220,6 +221,12 @@ RegisterBrokerRecord zkBrokerRecord(int id) { return record; } + ZkMigrationStateRecord preMigrationRecord() { + ZkMigrationStateRecord record = new ZkMigrationStateRecord(); + record.setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()); + return record; + } + /** * Enqueues a metadata change event with the migration driver and returns a future that can be waited on in * the test code. The future will complete once the metadata change event executes completely. @@ -264,6 +271,7 @@ public void testOnlySendNeededRPCsToBrokers() throws Exception { MetadataDelta delta = new MetadataDelta(image); driver.start(); + delta.replay(preMigrationRecord()); delta.replay(zkBrokerRecord(1)); delta.replay(zkBrokerRecord(2)); delta.replay(zkBrokerRecord(3)); @@ -345,6 +353,7 @@ public ZkMigrationLeadershipState claimControllerLeadership(ZkMigrationLeadershi MetadataDelta delta = new MetadataDelta(image); driver.start(); + delta.replay(preMigrationRecord()); delta.replay(zkBrokerRecord(1)); delta.replay(zkBrokerRecord(2)); delta.replay(zkBrokerRecord(3)); From 4c68c5dfbd21630ca6b75371ec61291b463a71e3 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Tue, 21 Mar 2023 15:02:36 -0400 Subject: [PATCH 10/27] Handle the case of updating metadata.version --- .../controller/FeatureControlManager.java | 28 ++- .../kafka/controller/QuorumController.java | 69 ++----- .../controller/ReplicationControlManager.java | 13 +- .../controller/ZkMigrationBootstrap.java | 39 ++++ .../controller/ZkMigrationControlManager.java | 105 +++++++++- .../controller/ClusterControlManagerTest.java | 10 + .../controller/FeatureControlManagerTest.java | 26 ++- .../ProducerIdControlManagerTest.java | 2 + .../controller/QuorumControllerTest.java | 6 +- .../ReplicationControlManagerTest.java | 3 + .../ZkMigrationControlManagerTest.java | 179 ++++++++++++++++++ 11 files changed, 392 insertions(+), 88 deletions(-) create mode 100644 metadata/src/main/java/org/apache/kafka/controller/ZkMigrationBootstrap.java create mode 100644 metadata/src/test/java/org/apache/kafka/controller/ZkMigrationControlManagerTest.java diff --git a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java index 7b2a9308e96b7..93b04bbf409a2 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java @@ -52,6 +52,7 @@ public static class Builder { private QuorumFeatures quorumFeatures = null; private MetadataVersion metadataVersion = MetadataVersion.latest(); private MetadataVersion minimumBootstrapVersion = MetadataVersion.MINIMUM_BOOTSTRAP_VERSION; + private ZkMigrationBootstrap zkMigrationBootstrap = null; Builder setLogContext(LogContext logContext) { this.logContext = logContext; @@ -78,6 +79,11 @@ Builder setMinimumBootstrapVersion(MetadataVersion minimumBootstrapVersion) { return this; } + Builder setZkMigrationBootstrap(ZkMigrationBootstrap zkMigrationBootstrap) { + this.zkMigrationBootstrap = zkMigrationBootstrap; + return this; + } + public FeatureControlManager build() { if (logContext == null) logContext = new LogContext(); if (snapshotRegistry == null) snapshotRegistry = new SnapshotRegistry(logContext); @@ -85,11 +91,17 @@ public FeatureControlManager build() { quorumFeatures = new QuorumFeatures(0, new ApiVersions(), QuorumFeatures.defaultFeatureMap(), Collections.emptyList()); } - return new FeatureControlManager(logContext, + if (zkMigrationBootstrap == null) { + throw new IllegalStateException("ZkMigrationBootstrap cannot be null"); + } + return new FeatureControlManager( + logContext, quorumFeatures, snapshotRegistry, metadataVersion, - minimumBootstrapVersion); + minimumBootstrapVersion, + zkMigrationBootstrap + ); } } @@ -115,18 +127,22 @@ public FeatureControlManager build() { */ private final MetadataVersion minimumBootstrapVersion; + private final ZkMigrationBootstrap zkMigrationBootstrap; + private FeatureControlManager( LogContext logContext, QuorumFeatures quorumFeatures, SnapshotRegistry snapshotRegistry, MetadataVersion metadataVersion, - MetadataVersion minimumBootstrapVersion + MetadataVersion minimumBootstrapVersion, + ZkMigrationBootstrap zkMigrationBootstrap ) { this.log = logContext.logger(FeatureControlManager.class); this.quorumFeatures = quorumFeatures; this.finalizedVersions = new TimelineHashMap<>(snapshotRegistry, 0); this.metadataVersion = new TimelineObject<>(snapshotRegistry, metadataVersion); this.minimumBootstrapVersion = minimumBootstrapVersion; + this.zkMigrationBootstrap = zkMigrationBootstrap; } ControllerResult> updateFeatures( @@ -224,7 +240,7 @@ private ApiError invalidUpdateVersion(String feature, short version, String mess } /** - * Perform some additional validation for metadata.version updates. + * Perform some additional validation for "metadata.version" updates. */ private ApiError updateMetadataVersion( short newVersionLevel, @@ -266,6 +282,10 @@ private ApiError updateMetadataVersion( new FeatureLevelRecord() .setName(MetadataVersion.FEATURE_NAME) .setFeatureLevel(newVersionLevel), FEATURE_LEVEL_RECORD.lowestSupportedVersion())); + + if (newVersion.isMigrationSupported()) { + zkMigrationBootstrap.bootstrapInitialMigrationState(newVersion, false, recordConsumer); + } return ApiError.NONE; } diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index f94d1c551d0e0..dc700c7e00854 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -688,9 +688,9 @@ public void run() throws Exception { if (!isActiveController(controllerEpoch)) { throw newNotControllerException(); } - if (zkMigrationControlManager.inPreMigrationMode() && !flags.contains(RUNS_IN_PREMIGRATION)) { - log.info("Cannot run write operation {} in premigration mode. Returning " + - "NOT_CONTROLLER.", name); + if (zkMigrationControlManager.inPreMigrationMode(featureControl.metadataVersion()) && + !flags.contains(RUNS_IN_PREMIGRATION)) { + log.info("Cannot run write operation {} in pre-migration mode. Returning NOT_CONTROLLER.", name); throw newNotControllerException(); } startProcessingTimeNs = OptionalLong.of(now); @@ -1199,22 +1199,9 @@ public ControllerResult generateRecordsAndResult() throws Exception { bootstrapMetadata.metadataVersion(), bootstrapMetadata.source()); records.addAll(bootstrapMetadata.records()); - if (bootstrapMetadata.metadataVersion().isMigrationSupported()) { - // Initialize the log with a ZkMigrationState - if (zkMigrationEnabled) { - log.info("Writing ZkMigrationState of PRE_MIGRATION to the log, since migrations are enabled."); - records.add(new ApiMessageAndVersion( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()), - ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION - )); - } else { - log.debug("Writing ZkMigrationState of NONE to the log, since migrations are not enabled."); - records.add(new ApiMessageAndVersion( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()), - ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION - )); - } - } + // Bootstrap the initial ZK Migration record + zkMigrationControlManager.bootstrapInitialMigrationState( + bootstrapMetadata.metadataVersion(), true, records::add); } else { // Logs have been replayed. We need to initialize some things here if upgrading from older KRaft versions if (featureControl.metadataVersion().equals(MetadataVersion.MINIMUM_KRAFT_VERSION)) { @@ -1222,36 +1209,9 @@ public ControllerResult generateRecordsAndResult() throws Exception { "Treating the log as version {}.", MetadataVersion.MINIMUM_KRAFT_VERSION); } - if (featureControl.metadataVersion().isMigrationSupported()) { - switch (zkMigrationControlManager.zkMigrationState()) { - case UNINITIALIZED: - // No ZkMigrationState record seen, put a NONE in the log - log.debug("Writing a ZkMigrationState of NONE to the log to indicate this cluster was not migrated from ZK."); - if (zkMigrationEnabled) { - log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); - } - records.add(new ApiMessageAndVersion( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()), - ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION - )); - break; - case NONE: - // This is a non-migrated KRaft cluster - log.debug("Read a ZkMigrationState of NONE from the log indicating this cluster was never migrated from ZK."); - if (zkMigrationEnabled) { - log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); - } - break; - case POST_MIGRATION: - // This is a migrated KRaft cluster - log.debug("Read a ZkMigrationState of POST_MIGRATION from the log indicating this cluster was previously migrated from ZK."); - break; - default: - // Once we have support for metadata transactions, this failure case will only exist for - // errant ZkMigrationStateRecord that are encountered outside a transaction. - throw fatalFaultHandler.handleFault("Detected an in-progress migration during startup, cannot continue."); - } - } + // Initialize the ZK migration state as NONE, or throw an error if we find an in-progress migration + zkMigrationControlManager.bootstrapInitialMigrationState( + featureControl.metadataVersion(), false, records::add); } return ControllerResult.atomicOf(records, null); } @@ -1368,8 +1328,7 @@ private void cancelMaybeFenceReplicas() { private void maybeScheduleNextBalancePartitionLeaders() { if (imbalancedScheduled != ImbalanceSchedule.SCHEDULED && leaderImbalanceCheckIntervalNs.isPresent() && - replicationControl.arePartitionLeadersImbalanced() && - zkMigrationControlManager.inPreMigrationMode()) { + replicationControl.arePartitionLeadersImbalanced()) { log.debug( "Scheduling write event for {} because scheduled ({}), checkIntervalNs ({}) and isImbalanced ({})", @@ -1775,10 +1734,6 @@ private enum ImbalanceSchedule { */ private final int maxRecordsPerBatch; - /** - * True if the controller was configured to allow Zookeeper migrations. - */ - private final boolean zkMigrationEnabled; private QuorumController( FaultHandler fatalFaultHandler, @@ -1829,6 +1784,7 @@ private QuorumController( setNodeId(nodeId). build(); this.clientQuotaControlManager = new ClientQuotaControlManager(snapshotRegistry); + this.zkMigrationControlManager = new ZkMigrationControlManager(snapshotRegistry, logContext, zkMigrationEnabled); this.featureControl = new FeatureControlManager.Builder(). setLogContext(logContext). setQuorumFeatures(quorumFeatures). @@ -1839,6 +1795,7 @@ private QuorumController( // are all treated as 3.0IV1. In newer versions the metadata.version will be specified // by the log. setMetadataVersion(MetadataVersion.MINIMUM_KRAFT_VERSION). + setZkMigrationBootstrap(zkMigrationControlManager). build(); this.clusterControl = new ClusterControlManager.Builder(). setLogContext(logContext). @@ -1881,8 +1838,6 @@ private QuorumController( this.curClaimEpoch = -1; this.needToCompleteAuthorizerLoad = authorizer.isPresent(); this.zkRecordConsumer = new MigrationRecordConsumer(); - this.zkMigrationEnabled = zkMigrationEnabled; - this.zkMigrationControlManager = new ZkMigrationControlManager(snapshotRegistry, logContext); updateWriteOffset(-1); resetToEmptyState(); diff --git a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java index 2b21d5c0b227a..012d6ea3c7bac 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java @@ -17,7 +17,6 @@ package org.apache.kafka.controller; -import org.apache.kafka.clients.ApiVersions; import org.apache.kafka.clients.admin.AlterConfigOp.OpType; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.common.ElectionType; @@ -90,7 +89,6 @@ import org.apache.kafka.metadata.placement.TopicAssignment; import org.apache.kafka.metadata.placement.UsableBroker; import org.apache.kafka.server.common.ApiMessageAndVersion; -import org.apache.kafka.server.common.MetadataVersion; import org.apache.kafka.server.policy.CreateTopicPolicy; import org.apache.kafka.timeline.SnapshotRegistry; import org.apache.kafka.timeline.TimelineHashMap; @@ -201,19 +199,12 @@ ReplicationControlManager build() { if (configurationControl == null) { throw new IllegalStateException("Configuration control must be set before building"); } else if (clusterControl == null) { - throw new IllegalStateException("Cluster controller must be set before building"); + throw new IllegalStateException("Cluster control must be set before building"); } if (logContext == null) logContext = new LogContext(); if (snapshotRegistry == null) snapshotRegistry = configurationControl.snapshotRegistry(); if (featureControl == null) { - featureControl = new FeatureControlManager.Builder(). - setLogContext(logContext). - setSnapshotRegistry(snapshotRegistry). - setQuorumFeatures(new QuorumFeatures(0, new ApiVersions(), - QuorumFeatures.defaultFeatureMap(), - Collections.singletonList(0))). - setMetadataVersion(MetadataVersion.latest()). - build(); + throw new IllegalStateException("FeatureControlManager must not be null"); } return new ReplicationControlManager(snapshotRegistry, logContext, diff --git a/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationBootstrap.java b/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationBootstrap.java new file mode 100644 index 0000000000000..052d64006e032 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationBootstrap.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.server.common.ApiMessageAndVersion; +import org.apache.kafka.server.common.MetadataVersion; + +import java.util.function.Consumer; + +public interface ZkMigrationBootstrap { + /** + * Initialize the metadata log with the correct ZkMigrationStateRecord. If the log is in an invalid + * state, this method may throw an exception. + * + * @param metadataVersion The current MetadataVersion of the log + * @param isMetadataLogEmpty True if the log is being initialized from empty + * @param recordConsumer A consumer for the ZkMigrationStateRecord + */ + void bootstrapInitialMigrationState( + MetadataVersion metadataVersion, + boolean isMetadataLogEmpty, + Consumer recordConsumer + ); +} diff --git a/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java index 833bdef40f758..2fce5ef763e9c 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java @@ -19,30 +19,123 @@ import org.apache.kafka.common.metadata.ZkMigrationStateRecord; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.metadata.migration.ZkMigrationState; +import org.apache.kafka.server.common.ApiMessageAndVersion; +import org.apache.kafka.server.common.MetadataVersion; import org.apache.kafka.timeline.SnapshotRegistry; import org.apache.kafka.timeline.TimelineObject; import org.slf4j.Logger; +import java.util.function.Consumer; -public class ZkMigrationControlManager { + +public class ZkMigrationControlManager implements ZkMigrationBootstrap { private final TimelineObject zkMigrationState; + private final boolean zkMigrationEnabled; + private final Logger log; ZkMigrationControlManager( SnapshotRegistry snapshotRegistry, - LogContext logContext + LogContext logContext, + boolean zkMigrationEnabled ) { - zkMigrationState = new TimelineObject<>(snapshotRegistry, ZkMigrationState.UNINITIALIZED); - log = logContext.logger(ZkMigrationControlManager.class); + this.zkMigrationState = new TimelineObject<>(snapshotRegistry, ZkMigrationState.UNINITIALIZED); + this.log = logContext.logger(ZkMigrationControlManager.class); + this.zkMigrationEnabled = zkMigrationEnabled; } + // Visible for testing ZkMigrationState zkMigrationState() { return zkMigrationState.get(); } - boolean inPreMigrationMode() { - return zkMigrationState.get() == ZkMigrationState.PRE_MIGRATION; + /** + * Tests if the controller should be preventing metadata updates due to being in the PRE_MIGRATION + * state. If the controller does not yet support migrations (before 3.4-IV0), then metadata updates + * are allowed in any state. Once the controller has been upgraded to a version that supports + * migrations, then this method checks if the controller is in the PRE_MIGRATION state. + */ + boolean inPreMigrationMode(MetadataVersion metadataVersion) { + if (metadataVersion.isMigrationSupported()) { + return zkMigrationState.get() == ZkMigrationState.PRE_MIGRATION; + } else { + return false; + } + } + + /** + * Optionally provides a ZkMigrationStateRecord to bootstrap into the metadata log. In the case of + * upgrades, the log will not be empty and this will return a NONE state record. For an empty log, + * this will return either a NONE or PRE_MIGRATION depending on the configuration and metadata.version. + * + * If the log is in PRE_MIGRATION, this will throw an error. + * + * @param metadataVersion The current MetadataVersion of the log + * @param isMetadataLogEmpty True if the log is being initialized from empty + * @param recordConsumer A consumer for the ZkMigrationStateRecord + */ + @Override + public void bootstrapInitialMigrationState( + MetadataVersion metadataVersion, + boolean isMetadataLogEmpty, + Consumer recordConsumer + ) { + if (!metadataVersion.isMigrationSupported()) { + return; + } + + if (isMetadataLogEmpty) { + // Initialize the log with a ZkMigrationState + if (zkMigrationEnabled) { + log.info("Writing ZkMigrationState of PRE_MIGRATION to the log, since migrations are enabled."); + recordConsumer.accept(new ApiMessageAndVersion( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()), + ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION + )); + } else { + log.debug("Writing ZkMigrationState of NONE to the log, since migrations are not enabled."); + recordConsumer.accept(new ApiMessageAndVersion( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()), + ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION + )); + } + } else { + // non-empty log + switch (zkMigrationState()) { + case UNINITIALIZED: + // No ZkMigrationState record seen, put a NONE in the log + log.debug("Writing a ZkMigrationState of NONE to the log to indicate this cluster was not migrated from ZK."); + if (zkMigrationEnabled) { + log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); + } + recordConsumer.accept(new ApiMessageAndVersion( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()), + ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION + )); + break; + case NONE: + // This is a non-migrated KRaft cluster + log.debug("Read a ZkMigrationState of NONE from the log indicating this cluster was never migrated from ZK."); + if (zkMigrationEnabled) { + log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); + } + break; + case POST_MIGRATION: + // This is a migrated KRaft cluster + log.debug("Read a ZkMigrationState of POST_MIGRATION from the log indicating this cluster was previously migrated from ZK."); + break; + case MIGRATION: + // This cluster is migrated, but still in dual-write mode + log.debug("Read a ZkMigrationState of MIGRATION from the log indicating this cluster is being migrated from ZK."); + break; + case PRE_MIGRATION: + default: + // Once we have support for metadata transactions, this failure case will only exist for + // errant ZkMigrationStateRecord that are encountered outside a transaction. + throw new IllegalStateException("Detected an in-progress migration during startup, cannot continue."); + } + } } void replay(ZkMigrationStateRecord record) { diff --git a/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java index c7dcbc5bfcd7c..616dc6f905f35 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java @@ -58,6 +58,7 @@ import java.util.List; import java.util.Optional; +import static org.apache.kafka.controller.FeatureControlManagerTest.NO_OP_MIGRATION_BOOTSTRAP; import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV2; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -79,6 +80,7 @@ public void testReplay(MetadataVersion metadataVersion) { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setTime(time). @@ -140,6 +142,7 @@ public void testReplayRegisterBrokerRecord() { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setClusterId("fPZv1VBsRFmnlRvmGcOW9w"). @@ -193,6 +196,7 @@ public void testReplayBrokerRegistrationChangeRecord() { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setClusterId("fPZv1VBsRFmnlRvmGcOW9w"). @@ -248,6 +252,7 @@ public void testRegistrationWithIncorrectClusterId() throws Exception { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setClusterId("fPZv1VBsRFmnlRvmGcOW9w"). @@ -277,6 +282,7 @@ public void testRegisterBrokerRecordVersion(MetadataVersion metadataVersion) { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(metadataVersion). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setClusterId("fPZv1VBsRFmnlRvmGcOW9w"). @@ -333,6 +339,7 @@ public void testUnregister() throws Exception { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setTime(new MockTime(0, 0, 0)). @@ -367,6 +374,7 @@ public void testPlaceReplicas(int numUsableBrokers) throws Exception { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setTime(time). @@ -420,6 +428,7 @@ public void testRegistrationsToRecords(MetadataVersion metadataVersion) throws E QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(metadataVersion). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setTime(time). @@ -498,6 +507,7 @@ public void testRegistrationWithUnsupportedMetadataVersion() { MetadataVersion.IBP_3_3_IV0.featureLevel())), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.IBP_3_3_IV0). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setClusterId("fPZv1VBsRFmnlRvmGcOW9w"). diff --git a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java index 8345b4e314955..1a19b9cd1075d 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java @@ -51,6 +51,8 @@ @Timeout(value = 40) public class FeatureControlManagerTest { + public static final ZkMigrationBootstrap NO_OP_MIGRATION_BOOTSTRAP = new ZkMigrationControlManagerTest.NoOpZkMigrationBootstrap(); + @SuppressWarnings("unchecked") private static Map rangeMap(Object... args) { Map result = new HashMap<>(); @@ -96,6 +98,7 @@ public void testUpdateFeatures() { setQuorumFeatures(features("foo", 1, 2)). setSnapshotRegistry(snapshotRegistry). setMetadataVersion(MetadataVersion.IBP_3_3_IV0). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); snapshotRegistry.getOrCreateSnapshot(-1); assertEquals(new FinalizedControllerFeatures(Collections.singletonMap("metadata.version", (short) 4), -1), @@ -134,6 +137,7 @@ public void testReplay() { setQuorumFeatures(features("foo", 1, 2)). setSnapshotRegistry(snapshotRegistry). setMetadataVersion(MetadataVersion.IBP_3_3_IV0). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); manager.replay(record); snapshotRegistry.getOrCreateSnapshot(123); @@ -149,6 +153,7 @@ public void testUpdateFeaturesErrorCases() { setLogContext(logContext). setQuorumFeatures(features("foo", 1, 5, "bar", 1, 2)). setSnapshotRegistry(snapshotRegistry). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); assertEquals( @@ -211,6 +216,7 @@ public void testReplayRecords() throws Exception { setQuorumFeatures(features("foo", 1, 5, "bar", 1, 2)). setSnapshotRegistry(snapshotRegistry). setMetadataVersion(MetadataVersion.IBP_3_3_IV0). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ControllerResult> result = manager. updateFeatures(updateMap("foo", 5, "bar", 1), @@ -225,10 +231,11 @@ public void testReplayRecords() throws Exception { } private static final FeatureControlManager.Builder TEST_MANAGER_BUILDER1 = - new FeatureControlManager.Builder(). - setQuorumFeatures(features(MetadataVersion.FEATURE_NAME, - MetadataVersion.IBP_3_3_IV0.featureLevel(), MetadataVersion.IBP_3_3_IV3.featureLevel())). - setMetadataVersion(MetadataVersion.IBP_3_3_IV2); + new FeatureControlManager.Builder(). + setQuorumFeatures(features(MetadataVersion.FEATURE_NAME, + MetadataVersion.IBP_3_3_IV0.featureLevel(), MetadataVersion.IBP_3_3_IV3.featureLevel())). + setMetadataVersion(MetadataVersion.IBP_3_3_IV2). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP); @Test public void testApplyMetadataVersionChangeRecord() { @@ -355,7 +362,9 @@ public void testCanUseSafeDowngradeIfMetadataDidNotChange() { setQuorumFeatures(features(MetadataVersion.FEATURE_NAME, MetadataVersion.IBP_3_0_IV0.featureLevel(), MetadataVersion.IBP_3_3_IV1.featureLevel())). setMetadataVersion(MetadataVersion.IBP_3_1_IV0). - setMinimumBootstrapVersion(MetadataVersion.IBP_3_0_IV0).build(); + setMinimumBootstrapVersion(MetadataVersion.IBP_3_0_IV0). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). + build(); assertEquals(ControllerResult.of(Collections.emptyList(), singletonMap(MetadataVersion.FEATURE_NAME, ApiError.NONE)), manager.updateFeatures( @@ -369,8 +378,10 @@ public void testCanUseSafeDowngradeIfMetadataDidNotChange() { public void testCanotDowngradeBefore3_3_IV0() { FeatureControlManager manager = new FeatureControlManager.Builder(). setQuorumFeatures(features(MetadataVersion.FEATURE_NAME, - MetadataVersion.IBP_3_0_IV0.featureLevel(), MetadataVersion.IBP_3_3_IV3.featureLevel())). - setMetadataVersion(MetadataVersion.IBP_3_3_IV0).build(); + MetadataVersion.IBP_3_0_IV0.featureLevel(), MetadataVersion.IBP_3_3_IV3.featureLevel())). + setMetadataVersion(MetadataVersion.IBP_3_3_IV0). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). + build(); assertEquals(ControllerResult.of(Collections.emptyList(), singletonMap(MetadataVersion.FEATURE_NAME, new ApiError(Errors.INVALID_UPDATE_VERSION, "Invalid metadata.version 3. Unable to set a metadata.version less than 3.3-IV0"))), @@ -389,6 +400,7 @@ public void testCreateFeatureLevelRecords() { localSupportedFeatures.put("foo", VersionRange.of(0, 2)); FeatureControlManager manager = new FeatureControlManager.Builder(). setQuorumFeatures(new QuorumFeatures(0, new ApiVersions(), localSupportedFeatures, emptyList())). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ControllerResult> result = manager.updateFeatures( Collections.singletonMap("foo", (short) 1), diff --git a/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java index b50083169e6ab..f091556cc8730 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java @@ -32,6 +32,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import static org.apache.kafka.controller.FeatureControlManagerTest.NO_OP_MIGRATION_BOOTSTRAP; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -53,6 +54,7 @@ public void setUp() { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); clusterControl = new ClusterControlManager.Builder(). setTime(time). diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java index 8fd27767d8265..55eb79b62e030 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -227,7 +227,7 @@ private void testDelayedConfigurationOperations( new ResultOrError<>(Collections.emptyMap())), controller.describeConfigs(ANONYMOUS_CONTEXT, Collections.singletonMap( BROKER0, Collections.emptyList())).get()); - logEnv.logManagers().forEach(m -> m.setMaxReadOffset(3L)); + logEnv.logManagers().forEach(m -> m.setMaxReadOffset(4L)); assertEquals(Collections.singletonMap(BROKER0, ApiError.NONE), future1.get()); } @@ -539,7 +539,7 @@ public void testUnregisterBroker() throws Throwable { setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwBA")). setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_5_IV1)). setListeners(listeners)); - assertEquals(2L, reply.get().epoch()); + assertEquals(3L, reply.get().epoch()); CreateTopicsRequestData createTopicsRequestData = new CreateTopicsRequestData().setTopics( new CreatableTopicCollection(Collections.singleton( @@ -555,7 +555,7 @@ public void testUnregisterBroker() throws Throwable { get().topics().find("foo").errorMessage()); assertEquals(new BrokerHeartbeatReply(true, false, false, false), active.processBrokerHeartbeat(ANONYMOUS_CONTEXT, new BrokerHeartbeatRequestData(). - setWantFence(false).setBrokerEpoch(2L).setBrokerId(0). + setWantFence(false).setBrokerEpoch(3L).setBrokerId(0). setCurrentMetadataOffset(100000L)).get()); assertEquals(Errors.NONE.code(), active.createTopics(ANONYMOUS_CONTEXT, createTopicsRequestData, Collections.singleton("foo")). diff --git a/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java index 975062fbde73f..69210ffd9ec19 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java @@ -134,6 +134,7 @@ import static org.apache.kafka.controller.ControllerRequestContextUtil.QUOTA_EXCEEDED_IN_TEST_MSG; import static org.apache.kafka.controller.ControllerRequestContextUtil.anonymousContextFor; import static org.apache.kafka.controller.ControllerRequestContextUtil.anonymousContextWithMutationQuotaExceededFor; +import static org.apache.kafka.controller.FeatureControlManagerTest.NO_OP_MIGRATION_BOOTSTRAP; import static org.apache.kafka.metadata.LeaderConstants.NO_LEADER; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -161,6 +162,7 @@ private static class ReplicationControlTestContext { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); final ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setLogContext(logContext). @@ -200,6 +202,7 @@ void replay(List records) throws Exception { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(metadataVersion). + setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); this.replicationControl = new ReplicationControlManager.Builder(). diff --git a/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationControlManagerTest.java new file mode 100644 index 0000000000000..435c25e5f2e14 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationControlManagerTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.controller; + +import org.apache.kafka.common.metadata.ZkMigrationStateRecord; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.metadata.migration.ZkMigrationState; +import org.apache.kafka.server.common.ApiMessageAndVersion; +import org.apache.kafka.server.common.MetadataVersion; +import org.apache.kafka.timeline.SnapshotRegistry; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.fail; + +public class ZkMigrationControlManagerTest { + public static class NoOpZkMigrationBootstrap implements ZkMigrationBootstrap { + @Override + public void bootstrapInitialMigrationState( + MetadataVersion metadataVersion, + boolean isMetadataLogEmpty, + Consumer recordConsumer + ) { + + } + } + + private ZkMigrationControlManager setupAndBootstrap( + MetadataVersion metadataVersion, + boolean emptyLog, + boolean zkMigrationEnabled + ) { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ZkMigrationControlManager migrationControl = new ZkMigrationControlManager( + snapshotRegistry, new LogContext(), zkMigrationEnabled + ); + List records = new ArrayList<>(); + migrationControl.bootstrapInitialMigrationState(metadataVersion, emptyLog, records::add); + records.forEach(record -> { + migrationControl.replay((ZkMigrationStateRecord) record.message()); + }); + return migrationControl; + } + + private void verifyCannotBootstrapAgain(MetadataVersion metadataVersion, ZkMigrationControlManager migrationControl) { + // Should not be able to produce another record in any circumstance after we've bootstrapped once + try { + migrationControl.bootstrapInitialMigrationState(metadataVersion, false, + record -> fail("Did not expect to get another record here, but got " + record + ". State was " + migrationControl.zkMigrationState())); + } catch (IllegalStateException e) { + assertEquals(e.getMessage(), "Detected an in-progress migration during startup, cannot continue."); + } + } + + @Test + public void testBootstrapEmptyLog() { + ZkMigrationControlManager migrationControl; + + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, true, true); + assertEquals(ZkMigrationState.UNINITIALIZED, migrationControl.zkMigrationState()); + verifyCannotBootstrapAgain(MetadataVersion.IBP_3_3_IV0, migrationControl); + + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, true, false); + assertEquals(ZkMigrationState.UNINITIALIZED, migrationControl.zkMigrationState()); + verifyCannotBootstrapAgain(MetadataVersion.IBP_3_3_IV0, migrationControl); + + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, true); + assertEquals(ZkMigrationState.PRE_MIGRATION, migrationControl.zkMigrationState()); + verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl); + + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, false); + assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); + verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl); + } + + @Test + public void testBootstrapNonEmptyLog() { + ZkMigrationControlManager migrationControl; + + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, false, true); + assertEquals(ZkMigrationState.UNINITIALIZED, migrationControl.zkMigrationState()); + + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, false, false); + assertEquals(ZkMigrationState.UNINITIALIZED, migrationControl.zkMigrationState()); + + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, true); + assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); + + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, false); + assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); + } + + @Test + public void testMigrationStateTransitions() { + ZkMigrationControlManager migrationControl; + + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, true); + assertEquals(ZkMigrationState.PRE_MIGRATION, migrationControl.zkMigrationState()); + + assertThrows(IllegalStateException.class, () -> + migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()))); + assertThrows(IllegalStateException.class, () -> + migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()))); + assertThrows(IllegalStateException.class, () -> + migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.POST_MIGRATION.value()))); + + migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value())); + assertEquals(ZkMigrationState.MIGRATION, migrationControl.zkMigrationState()); + + assertThrows(IllegalStateException.class, () -> + migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()))); + assertThrows(IllegalStateException.class, () -> + migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()))); + assertThrows(IllegalStateException.class, () -> + migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()))); + + migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.POST_MIGRATION.value())); + assertEquals(ZkMigrationState.POST_MIGRATION, migrationControl.zkMigrationState()); + + verifyNoStateTransitionAllowed(migrationControl); + } + + private void verifyNoStateTransitionAllowed(ZkMigrationControlManager migrationControl) { + assertThrows(IllegalStateException.class, () -> + migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()))); + assertThrows(IllegalStateException.class, () -> + migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()))); + assertThrows(IllegalStateException.class, () -> + migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()))); + assertThrows(IllegalStateException.class, () -> + migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.POST_MIGRATION.value()))); + } + + @Test + public void testNonMigrationStateTransitions() { + // When migrations should not be present, ensure we can't change state from NONE + ZkMigrationControlManager migrationControl; + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, true); + assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); + verifyNoStateTransitionAllowed(migrationControl); + + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, false); + assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); + verifyNoStateTransitionAllowed(migrationControl); + } + + @Test + public void testMigrationDisabledStateTransitions() { + // When migrations are disabled, ensure we can't change state from NONE + ZkMigrationControlManager migrationControl; + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, false); + assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); + verifyNoStateTransitionAllowed(migrationControl); + + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, false); + assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); + verifyNoStateTransitionAllowed(migrationControl); + } +} From 35165fb3297284685ac913de9c7226fb905bd046 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Wed, 22 Mar 2023 10:17:44 -0400 Subject: [PATCH 11/27] Handle PRE_MIGRATION record when upgrading from 3.4 --- .../controller/FeatureControlManager.java | 5 +- .../controller/ZkMigrationControlManager.java | 126 ++++++++++++------ .../migration/KRaftMigrationDriver.java | 14 +- .../common/metadata/ZkMigrationRecord.json | 8 +- .../ZkMigrationControlManagerTest.java | 103 +++++++++++++- .../tests/core/zookeeper_migration_test.py | 75 ++++++++++- 6 files changed, 285 insertions(+), 46 deletions(-) diff --git a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java index 93b04bbf409a2..c8447cb933730 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java @@ -100,7 +100,7 @@ public FeatureControlManager build() { snapshotRegistry, metadataVersion, minimumBootstrapVersion, - zkMigrationBootstrap + zkMigrationBootstrap ); } } @@ -240,7 +240,7 @@ private ApiError invalidUpdateVersion(String feature, short version, String mess } /** - * Perform some additional validation for "metadata.version" updates. + * Perform some additional validation for metadata.version updates. */ private ApiError updateMetadataVersion( short newVersionLevel, @@ -283,6 +283,7 @@ private ApiError updateMetadataVersion( .setName(MetadataVersion.FEATURE_NAME) .setFeatureLevel(newVersionLevel), FEATURE_LEVEL_RECORD.lowestSupportedVersion())); + // If we are moving to a version that supports migrations, we need to write the correct state into the log if (newVersion.isMigrationSupported()) { zkMigrationBootstrap.bootstrapInitialMigrationState(newVersion, false, recordConsumer); } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java index 2fce5ef763e9c..24d91078b9b30 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java @@ -29,7 +29,26 @@ public class ZkMigrationControlManager implements ZkMigrationBootstrap { - private final TimelineObject zkMigrationState; + + private static class ZkMigrationControlState { + private final ZkMigrationState zkMigrationState; + private final boolean preMigrationSupported; + + private ZkMigrationControlState(ZkMigrationState zkMigrationState, boolean preMigrationSupported) { + this.zkMigrationState = zkMigrationState; + this.preMigrationSupported = preMigrationSupported; + } + + public ZkMigrationState zkMigrationState() { + return zkMigrationState; + } + + public boolean preMigrationSupported() { + return preMigrationSupported; + } + } + + private final TimelineObject migrationControlState; private final boolean zkMigrationEnabled; @@ -40,14 +59,15 @@ public class ZkMigrationControlManager implements ZkMigrationBootstrap { LogContext logContext, boolean zkMigrationEnabled ) { - this.zkMigrationState = new TimelineObject<>(snapshotRegistry, ZkMigrationState.UNINITIALIZED); + this.migrationControlState = new TimelineObject<>(snapshotRegistry, + new ZkMigrationControlState(ZkMigrationState.UNINITIALIZED, false)); this.log = logContext.logger(ZkMigrationControlManager.class); this.zkMigrationEnabled = zkMigrationEnabled; } // Visible for testing ZkMigrationState zkMigrationState() { - return zkMigrationState.get(); + return migrationControlState.get().zkMigrationState(); } /** @@ -57,8 +77,9 @@ ZkMigrationState zkMigrationState() { * migrations, then this method checks if the controller is in the PRE_MIGRATION state. */ boolean inPreMigrationMode(MetadataVersion metadataVersion) { - if (metadataVersion.isMigrationSupported()) { - return zkMigrationState.get() == ZkMigrationState.PRE_MIGRATION; + ZkMigrationControlState state = migrationControlState.get(); + if (metadataVersion.isMigrationSupported() && state.preMigrationSupported()) { + return state.zkMigrationState() == ZkMigrationState.PRE_MIGRATION; } else { return false; } @@ -68,7 +89,7 @@ boolean inPreMigrationMode(MetadataVersion metadataVersion) { * Optionally provides a ZkMigrationStateRecord to bootstrap into the metadata log. In the case of * upgrades, the log will not be empty and this will return a NONE state record. For an empty log, * this will return either a NONE or PRE_MIGRATION depending on the configuration and metadata.version. - * + *

* If the log is in PRE_MIGRATION, this will throw an error. * * @param metadataVersion The current MetadataVersion of the log @@ -89,73 +110,100 @@ public void bootstrapInitialMigrationState( // Initialize the log with a ZkMigrationState if (zkMigrationEnabled) { log.info("Writing ZkMigrationState of PRE_MIGRATION to the log, since migrations are enabled."); - recordConsumer.accept(new ApiMessageAndVersion( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()), - ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION - )); + recordConsumer.accept(buildRecord(ZkMigrationState.PRE_MIGRATION)); } else { log.debug("Writing ZkMigrationState of NONE to the log, since migrations are not enabled."); - recordConsumer.accept(new ApiMessageAndVersion( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()), - ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION - )); + recordConsumer.accept(buildRecord(ZkMigrationState.NONE)); } } else { // non-empty log - switch (zkMigrationState()) { + String prefix = "During metadata log initialization,"; + ZkMigrationControlState state = migrationControlState.get(); + switch (state.zkMigrationState()) { case UNINITIALIZED: - // No ZkMigrationState record seen, put a NONE in the log - log.debug("Writing a ZkMigrationState of NONE to the log to indicate this cluster was not migrated from ZK."); + // No ZkMigrationState record has been seen yet + log.debug("{} did not read any ZkMigrationState. Writing a ZkMigrationState of NONE to the log to " + + "indicate this cluster was not migrated from ZK.", prefix); if (zkMigrationEnabled) { log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); } - recordConsumer.accept(new ApiMessageAndVersion( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()), - ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION - )); + recordConsumer.accept(buildRecord(ZkMigrationState.NONE)); break; case NONE: // This is a non-migrated KRaft cluster - log.debug("Read a ZkMigrationState of NONE from the log indicating this cluster was never migrated from ZK."); + log.debug("{} read a ZkMigrationState of NONE indicating this cluster was never migrated from ZK.", prefix); if (zkMigrationEnabled) { log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); } break; case POST_MIGRATION: // This is a migrated KRaft cluster - log.debug("Read a ZkMigrationState of POST_MIGRATION from the log indicating this cluster was previously migrated from ZK."); + log.debug("{} read a ZkMigrationState of POST_MIGRATION indicating this cluster was previously migrated from ZK.", prefix); break; case MIGRATION: // This cluster is migrated, but still in dual-write mode - log.debug("Read a ZkMigrationState of MIGRATION from the log indicating this cluster is being migrated from ZK."); + if (zkMigrationEnabled) { + log.debug("{} read a ZkMigrationState of MIGRATION indicating this cluster is being migrated from ZK.", prefix); + } else { + throw new IllegalStateException( + prefix + " read a ZkMigrationState of MIGRATION indicating this cluster is being migrated " + + "from ZK, but the controller does not have migrations enabled." + ); + } break; case PRE_MIGRATION: + if (!state.preMigrationSupported()) { + // Upgrade case from 3.4. The controller only wrote PRE_MIGRATION during migrations in that version, + // so this needs to complete that migration. + log.info("{} read a ZkMigrationState of PRE_MIGRATION from a ZK migration on Apache Kafka 3.4.", prefix); + if (zkMigrationEnabled) { + recordConsumer.accept(buildRecord(ZkMigrationState.MIGRATION)); + log.info("Writing ZkMigrationState of MIGRATION since migration mode is still active on the controller."); + } else { + recordConsumer.accept(buildRecord(ZkMigrationState.MIGRATION)); + recordConsumer.accept(buildRecord(ZkMigrationState.POST_MIGRATION)); + log.info("Writing ZkMigrationState of POST_MIGRATION since migration mode is not active on the controller."); + } + } else { + log.error("{} read a ZkMigrationState of PRE_MIGRATION indicating this cluster failed during a ZK migration.", prefix); + throw new IllegalStateException("Detected an invalid migration state during startup, cannot continue."); + } + break; default: - // Once we have support for metadata transactions, this failure case will only exist for - // errant ZkMigrationStateRecord that are encountered outside a transaction. - throw new IllegalStateException("Detected an in-progress migration during startup, cannot continue."); + throw new IllegalStateException("Unsupported migration state " + state.zkMigrationState()); } } } + private ApiMessageAndVersion buildRecord(ZkMigrationState state) { + return new ApiMessageAndVersion( + new ZkMigrationStateRecord() + .setZkMigrationState(state.value()) + .setPreMigrationSupported(true), + ZkMigrationStateRecord.HIGHEST_SUPPORTED_VERSION + ); + } + + /** + * The state changes we allow are: + *

  • UNINITIALIZED -> ANY
    • + *
  • PRE_MIGRATION -> MIGRATION
    • + *
  • MIGRATION -> POST_MIGRATION
    • + */ void replay(ZkMigrationStateRecord record) { ZkMigrationState recordState = ZkMigrationState.of(record.zkMigrationState()); - // From uninitialized, only allow PRE_MIGRATION - if (zkMigrationState.get().equals(ZkMigrationState.UNINITIALIZED)) { - if (recordState.equals(ZkMigrationState.PRE_MIGRATION) || recordState.equals(ZkMigrationState.NONE)) { - log.info("Initializing ZK migration state as {}", recordState); - zkMigrationState.set(recordState); - } else { - throw new IllegalStateException("The first migration state seen can only be PRE_MIGRATION or NONE, not " + recordState.name()); - } + ZkMigrationControlState currentState = migrationControlState.get(); + if (currentState.zkMigrationState().equals(ZkMigrationState.UNINITIALIZED)) { + log.info("Initializing ZK migration state as {}", recordState); + migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); } else { - switch (zkMigrationState.get()) { + switch (currentState.zkMigrationState()) { case NONE: - throw new IllegalStateException("Cannot ever change migration state from NONE"); + throw new IllegalStateException("Cannot ever change migration state away from NONE"); case PRE_MIGRATION: if (recordState.equals(ZkMigrationState.MIGRATION)) { log.info("Transitioning ZK migration state to {}", recordState); - zkMigrationState.set(recordState); + migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); } else { throw new IllegalStateException("Cannot change migration state from PRE_MIGRATION to " + recordState); } @@ -163,7 +211,7 @@ void replay(ZkMigrationStateRecord record) { case MIGRATION: if (recordState.equals(ZkMigrationState.POST_MIGRATION)) { log.info("Transitioning ZK migration state to {}", recordState); - zkMigrationState.set(recordState); + migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); } else { throw new IllegalStateException("Cannot change migration state from MIGRATION to " + recordState); } diff --git a/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java b/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java index c9ec4be659b1e..9513a63befc50 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java +++ b/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java @@ -157,8 +157,20 @@ private boolean areZkBrokersReadyForMigration() { return false; } - // First check the brokers registered in ZK + if (image.cluster().isEmpty()) { + // This primarily happens in system tests when we are starting a new ZK cluster and KRaft quorum + // around the same time. + log.info("No brokers are known to KRaft, waiting for brokers to register."); + return false; + } + Set zkBrokerRegistrations = zkMigrationClient.readBrokerIds(); + if (zkBrokerRegistrations.isEmpty()) { + // Similar to the above empty check + log.info("No brokers are registered in ZK, waiting for brokers to register."); + return false; + } + if (imageDoesNotContainAllBrokers(image, zkBrokerRegistrations)) { log.info("Still waiting for ZK brokers {} to register with KRaft.", zkBrokerRegistrations); return false; diff --git a/metadata/src/main/resources/common/metadata/ZkMigrationRecord.json b/metadata/src/main/resources/common/metadata/ZkMigrationRecord.json index 65910dc6a45c0..b89d304666574 100644 --- a/metadata/src/main/resources/common/metadata/ZkMigrationRecord.json +++ b/metadata/src/main/resources/common/metadata/ZkMigrationRecord.json @@ -18,11 +18,15 @@ "type": "metadata", "name": "ZkMigrationStateRecord", // Version 0 adds ZkMigrationState which is used by the KRaft controller to mark the beginning and end - // of the ZK to KRaft migration. Possible values are 1 (PreMigration), 2 (Migration), 3 (PostMigration). + // of the ZK to KRaft migration. Possible values are 0 (None), 1 (PreMigration), 2 (Migration), 3 (PostMigration). + // + // Version 0 (3.5) adds PreMigrationSupported tagged field to distinguish between a 3.4 migration and a 3.5+ migration "validVersions": "0", "flexibleVersions": "0+", "fields": [ { "name": "ZkMigrationState", "type": "int8", "versions": "0+", - "about": "One of the possible migration states." } + "about": "One of the possible migration states." }, + { "name": "PreMigrationSupported", "type": "bool", "versions": "0+", "taggedVersions": "0+", "tag": 0, + "about": "True if the cluster supports pre-migration mode" } ] } \ No newline at end of file diff --git a/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationControlManagerTest.java index 435c25e5f2e14..2c5275086b25e 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationControlManagerTest.java @@ -30,7 +30,9 @@ import java.util.function.Consumer; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; public class ZkMigrationControlManagerTest { @@ -68,7 +70,7 @@ private void verifyCannotBootstrapAgain(MetadataVersion metadataVersion, ZkMigra migrationControl.bootstrapInitialMigrationState(metadataVersion, false, record -> fail("Did not expect to get another record here, but got " + record + ". State was " + migrationControl.zkMigrationState())); } catch (IllegalStateException e) { - assertEquals(e.getMessage(), "Detected an in-progress migration during startup, cannot continue."); + assertEquals(e.getMessage(), "Detected an invalid migration state during startup, cannot continue."); } } @@ -158,10 +160,73 @@ public void testNonMigrationStateTransitions() { migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, true); assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); verifyNoStateTransitionAllowed(migrationControl); + verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl); migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, false); assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); verifyNoStateTransitionAllowed(migrationControl); + verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl); + } + + @Test + public void testFailoverToNonMigrationControllerDuringMigration() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ZkMigrationControlManager migrationControl = new ZkMigrationControlManager( + snapshotRegistry, new LogContext(), true + ); + + // Controller has completed migration + migrationControl.replay( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()) + .setPreMigrationSupported(true)); + migrationControl.replay( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()) + .setPreMigrationSupported(true)); + assertEquals(ZkMigrationState.MIGRATION, migrationControl.zkMigrationState()); + + // Failover to controller that does not have migrations + ZkMigrationControlManager migrationControl2 = new ZkMigrationControlManager( + snapshotRegistry, new LogContext(), false + ); + migrationControl2.replay( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()) + .setPreMigrationSupported(true)); + assertThrows(IllegalStateException.class, () -> migrationControl2.bootstrapInitialMigrationState( + MetadataVersion.IBP_3_4_IV0, false, __ -> { })); + } + + @Test + public void testFailoverToNonMigrationControllerAfterMigration() { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ZkMigrationControlManager migrationControl = new ZkMigrationControlManager( + snapshotRegistry, new LogContext(), true + ); + + // Controller has completed migration + migrationControl.replay( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()) + .setPreMigrationSupported(true)); + migrationControl.replay( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()) + .setPreMigrationSupported(true)); + migrationControl.replay( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.POST_MIGRATION.value()) + .setPreMigrationSupported(true)); + assertEquals(ZkMigrationState.POST_MIGRATION, migrationControl.zkMigrationState()); + + // Failover to controller that does not have migrations + ZkMigrationControlManager migrationControl2 = new ZkMigrationControlManager( + snapshotRegistry, new LogContext(), false + ); + migrationControl2.replay( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.POST_MIGRATION.value()) + .setPreMigrationSupported(true)); + + // Bootstrap shouldn't do anything + List records = new ArrayList<>(); + migrationControl2.bootstrapInitialMigrationState(MetadataVersion.IBP_3_4_IV0, false, records::add); + assertTrue(records.isEmpty()); + assertEquals(ZkMigrationState.POST_MIGRATION, migrationControl2.zkMigrationState()); } @Test @@ -176,4 +241,40 @@ public void testMigrationDisabledStateTransitions() { assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); verifyNoStateTransitionAllowed(migrationControl); } + + private ZkMigrationControlManager verifyUpgradeFrom34(boolean migrationEnabled) { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + ZkMigrationControlManager migrationControl = new ZkMigrationControlManager( + snapshotRegistry, new LogContext(), migrationEnabled + ); + + // In 3.4, we only ever wrote PRE_MIGRATION and PreMigrationSupported tagged field wasn't present + migrationControl.replay( + new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value())); + + assertEquals(migrationControl.zkMigrationState(), ZkMigrationState.PRE_MIGRATION); + assertFalse(migrationControl.inPreMigrationMode(MetadataVersion.IBP_3_4_IV0)); + + // Now bootstrap as if we're starting up in 3.5 + List records = new ArrayList<>(); + migrationControl.bootstrapInitialMigrationState(MetadataVersion.IBP_3_4_IV0, false, records::add); + records.forEach(record -> { + migrationControl.replay((ZkMigrationStateRecord) record.message()); + }); + return migrationControl; + } + + @Test + public void testUpgradeFrom34MigrationEnabled() { + ZkMigrationControlManager migrationControl = verifyUpgradeFrom34(true); + assertEquals(migrationControl.zkMigrationState(), ZkMigrationState.MIGRATION); + assertFalse(migrationControl.inPreMigrationMode(MetadataVersion.IBP_3_4_IV0)); + } + + @Test + public void testUpgradeFrom34MigrationDisabled() { + ZkMigrationControlManager migrationControl = verifyUpgradeFrom34(false); + assertEquals(migrationControl.zkMigrationState(), ZkMigrationState.POST_MIGRATION); + assertFalse(migrationControl.inPreMigrationMode(MetadataVersion.IBP_3_4_IV0)); + } } diff --git a/tests/kafkatest/tests/core/zookeeper_migration_test.py b/tests/kafkatest/tests/core/zookeeper_migration_test.py index d5ad2ec6f66b8..0a0a830a3ea80 100644 --- a/tests/kafkatest/tests/core/zookeeper_migration_test.py +++ b/tests/kafkatest/tests/core/zookeeper_migration_test.py @@ -131,7 +131,7 @@ def test_pre_migration_mode_3_4(self, metadata_quorum): This test ensures that even if we enable migrations after the upgrade to 3.5, that no migration is able to take place. """ - self.zk = ZookeeperService(self.test_context, num_nodes=1, version=DEV_BRANCH) + self.zk = ZookeeperService(self.test_context, num_nodes=1, version=V_3_4_0) self.zk.start() self.kafka = KafkaService(self.test_context, @@ -187,3 +187,76 @@ def test_pre_migration_mode_3_4(self, metadata_quorum): continue assert saw_expected_error, "Did not see expected ERROR log for ZkMigrationState = NONE in the controller logs" + + def test_upgrade_after_3_4_migration(self): + """ + Perform a migration on version 3.4.0. Then do a rolling upgrade to 3.5+ and ensure we see + the correct migration state in the log. + """ + zk_quorum = partial(ServiceQuorumInfo, zk) + self.zk = ZookeeperService(self.test_context, num_nodes=1, version=V_3_4_0) + self.kafka = KafkaService(self.test_context, + num_nodes=3, + zk=self.zk, + version=V_3_4_0, + quorum_info_provider=zk_quorum, + allow_zk_with_kraft=True, + server_prop_overrides=[["zookeeper.metadata.migration.enable", "true"]]) + + remote_quorum = partial(ServiceQuorumInfo, isolated_kraft) + controller = KafkaService(self.test_context, num_nodes=1, zk=self.zk, version=V_3_4_0, + allow_zk_with_kraft=True, + isolated_kafka=self.kafka, + server_prop_overrides=[["zookeeper.connect", self.zk.connect_setting()], + ["zookeeper.metadata.migration.enable", "true"]], + quorum_info_provider=remote_quorum) + + self.kafka.security_protocol = "PLAINTEXT" + self.kafka.interbroker_security_protocol = "PLAINTEXT" + self.zk.start() + + controller.start() + + self.logger.info("Pre-generating clusterId for ZK.") + cluster_id_json = """{"version": "1", "id": "%s"}""" % CLUSTER_ID + self.zk.create(path="/cluster") + self.zk.create(path="/cluster/id", value=cluster_id_json) + self.kafka.reconfigure_zk_for_migration(controller) + self.kafka.start() + + topic_cfg = { + "topic": self.topic, + "partitions": self.partitions, + "replication-factor": self.replication_factor, + "configs": {"min.insync.replicas": 2} + } + self.kafka.create_topic(topic_cfg) + + # Now we're in dual-write mode. Upgrade the controller to 3.5+ + for node in controller.nodes: + self.logger.info("Stopping controller node %s" % node.account.hostname) + self.kafka.controller_quorum.stop_node(node) + node.version = DEV_BRANCH + self.logger.info("Restarting controller node %s" % node.account.hostname) + self.kafka.controller_quorum.start_node(node) + self.wait_until_rejoin() + self.logger.info("Successfully restarted controller node %s" % node.account.hostname) + + # Check the controller's logs for the error message about the migration state + saw_expected_error = False + for node in self.kafka.controller_quorum.nodes: + with node.account.monitor_log(KafkaService.STDOUT_STDERR_CAPTURE) as monitor: + monitor.offset = 0 + try: + # Shouldn't have to wait too long to see this log message after startup + monitor.wait_until( + "read a ZkMigrationState of PRE_MIGRATION from a ZK migration on Apache Kafka 3.4.", + timeout_sec=10.0, backoff_sec=.25, + err_msg="" + ) + saw_expected_error = True + break + except TimeoutError: + continue + + assert saw_expected_error, "Did not see expected INFO log for detecting upgrade after 3.4 migration" From b3f52b0bbf7ca9f761d43122696b215e63fa0bb9 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Mon, 27 Mar 2023 10:27:17 -0400 Subject: [PATCH 12/27] Better log message in ZkMigrationControlManager --- .../kafka/controller/ZkMigrationControlManager.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java index 24d91078b9b30..26801c8805b72 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java @@ -185,7 +185,7 @@ private ApiMessageAndVersion buildRecord(ZkMigrationState state) { } /** - * The state changes we allow are: + * Allowed state changes: *
  • UNINITIALIZED -> ANY
    • *
  • PRE_MIGRATION -> MIGRATION
    • *
  • MIGRATION -> POST_MIGRATION
    • @@ -202,7 +202,8 @@ void replay(ZkMigrationStateRecord record) { throw new IllegalStateException("Cannot ever change migration state away from NONE"); case PRE_MIGRATION: if (recordState.equals(ZkMigrationState.MIGRATION)) { - log.info("Transitioning ZK migration state to {}", recordState); + log.info("Transitioning ZK migration state from {} to {}", + currentState.zkMigrationState(), recordState); migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); } else { throw new IllegalStateException("Cannot change migration state from PRE_MIGRATION to " + recordState); @@ -210,7 +211,8 @@ void replay(ZkMigrationStateRecord record) { break; case MIGRATION: if (recordState.equals(ZkMigrationState.POST_MIGRATION)) { - log.info("Transitioning ZK migration state to {}", recordState); + log.info("Transitioning ZK migration state from {} to {}", + currentState.zkMigrationState(), recordState); migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); } else { throw new IllegalStateException("Cannot change migration state from MIGRATION to " + recordState); From b52e5a10392dd0ffc1acd866de957d00cc63f534 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Tue, 28 Mar 2023 13:23:24 -0400 Subject: [PATCH 13/27] Remove ControllerMetrics --- .../main/java/org/apache/kafka/controller/ControllerMetrics.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 metadata/src/main/java/org/apache/kafka/controller/ControllerMetrics.java diff --git a/metadata/src/main/java/org/apache/kafka/controller/ControllerMetrics.java b/metadata/src/main/java/org/apache/kafka/controller/ControllerMetrics.java deleted file mode 100644 index e69de29bb2d1d..0000000000000 From ce8fe6c56e0d6d47e215b0acd4970dda876e8c3f Mon Sep 17 00:00:00 2001 From: David Arthur Date: Thu, 30 Mar 2023 13:50:27 -0400 Subject: [PATCH 14/27] Move ZkMigrationControlManager into FeatureControlManager --- .../unit/kafka/server/MetadataCacheTest.scala | 5 +- .../kafka/server/ReplicaManagerTest.scala | 1 - .../controller/FeatureControlManager.java | 203 +++++++++++++++- .../kafka/controller/QuorumController.java | 18 +- .../controller/ZkMigrationBootstrap.java | 39 --- .../controller/ZkMigrationControlManager.java | 227 ------------------ .../ControllerMetadataMetricsPublisher.java | 6 +- .../org/apache/kafka/image/FeaturesDelta.java | 22 +- .../org/apache/kafka/image/FeaturesImage.java | 38 ++- .../org/apache/kafka/image/MetadataDelta.java | 24 +- .../org/apache/kafka/image/MetadataImage.java | 24 +- .../kafka/image/ZkMigrationStateDelta.java | 47 ---- .../kafka/image/ZkMigrationStateImage.java | 73 ------ .../migration/KRaftMigrationDriver.java | 4 +- .../controller/ClusterControlManagerTest.java | 10 - .../controller/FeatureControlManagerTest.java | 12 +- .../ProducerIdControlManagerTest.java | 2 - .../controller/QuorumControllerTest.java | 2 +- .../ReplicationControlManagerTest.java | 2 - ...lManagerTest.java => ZkMigrationTest.java} | 94 ++++---- ...ontrollerMetadataMetricsPublisherTest.java | 4 +- .../apache/kafka/image/FeaturesImageTest.java | 5 +- .../apache/kafka/image/MetadataImageTest.java | 6 +- 23 files changed, 319 insertions(+), 549 deletions(-) delete mode 100644 metadata/src/main/java/org/apache/kafka/controller/ZkMigrationBootstrap.java delete mode 100644 metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java delete mode 100644 metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateDelta.java delete mode 100644 metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateImage.java rename metadata/src/test/java/org/apache/kafka/controller/{ZkMigrationControlManagerTest.java => ZkMigrationTest.java} (82%) diff --git a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala index 28e4349387ff3..9685a613970ce 100644 --- a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala +++ b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala @@ -32,7 +32,7 @@ import org.apache.kafka.common.requests.UpdateMetadataRequest import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.metadata.{BrokerRegistrationChangeRecord, PartitionRecord, RegisterBrokerRecord, RemoveTopicRecord, TopicRecord} import org.apache.kafka.common.metadata.RegisterBrokerRecord.{BrokerEndpoint, BrokerEndpointCollection} -import org.apache.kafka.image.{ClusterImage, MetadataDelta, MetadataImage, MetadataProvenance, ZkMigrationStateImage} +import org.apache.kafka.image.{ClusterImage, MetadataDelta, MetadataImage, MetadataProvenance} import org.apache.kafka.server.common.MetadataVersion import org.junit.jupiter.api.Assertions._ @@ -72,8 +72,7 @@ object MetadataCacheTest { image.clientQuotas(), image.producerIds(), image.acls(), - image.scram(), - ZkMigrationStateImage.EMPTY) + image.scram()) val delta = new MetadataDelta.Builder().setImage(partialImage).build() def toRecord(broker: UpdateMetadataBroker): RegisterBrokerRecord = { diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index e2c681184973b..9bf7cd51c30c1 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -4365,7 +4365,6 @@ class ReplicaManagerTest { ProducerIdsImage.EMPTY, AclsImage.EMPTY, ScramImage.EMPTY, - ZkMigrationStateImage.EMPTY ) } diff --git a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java index c8447cb933730..fa296ab543e01 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java @@ -30,11 +30,13 @@ import org.apache.kafka.clients.ApiVersions; import org.apache.kafka.clients.admin.FeatureUpdate; import org.apache.kafka.common.metadata.FeatureLevelRecord; +import org.apache.kafka.common.metadata.ZkMigrationStateRecord; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ApiError; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.metadata.FinalizedControllerFeatures; import org.apache.kafka.metadata.VersionRange; +import org.apache.kafka.metadata.migration.ZkMigrationState; import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; import org.apache.kafka.timeline.SnapshotRegistry; @@ -46,13 +48,35 @@ public class FeatureControlManager { + + /** + * Value class for the migration state and support for pre-migration (derived from MetadataVersion). + */ + private static class ZkMigrationControlState { + private final ZkMigrationState zkMigrationState; + private final boolean preMigrationSupported; + + private ZkMigrationControlState(ZkMigrationState zkMigrationState, boolean preMigrationSupported) { + this.zkMigrationState = zkMigrationState; + this.preMigrationSupported = preMigrationSupported; + } + + public ZkMigrationState zkMigrationState() { + return zkMigrationState; + } + + public boolean preMigrationSupported() { + return preMigrationSupported; + } + } + public static class Builder { private LogContext logContext = null; private SnapshotRegistry snapshotRegistry = null; private QuorumFeatures quorumFeatures = null; private MetadataVersion metadataVersion = MetadataVersion.latest(); private MetadataVersion minimumBootstrapVersion = MetadataVersion.MINIMUM_BOOTSTRAP_VERSION; - private ZkMigrationBootstrap zkMigrationBootstrap = null; + private boolean zkMigrationEnabled = false; Builder setLogContext(LogContext logContext) { this.logContext = logContext; @@ -79,8 +103,8 @@ Builder setMinimumBootstrapVersion(MetadataVersion minimumBootstrapVersion) { return this; } - Builder setZkMigrationBootstrap(ZkMigrationBootstrap zkMigrationBootstrap) { - this.zkMigrationBootstrap = zkMigrationBootstrap; + Builder setZkMigrationEnabled(boolean zkMigrationEnabled) { + this.zkMigrationEnabled = zkMigrationEnabled; return this; } @@ -91,16 +115,13 @@ public FeatureControlManager build() { quorumFeatures = new QuorumFeatures(0, new ApiVersions(), QuorumFeatures.defaultFeatureMap(), Collections.emptyList()); } - if (zkMigrationBootstrap == null) { - throw new IllegalStateException("ZkMigrationBootstrap cannot be null"); - } return new FeatureControlManager( logContext, quorumFeatures, snapshotRegistry, metadataVersion, minimumBootstrapVersion, - zkMigrationBootstrap + zkMigrationEnabled ); } } @@ -122,12 +143,14 @@ public FeatureControlManager build() { */ private final TimelineObject metadataVersion; + private final TimelineObject migrationControlState; + /** * The minimum bootstrap version that we can't downgrade before. */ private final MetadataVersion minimumBootstrapVersion; - private final ZkMigrationBootstrap zkMigrationBootstrap; + private final boolean zkMigrationEnabled; private FeatureControlManager( LogContext logContext, @@ -135,14 +158,16 @@ private FeatureControlManager( SnapshotRegistry snapshotRegistry, MetadataVersion metadataVersion, MetadataVersion minimumBootstrapVersion, - ZkMigrationBootstrap zkMigrationBootstrap + boolean zkMigrationEnabled ) { this.log = logContext.logger(FeatureControlManager.class); this.quorumFeatures = quorumFeatures; this.finalizedVersions = new TimelineHashMap<>(snapshotRegistry, 0); this.metadataVersion = new TimelineObject<>(snapshotRegistry, metadataVersion); this.minimumBootstrapVersion = minimumBootstrapVersion; - this.zkMigrationBootstrap = zkMigrationBootstrap; + this.migrationControlState = new TimelineObject<>(snapshotRegistry, + new ZkMigrationControlState(ZkMigrationState.UNINITIALIZED, false)); + this.zkMigrationEnabled = zkMigrationEnabled; } ControllerResult> updateFeatures( @@ -169,6 +194,10 @@ MetadataVersion metadataVersion() { return metadataVersion.get(); } + ZkMigrationState zkMigrationState() { + return migrationControlState.get().zkMigrationState(); + } + private ApiError updateFeature( String featureName, short newVersion, @@ -285,7 +314,7 @@ private ApiError updateMetadataVersion( // If we are moving to a version that supports migrations, we need to write the correct state into the log if (newVersion.isMigrationSupported()) { - zkMigrationBootstrap.bootstrapInitialMigrationState(newVersion, false, recordConsumer); + generateZkMigrationRecord(newVersion, false, recordConsumer); } return ApiError.NONE; } @@ -305,6 +334,120 @@ FinalizedControllerFeatures finalizedFeatures(long epoch) { return new FinalizedControllerFeatures(features, epoch); } + + /** + * Optionally provides a ZkMigrationStateRecord to bootstrap into the metadata log. In the case of + * upgrades, the log will not be empty and this will return a NONE state record. For an empty log, + * this will return either a NONE or PRE_MIGRATION depending on the configuration and metadata.version. + *

    + * If the log is in PRE_MIGRATION, this will throw an error. + * + * @param metadataVersion The current MetadataVersion of the log + * @param isMetadataLogEmpty True if the log is being initialized from empty + * @param recordConsumer A consumer for the ZkMigrationStateRecord + */ + public void generateZkMigrationRecord( + MetadataVersion metadataVersion, + boolean isMetadataLogEmpty, + Consumer recordConsumer + ) { + if (!metadataVersion.isMigrationSupported()) { + return; + } + + if (isMetadataLogEmpty) { + // Initialize the log with a ZkMigrationState + if (zkMigrationEnabled) { + log.info("Writing ZkMigrationState of PRE_MIGRATION to the log, since migrations are enabled."); + recordConsumer.accept(buildRecord(ZkMigrationState.PRE_MIGRATION)); + } else { + log.debug("Writing ZkMigrationState of NONE to the log, since migrations are not enabled."); + recordConsumer.accept(buildRecord(ZkMigrationState.NONE)); + } + } else { + // non-empty log + String prefix = "During metadata log initialization,"; + ZkMigrationControlState state = migrationControlState.get(); + switch (state.zkMigrationState()) { + case UNINITIALIZED: + // No ZkMigrationState record has been seen yet + log.debug("{} did not read any ZkMigrationState. Writing a ZkMigrationState of NONE to the log to " + + "indicate this cluster was not migrated from ZK.", prefix); + if (zkMigrationEnabled) { + log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); + } + recordConsumer.accept(buildRecord(ZkMigrationState.NONE)); + break; + case NONE: + // This is a non-migrated KRaft cluster + log.debug("{} read a ZkMigrationState of NONE indicating this cluster was never migrated from ZK.", prefix); + if (zkMigrationEnabled) { + log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); + } + break; + case POST_MIGRATION: + // This is a migrated KRaft cluster + log.debug("{} read a ZkMigrationState of POST_MIGRATION indicating this cluster was previously migrated from ZK.", prefix); + break; + case MIGRATION: + // This cluster is migrated, but still in dual-write mode + if (zkMigrationEnabled) { + log.debug("{} read a ZkMigrationState of MIGRATION indicating this cluster is being migrated from ZK.", prefix); + } else { + throw new IllegalStateException( + prefix + " read a ZkMigrationState of MIGRATION indicating this cluster is being migrated " + + "from ZK, but the controller does not have migrations enabled." + ); + } + break; + case PRE_MIGRATION: + if (!state.preMigrationSupported()) { + // Upgrade case from 3.4. The controller only wrote PRE_MIGRATION during migrations in that version, + // so this needs to complete that migration. + log.info("{} read a ZkMigrationState of PRE_MIGRATION from a ZK migration on Apache Kafka 3.4.", prefix); + if (zkMigrationEnabled) { + recordConsumer.accept(buildRecord(ZkMigrationState.MIGRATION)); + log.info("Writing ZkMigrationState of MIGRATION since migration mode is still active on the controller."); + } else { + recordConsumer.accept(buildRecord(ZkMigrationState.MIGRATION)); + recordConsumer.accept(buildRecord(ZkMigrationState.POST_MIGRATION)); + log.info("Writing ZkMigrationState of POST_MIGRATION since migration mode is not active on the controller."); + } + } else { + log.error("{} read a ZkMigrationState of PRE_MIGRATION indicating this cluster failed during a ZK migration.", prefix); + throw new IllegalStateException("Detected an invalid migration state during startup, cannot continue."); + } + break; + default: + throw new IllegalStateException("Unsupported migration state " + state.zkMigrationState()); + } + } + } + + /** + * Tests if the controller should be preventing metadata updates due to being in the PRE_MIGRATION + * state. If the controller does not yet support migrations (before 3.4-IV0), then metadata updates + * are allowed in any state. Once the controller has been upgraded to a version that supports + * migrations, then this method checks if the controller is in the PRE_MIGRATION state. + */ + boolean inPreMigrationMode(MetadataVersion metadataVersion) { + ZkMigrationControlState state = migrationControlState.get(); + if (metadataVersion.isMigrationSupported() && state.preMigrationSupported()) { + return state.zkMigrationState() == ZkMigrationState.PRE_MIGRATION; + } else { + return false; + } + } + + private ApiMessageAndVersion buildRecord(ZkMigrationState state) { + return new ApiMessageAndVersion( + new ZkMigrationStateRecord() + .setZkMigrationState(state.value()) + .setPreMigrationSupported(true), + ZkMigrationStateRecord.HIGHEST_SUPPORTED_VERSION + ); + } + public void replay(FeatureLevelRecord record) { VersionRange range = quorumFeatures.localSupportedFeature(record.name()); if (!range.contains(record.featureLevel())) { @@ -325,6 +468,44 @@ public void replay(FeatureLevelRecord record) { } } + /** + * Allowed state changes: + *

  • UNINITIALIZED -> ANY
    • + *
  • PRE_MIGRATION -> MIGRATION
    • + *
  • MIGRATION -> POST_MIGRATION
    • + */ + void replay(ZkMigrationStateRecord record) { + ZkMigrationState recordState = ZkMigrationState.of(record.zkMigrationState()); + ZkMigrationControlState currentState = migrationControlState.get(); + if (currentState.zkMigrationState().equals(ZkMigrationState.UNINITIALIZED)) { + log.info("Initializing ZK migration state as {}", recordState); + migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); + } else { + switch (currentState.zkMigrationState()) { + case NONE: + throw new IllegalStateException("Cannot ever change migration state away from NONE"); + case PRE_MIGRATION: + if (recordState.equals(ZkMigrationState.MIGRATION)) { + log.info("Transitioning ZK migration state from {} to {}", currentState.zkMigrationState(), recordState); + migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); + } else { + throw new IllegalStateException("Cannot change migration state from PRE_MIGRATION to " + recordState); + } + break; + case MIGRATION: + if (recordState.equals(ZkMigrationState.POST_MIGRATION)) { + log.info("Transitioning ZK migration state from {} to {}", currentState.zkMigrationState(), recordState); + migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); + } else { + throw new IllegalStateException("Cannot change migration state from MIGRATION to " + recordState); + } + break; + case POST_MIGRATION: + throw new IllegalStateException("Cannot ever change migration state from POST_MIGRATION"); + } + } + } + boolean isControllerId(int nodeId) { return quorumFeatures.isControllerId(nodeId); } diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index b77bac8e5dc1c..9b48958bb2062 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -576,11 +576,6 @@ ConfigurationControlManager configurationControl() { return configurationControl; } - // Visible for testing - ZkMigrationControlManager migrationControl() { - return zkMigrationControlManager; - } - public ZkRecordConsumer zkRecordConsumer() { return zkRecordConsumer; } @@ -688,7 +683,7 @@ public void run() throws Exception { if (!isActiveController(controllerEpoch)) { throw newNotControllerException(); } - if (zkMigrationControlManager.inPreMigrationMode(featureControl.metadataVersion()) && + if (featureControl.inPreMigrationMode(featureControl.metadataVersion()) && !flags.contains(RUNS_IN_PREMIGRATION)) { log.info("Cannot run write operation {} in pre-migration mode. Returning NOT_CONTROLLER.", name); throw newNotControllerException(); @@ -1197,7 +1192,7 @@ public ControllerResult generateRecordsAndResult() throws Exception { records.addAll(bootstrapMetadata.records()); // Bootstrap the initial ZK Migration record - zkMigrationControlManager.bootstrapInitialMigrationState( + featureControl.generateZkMigrationRecord( bootstrapMetadata.metadataVersion(), true, records::add); } else { // Logs have been replayed. We need to initialize some things here if upgrading from older KRaft versions @@ -1207,7 +1202,7 @@ public ControllerResult generateRecordsAndResult() throws Exception { } // Initialize the ZK migration state as NONE, or throw an error if we find an in-progress migration - zkMigrationControlManager.bootstrapInitialMigrationState( + featureControl.generateZkMigrationRecord( featureControl.metadataVersion(), false, records::add); } return ControllerResult.atomicOf(records, null); @@ -1493,7 +1488,7 @@ private void replay(ApiMessage message, Optional snapshotId, lon // NoOpRecord is an empty record and doesn't need to be replayed break; case ZK_MIGRATION_STATE_RECORD: - zkMigrationControlManager.replay((ZkMigrationStateRecord) message); + featureControl.replay((ZkMigrationStateRecord) message); break; default: throw new RuntimeException("Unhandled record type " + type); @@ -1619,8 +1614,6 @@ private void resetToEmptyState() { */ private final AclControlManager aclControlManager; - private final ZkMigrationControlManager zkMigrationControlManager; - /** * Tracks replaying the log. * This must be accessed only by the event queue thread. @@ -1773,7 +1766,6 @@ private QuorumController( setNodeId(nodeId). build(); this.clientQuotaControlManager = new ClientQuotaControlManager(snapshotRegistry); - this.zkMigrationControlManager = new ZkMigrationControlManager(snapshotRegistry, logContext, zkMigrationEnabled); this.featureControl = new FeatureControlManager.Builder(). setLogContext(logContext). setQuorumFeatures(quorumFeatures). @@ -1784,7 +1776,7 @@ private QuorumController( // are all treated as 3.0IV1. In newer versions the metadata.version will be specified // by the log. setMetadataVersion(MetadataVersion.MINIMUM_KRAFT_VERSION). - setZkMigrationBootstrap(zkMigrationControlManager). + setZkMigrationEnabled(zkMigrationEnabled). build(); this.clusterControl = new ClusterControlManager.Builder(). setLogContext(logContext). diff --git a/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationBootstrap.java b/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationBootstrap.java deleted file mode 100644 index 052d64006e032..0000000000000 --- a/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationBootstrap.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kafka.controller; - -import org.apache.kafka.server.common.ApiMessageAndVersion; -import org.apache.kafka.server.common.MetadataVersion; - -import java.util.function.Consumer; - -public interface ZkMigrationBootstrap { - /** - * Initialize the metadata log with the correct ZkMigrationStateRecord. If the log is in an invalid - * state, this method may throw an exception. - * - * @param metadataVersion The current MetadataVersion of the log - * @param isMetadataLogEmpty True if the log is being initialized from empty - * @param recordConsumer A consumer for the ZkMigrationStateRecord - */ - void bootstrapInitialMigrationState( - MetadataVersion metadataVersion, - boolean isMetadataLogEmpty, - Consumer recordConsumer - ); -} diff --git a/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java deleted file mode 100644 index 26801c8805b72..0000000000000 --- a/metadata/src/main/java/org/apache/kafka/controller/ZkMigrationControlManager.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.controller; - -import org.apache.kafka.common.metadata.ZkMigrationStateRecord; -import org.apache.kafka.common.utils.LogContext; -import org.apache.kafka.metadata.migration.ZkMigrationState; -import org.apache.kafka.server.common.ApiMessageAndVersion; -import org.apache.kafka.server.common.MetadataVersion; -import org.apache.kafka.timeline.SnapshotRegistry; -import org.apache.kafka.timeline.TimelineObject; -import org.slf4j.Logger; - -import java.util.function.Consumer; - - -public class ZkMigrationControlManager implements ZkMigrationBootstrap { - - private static class ZkMigrationControlState { - private final ZkMigrationState zkMigrationState; - private final boolean preMigrationSupported; - - private ZkMigrationControlState(ZkMigrationState zkMigrationState, boolean preMigrationSupported) { - this.zkMigrationState = zkMigrationState; - this.preMigrationSupported = preMigrationSupported; - } - - public ZkMigrationState zkMigrationState() { - return zkMigrationState; - } - - public boolean preMigrationSupported() { - return preMigrationSupported; - } - } - - private final TimelineObject migrationControlState; - - private final boolean zkMigrationEnabled; - - private final Logger log; - - ZkMigrationControlManager( - SnapshotRegistry snapshotRegistry, - LogContext logContext, - boolean zkMigrationEnabled - ) { - this.migrationControlState = new TimelineObject<>(snapshotRegistry, - new ZkMigrationControlState(ZkMigrationState.UNINITIALIZED, false)); - this.log = logContext.logger(ZkMigrationControlManager.class); - this.zkMigrationEnabled = zkMigrationEnabled; - } - - // Visible for testing - ZkMigrationState zkMigrationState() { - return migrationControlState.get().zkMigrationState(); - } - - /** - * Tests if the controller should be preventing metadata updates due to being in the PRE_MIGRATION - * state. If the controller does not yet support migrations (before 3.4-IV0), then metadata updates - * are allowed in any state. Once the controller has been upgraded to a version that supports - * migrations, then this method checks if the controller is in the PRE_MIGRATION state. - */ - boolean inPreMigrationMode(MetadataVersion metadataVersion) { - ZkMigrationControlState state = migrationControlState.get(); - if (metadataVersion.isMigrationSupported() && state.preMigrationSupported()) { - return state.zkMigrationState() == ZkMigrationState.PRE_MIGRATION; - } else { - return false; - } - } - - /** - * Optionally provides a ZkMigrationStateRecord to bootstrap into the metadata log. In the case of - * upgrades, the log will not be empty and this will return a NONE state record. For an empty log, - * this will return either a NONE or PRE_MIGRATION depending on the configuration and metadata.version. - *

    - * If the log is in PRE_MIGRATION, this will throw an error. - * - * @param metadataVersion The current MetadataVersion of the log - * @param isMetadataLogEmpty True if the log is being initialized from empty - * @param recordConsumer A consumer for the ZkMigrationStateRecord - */ - @Override - public void bootstrapInitialMigrationState( - MetadataVersion metadataVersion, - boolean isMetadataLogEmpty, - Consumer recordConsumer - ) { - if (!metadataVersion.isMigrationSupported()) { - return; - } - - if (isMetadataLogEmpty) { - // Initialize the log with a ZkMigrationState - if (zkMigrationEnabled) { - log.info("Writing ZkMigrationState of PRE_MIGRATION to the log, since migrations are enabled."); - recordConsumer.accept(buildRecord(ZkMigrationState.PRE_MIGRATION)); - } else { - log.debug("Writing ZkMigrationState of NONE to the log, since migrations are not enabled."); - recordConsumer.accept(buildRecord(ZkMigrationState.NONE)); - } - } else { - // non-empty log - String prefix = "During metadata log initialization,"; - ZkMigrationControlState state = migrationControlState.get(); - switch (state.zkMigrationState()) { - case UNINITIALIZED: - // No ZkMigrationState record has been seen yet - log.debug("{} did not read any ZkMigrationState. Writing a ZkMigrationState of NONE to the log to " + - "indicate this cluster was not migrated from ZK.", prefix); - if (zkMigrationEnabled) { - log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); - } - recordConsumer.accept(buildRecord(ZkMigrationState.NONE)); - break; - case NONE: - // This is a non-migrated KRaft cluster - log.debug("{} read a ZkMigrationState of NONE indicating this cluster was never migrated from ZK.", prefix); - if (zkMigrationEnabled) { - log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); - } - break; - case POST_MIGRATION: - // This is a migrated KRaft cluster - log.debug("{} read a ZkMigrationState of POST_MIGRATION indicating this cluster was previously migrated from ZK.", prefix); - break; - case MIGRATION: - // This cluster is migrated, but still in dual-write mode - if (zkMigrationEnabled) { - log.debug("{} read a ZkMigrationState of MIGRATION indicating this cluster is being migrated from ZK.", prefix); - } else { - throw new IllegalStateException( - prefix + " read a ZkMigrationState of MIGRATION indicating this cluster is being migrated " + - "from ZK, but the controller does not have migrations enabled." - ); - } - break; - case PRE_MIGRATION: - if (!state.preMigrationSupported()) { - // Upgrade case from 3.4. The controller only wrote PRE_MIGRATION during migrations in that version, - // so this needs to complete that migration. - log.info("{} read a ZkMigrationState of PRE_MIGRATION from a ZK migration on Apache Kafka 3.4.", prefix); - if (zkMigrationEnabled) { - recordConsumer.accept(buildRecord(ZkMigrationState.MIGRATION)); - log.info("Writing ZkMigrationState of MIGRATION since migration mode is still active on the controller."); - } else { - recordConsumer.accept(buildRecord(ZkMigrationState.MIGRATION)); - recordConsumer.accept(buildRecord(ZkMigrationState.POST_MIGRATION)); - log.info("Writing ZkMigrationState of POST_MIGRATION since migration mode is not active on the controller."); - } - } else { - log.error("{} read a ZkMigrationState of PRE_MIGRATION indicating this cluster failed during a ZK migration.", prefix); - throw new IllegalStateException("Detected an invalid migration state during startup, cannot continue."); - } - break; - default: - throw new IllegalStateException("Unsupported migration state " + state.zkMigrationState()); - } - } - } - - private ApiMessageAndVersion buildRecord(ZkMigrationState state) { - return new ApiMessageAndVersion( - new ZkMigrationStateRecord() - .setZkMigrationState(state.value()) - .setPreMigrationSupported(true), - ZkMigrationStateRecord.HIGHEST_SUPPORTED_VERSION - ); - } - - /** - * Allowed state changes: - *

  • UNINITIALIZED -> ANY
    • - *
  • PRE_MIGRATION -> MIGRATION
    • - *
  • MIGRATION -> POST_MIGRATION
    • - */ - void replay(ZkMigrationStateRecord record) { - ZkMigrationState recordState = ZkMigrationState.of(record.zkMigrationState()); - ZkMigrationControlState currentState = migrationControlState.get(); - if (currentState.zkMigrationState().equals(ZkMigrationState.UNINITIALIZED)) { - log.info("Initializing ZK migration state as {}", recordState); - migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); - } else { - switch (currentState.zkMigrationState()) { - case NONE: - throw new IllegalStateException("Cannot ever change migration state away from NONE"); - case PRE_MIGRATION: - if (recordState.equals(ZkMigrationState.MIGRATION)) { - log.info("Transitioning ZK migration state from {} to {}", - currentState.zkMigrationState(), recordState); - migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); - } else { - throw new IllegalStateException("Cannot change migration state from PRE_MIGRATION to " + recordState); - } - break; - case MIGRATION: - if (recordState.equals(ZkMigrationState.POST_MIGRATION)) { - log.info("Transitioning ZK migration state from {} to {}", - currentState.zkMigrationState(), recordState); - migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); - } else { - throw new IllegalStateException("Cannot change migration state from MIGRATION to " + recordState); - } - break; - case POST_MIGRATION: - throw new IllegalStateException("Cannot ever change migration state from POST_MIGRATION"); - } - } - - } -} diff --git a/metadata/src/main/java/org/apache/kafka/controller/metrics/ControllerMetadataMetricsPublisher.java b/metadata/src/main/java/org/apache/kafka/controller/metrics/ControllerMetadataMetricsPublisher.java index 598c6b4a7efbe..5d34fa22533e8 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/metrics/ControllerMetadataMetricsPublisher.java +++ b/metadata/src/main/java/org/apache/kafka/controller/metrics/ControllerMetadataMetricsPublisher.java @@ -115,8 +115,8 @@ private void publishDelta(MetadataDelta delta) { } } changes.apply(metrics); - if (delta.zkMigrationDelta() != null) { - metrics.setZkMigrationState(delta.zkMigrationDelta().updatedState()); + if (delta.featuresDelta() != null) { + delta.featuresDelta().getZkMigrationStateChange().ifPresent(metrics::setZkMigrationState); } } @@ -150,7 +150,7 @@ private void publishSnapshot(MetadataImage newImage) { metrics.setGlobalPartitionCount(totalPartitions); metrics.setOfflinePartitionCount(offlinePartitions); metrics.setPreferredReplicaImbalanceCount(partitionsWithoutPreferredLeader); - metrics.setZkMigrationState(newImage.migration().migrationState()); + metrics.setZkMigrationState(newImage.features().zkMigrationState()); } @Override diff --git a/metadata/src/main/java/org/apache/kafka/image/FeaturesDelta.java b/metadata/src/main/java/org/apache/kafka/image/FeaturesDelta.java index 7f431c2d0615f..66e371835fa93 100644 --- a/metadata/src/main/java/org/apache/kafka/image/FeaturesDelta.java +++ b/metadata/src/main/java/org/apache/kafka/image/FeaturesDelta.java @@ -18,6 +18,8 @@ package org.apache.kafka.image; import org.apache.kafka.common.metadata.FeatureLevelRecord; +import org.apache.kafka.common.metadata.ZkMigrationStateRecord; +import org.apache.kafka.metadata.migration.ZkMigrationState; import org.apache.kafka.server.common.MetadataVersion; import java.util.HashMap; @@ -36,6 +38,8 @@ public final class FeaturesDelta { private MetadataVersion metadataVersionChange = null; + private ZkMigrationState zkMigrationStateChange = null; + public FeaturesDelta(FeaturesImage image) { this.image = image; } @@ -44,6 +48,10 @@ public Map> changes() { return changes; } + public Optional getZkMigrationStateChange() { + return Optional.ofNullable(zkMigrationStateChange); + } + public Optional metadataVersionChange() { return Optional.ofNullable(metadataVersionChange); } @@ -68,6 +76,10 @@ public void replay(FeatureLevelRecord record) { } } + public void replay(ZkMigrationStateRecord record) { + this.zkMigrationStateChange = ZkMigrationState.of(record.zkMigrationState()); + } + public FeaturesImage apply() { Map newFinalizedVersions = new HashMap<>(image.finalizedVersions().size()); @@ -96,7 +108,14 @@ public FeaturesImage apply() { } else { metadataVersion = metadataVersionChange; } - return new FeaturesImage(newFinalizedVersions, metadataVersion); + + final ZkMigrationState zkMigrationState; + if (zkMigrationStateChange == null) { + zkMigrationState = image.zkMigrationState(); + } else { + zkMigrationState = zkMigrationStateChange; + } + return new FeaturesImage(newFinalizedVersions, metadataVersion, zkMigrationState); } @Override @@ -104,6 +123,7 @@ public String toString() { return "FeaturesDelta(" + "changes=" + changes + ", metadataVersionChange=" + metadataVersionChange + + ", zkMigrationStateChange=" + zkMigrationStateChange + ')'; } } diff --git a/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java b/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java index 175bf464784b3..c063549836bfc 100644 --- a/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java +++ b/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java @@ -18,8 +18,10 @@ package org.apache.kafka.image; import org.apache.kafka.common.metadata.FeatureLevelRecord; +import org.apache.kafka.common.metadata.ZkMigrationStateRecord; import org.apache.kafka.image.writer.ImageWriter; import org.apache.kafka.image.writer.ImageWriterOptions; +import org.apache.kafka.metadata.migration.ZkMigrationState; import org.apache.kafka.server.common.MetadataVersion; import java.util.ArrayList; @@ -27,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -37,19 +40,30 @@ * This class is thread-safe. */ public final class FeaturesImage { - public static final FeaturesImage EMPTY = new FeaturesImage(Collections.emptyMap(), MetadataVersion.MINIMUM_KRAFT_VERSION); + public static final FeaturesImage EMPTY = new FeaturesImage( + Collections.emptyMap(), + MetadataVersion.MINIMUM_KRAFT_VERSION, + ZkMigrationState.UNINITIALIZED + ); private final Map finalizedVersions; private final MetadataVersion metadataVersion; - public FeaturesImage(Map finalizedVersions, MetadataVersion metadataVersion) { + private final ZkMigrationState zkMigrationState; + + public FeaturesImage( + Map finalizedVersions, + MetadataVersion metadataVersion, + ZkMigrationState zkMigrationState + ) { this.finalizedVersions = Collections.unmodifiableMap(finalizedVersions); this.metadataVersion = metadataVersion; + this.zkMigrationState = zkMigrationState; } public boolean isEmpty() { - return finalizedVersions.isEmpty(); + return finalizedVersions.isEmpty() && zkMigrationState.equals(ZkMigrationState.UNINITIALIZED); } public MetadataVersion metadataVersion() { @@ -60,6 +74,10 @@ public Map finalizedVersions() { return finalizedVersions; } + public ZkMigrationState zkMigrationState() { + return zkMigrationState; + } + private Optional finalizedVersion(String feature) { return Optional.ofNullable(finalizedVersions.get(feature)); } @@ -70,6 +88,14 @@ public void write(ImageWriter writer, ImageWriterOptions options) { } else { writeFeatureLevels(writer, options); } + + if (!zkMigrationState.equals(ZkMigrationState.UNINITIALIZED)) { + if (options.metadataVersion().isMigrationSupported()) { + writer.write(0, new ZkMigrationStateRecord().setZkMigrationState(zkMigrationState.value())); + } else { + options.handleLoss("the ZK Migration state"); + } + } } private void handleFeatureLevelNotSupported(ImageWriterOptions options) { @@ -105,14 +131,15 @@ private void writeFeatureLevels(ImageWriter writer, ImageWriterOptions options) @Override public int hashCode() { - return finalizedVersions.hashCode(); + return Objects.hash(finalizedVersions, zkMigrationState); } @Override public boolean equals(Object o) { if (!(o instanceof FeaturesImage)) return false; FeaturesImage other = (FeaturesImage) o; - return finalizedVersions.equals(other.finalizedVersions); + return finalizedVersions.equals(other.finalizedVersions) && + zkMigrationState.equals(other.zkMigrationState); } @@ -121,6 +148,7 @@ public String toString() { return "FeaturesImage{" + "finalizedVersions=" + finalizedVersions + ", metadataVersion=" + metadataVersion + + ", zkMigrationState=" + zkMigrationState + '}'; } } diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java b/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java index 67c2f4034737f..7b47d54e30706 100644 --- a/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java @@ -77,8 +77,6 @@ public MetadataDelta build() { private ScramDelta scramDelta = null; - private ZkMigrationStateDelta migrationDelta = null; - public MetadataDelta(MetadataImage image) { this.image = image; } @@ -161,15 +159,6 @@ public ScramDelta getOrCreateScramDelta() { return scramDelta; } - public ZkMigrationStateDelta zkMigrationDelta() { - return migrationDelta; - } - - public ZkMigrationStateDelta getOrCreateZkMigrationDelta() { - if (migrationDelta == null) migrationDelta = new ZkMigrationStateDelta(image.migration()); - return migrationDelta; - } - public Optional metadataVersionChanged() { if (featuresDelta == null) { return Optional.empty(); @@ -325,7 +314,7 @@ public void replay(RemoveUserScramCredentialRecord record) { } public void replay(ZkMigrationStateRecord record) { - getOrCreateZkMigrationDelta().replay(record); + getOrCreateFeaturesDelta().replay(record); } /** @@ -341,7 +330,6 @@ public void finishSnapshot() { getOrCreateProducerIdsDelta().finishSnapshot(); getOrCreateAclsDelta().finishSnapshot(); getOrCreateScramDelta().finishSnapshot(); - getOrCreateZkMigrationDelta().finishSnapshot(); } public MetadataImage apply(MetadataProvenance provenance) { @@ -393,12 +381,6 @@ public MetadataImage apply(MetadataProvenance provenance) { } else { newScram = scramDelta.apply(); } - ZkMigrationStateImage newMigration; - if (migrationDelta == null) { - newMigration = image.migration(); - } else { - newMigration = migrationDelta.apply(); - } return new MetadataImage( provenance, newFeatures, @@ -408,8 +390,7 @@ public MetadataImage apply(MetadataProvenance provenance) { newClientQuotas, newProducerIds, newAcls, - newScram, - newMigration + newScram ); } @@ -424,7 +405,6 @@ public String toString() { ", producerIdsDelta=" + producerIdsDelta + ", aclsDelta=" + aclsDelta + ", scramDelta=" + scramDelta + - ", migrationDelta=" + migrationDelta + ')'; } } diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java b/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java index 1e6b460174320..8643a23e03c58 100644 --- a/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java @@ -39,8 +39,7 @@ public final class MetadataImage { ClientQuotasImage.EMPTY, ProducerIdsImage.EMPTY, AclsImage.EMPTY, - ScramImage.EMPTY, - ZkMigrationStateImage.EMPTY); + ScramImage.EMPTY); private final MetadataProvenance provenance; @@ -60,8 +59,6 @@ public final class MetadataImage { private final ScramImage scram; - private final ZkMigrationStateImage zkMigration; - public MetadataImage( MetadataProvenance provenance, FeaturesImage features, @@ -71,8 +68,7 @@ public MetadataImage( ClientQuotasImage clientQuotas, ProducerIdsImage producerIds, AclsImage acls, - ScramImage scram, - ZkMigrationStateImage migration + ScramImage scram ) { this.provenance = provenance; this.features = features; @@ -83,7 +79,6 @@ public MetadataImage( this.producerIds = producerIds; this.acls = acls; this.scram = scram; - this.zkMigration = migration; } public boolean isEmpty() { @@ -94,8 +89,7 @@ public boolean isEmpty() { clientQuotas.isEmpty() && producerIds.isEmpty() && acls.isEmpty() && - scram.isEmpty() && - zkMigration.isEmpty(); + scram.isEmpty(); } public MetadataProvenance provenance() { @@ -142,10 +136,6 @@ public ScramImage scram() { return scram; } - public ZkMigrationStateImage migration() { - return zkMigration; - } - public void write(ImageWriter writer, ImageWriterOptions options) { // Features should be written out first so we can include the metadata.version at the beginning of the // snapshot @@ -157,7 +147,6 @@ public void write(ImageWriter writer, ImageWriterOptions options) { producerIds.write(writer, options); acls.write(writer, options); scram.write(writer, options); - zkMigration.write(writer, options); writer.close(true); } @@ -173,8 +162,7 @@ public boolean equals(Object o) { clientQuotas.equals(other.clientQuotas) && producerIds.equals(other.producerIds) && acls.equals(other.acls) && - scram.equals(other.scram) && - zkMigration.equals(other.zkMigration); + scram.equals(other.scram); } @Override @@ -188,8 +176,7 @@ public int hashCode() { clientQuotas, producerIds, acls, - scram, - zkMigration); + scram); } @Override @@ -204,7 +191,6 @@ public String toString() { ", producerIdsImage=" + producerIds + ", acls=" + acls + ", scram=" + scram + - ", migration=" + zkMigration + ")"; } } diff --git a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateDelta.java b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateDelta.java deleted file mode 100644 index 09602e61f3f98..0000000000000 --- a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateDelta.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kafka.image; - -import org.apache.kafka.common.metadata.ZkMigrationStateRecord; -import org.apache.kafka.metadata.migration.ZkMigrationState; - -public class ZkMigrationStateDelta { - private final ZkMigrationStateImage image; - - private ZkMigrationState updatedState; - - public ZkMigrationStateDelta(ZkMigrationStateImage image) { - this.image = image; - } - - public ZkMigrationState updatedState() { - return updatedState; - } - - public void replay(ZkMigrationStateRecord record) { - this.updatedState = ZkMigrationState.of(record.zkMigrationState()); - } - - public void finishSnapshot() { - // no-op - } - - public ZkMigrationStateImage apply() { - return new ZkMigrationStateImage(updatedState); - } -} diff --git a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateImage.java b/metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateImage.java deleted file mode 100644 index f7aaf59dded48..0000000000000 --- a/metadata/src/main/java/org/apache/kafka/image/ZkMigrationStateImage.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kafka.image; - -import org.apache.kafka.common.metadata.ZkMigrationStateRecord; -import org.apache.kafka.image.writer.ImageWriter; -import org.apache.kafka.image.writer.ImageWriterOptions; -import org.apache.kafka.metadata.migration.ZkMigrationState; - -import java.util.Objects; - -public class ZkMigrationStateImage { - public static final ZkMigrationStateImage EMPTY = new ZkMigrationStateImage(ZkMigrationState.UNINITIALIZED); - - private final ZkMigrationState state; - - ZkMigrationStateImage(ZkMigrationState state) { - this.state = state; - } - - public ZkMigrationState migrationState() { - return state; - } - - public boolean isEmpty() { - return this.equals(ZkMigrationStateImage.EMPTY); - } - - public void write(ImageWriter writer, ImageWriterOptions options) { - if (!isEmpty()) { - if (options.metadataVersion().isMigrationSupported()) { - writer.write(0, new ZkMigrationStateRecord().setZkMigrationState(state.value())); - } else { - options.handleLoss("the ZK Migration state"); - } - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ZkMigrationStateImage that = (ZkMigrationStateImage) o; - return state == that.state; - } - - @Override - public int hashCode() { - return Objects.hash(state); - } - - @Override - public String toString() { - return "ZkMigrationImage{" + - "state=" + state + - '}'; - } -} diff --git a/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java b/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java index 455ed8fd734f9..6868fccf84040 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java +++ b/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java @@ -422,12 +422,12 @@ public void run() throws Exception { return; } - if (image.migration().isEmpty()) { + if (image.features().zkMigrationState().equals(ZkMigrationState.UNINITIALIZED)) { log.trace("Waiting until we have received metadata regarding the ZK migration before proceeding with migration"); return; } - if (image.migration().migrationState().equals(ZkMigrationState.NONE)) { + if (image.features().zkMigrationState().equals(ZkMigrationState.NONE)) { // This error message is used in zookeeper_migration_test.py::TestMigration.test_pre_migration_mode_3_4 log.error("The controller's ZkMigrationState is NONE which means this cluster should not be migrated from ZooKeeper. " + "This controller should not be configured with 'zookeeper.metadata.migration.enable' set to true. " + diff --git a/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java index 616dc6f905f35..c7dcbc5bfcd7c 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java @@ -58,7 +58,6 @@ import java.util.List; import java.util.Optional; -import static org.apache.kafka.controller.FeatureControlManagerTest.NO_OP_MIGRATION_BOOTSTRAP; import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV2; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -80,7 +79,6 @@ public void testReplay(MetadataVersion metadataVersion) { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setTime(time). @@ -142,7 +140,6 @@ public void testReplayRegisterBrokerRecord() { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setClusterId("fPZv1VBsRFmnlRvmGcOW9w"). @@ -196,7 +193,6 @@ public void testReplayBrokerRegistrationChangeRecord() { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setClusterId("fPZv1VBsRFmnlRvmGcOW9w"). @@ -252,7 +248,6 @@ public void testRegistrationWithIncorrectClusterId() throws Exception { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setClusterId("fPZv1VBsRFmnlRvmGcOW9w"). @@ -282,7 +277,6 @@ public void testRegisterBrokerRecordVersion(MetadataVersion metadataVersion) { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(metadataVersion). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setClusterId("fPZv1VBsRFmnlRvmGcOW9w"). @@ -339,7 +333,6 @@ public void testUnregister() throws Exception { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setTime(new MockTime(0, 0, 0)). @@ -374,7 +367,6 @@ public void testPlaceReplicas(int numUsableBrokers) throws Exception { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setTime(time). @@ -428,7 +420,6 @@ public void testRegistrationsToRecords(MetadataVersion metadataVersion) throws E QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(metadataVersion). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setTime(time). @@ -507,7 +498,6 @@ public void testRegistrationWithUnsupportedMetadataVersion() { MetadataVersion.IBP_3_3_IV0.featureLevel())), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.IBP_3_3_IV0). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ClusterControlManager clusterControl = new ClusterControlManager.Builder(). setClusterId("fPZv1VBsRFmnlRvmGcOW9w"). diff --git a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java index 1a19b9cd1075d..d0d64ea0d2200 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java @@ -51,8 +51,6 @@ @Timeout(value = 40) public class FeatureControlManagerTest { - public static final ZkMigrationBootstrap NO_OP_MIGRATION_BOOTSTRAP = new ZkMigrationControlManagerTest.NoOpZkMigrationBootstrap(); - @SuppressWarnings("unchecked") private static Map rangeMap(Object... args) { Map result = new HashMap<>(); @@ -98,7 +96,6 @@ public void testUpdateFeatures() { setQuorumFeatures(features("foo", 1, 2)). setSnapshotRegistry(snapshotRegistry). setMetadataVersion(MetadataVersion.IBP_3_3_IV0). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); snapshotRegistry.getOrCreateSnapshot(-1); assertEquals(new FinalizedControllerFeatures(Collections.singletonMap("metadata.version", (short) 4), -1), @@ -137,7 +134,6 @@ public void testReplay() { setQuorumFeatures(features("foo", 1, 2)). setSnapshotRegistry(snapshotRegistry). setMetadataVersion(MetadataVersion.IBP_3_3_IV0). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); manager.replay(record); snapshotRegistry.getOrCreateSnapshot(123); @@ -153,7 +149,6 @@ public void testUpdateFeaturesErrorCases() { setLogContext(logContext). setQuorumFeatures(features("foo", 1, 5, "bar", 1, 2)). setSnapshotRegistry(snapshotRegistry). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); assertEquals( @@ -216,7 +211,6 @@ public void testReplayRecords() throws Exception { setQuorumFeatures(features("foo", 1, 5, "bar", 1, 2)). setSnapshotRegistry(snapshotRegistry). setMetadataVersion(MetadataVersion.IBP_3_3_IV0). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ControllerResult> result = manager. updateFeatures(updateMap("foo", 5, "bar", 1), @@ -234,8 +228,7 @@ public void testReplayRecords() throws Exception { new FeatureControlManager.Builder(). setQuorumFeatures(features(MetadataVersion.FEATURE_NAME, MetadataVersion.IBP_3_3_IV0.featureLevel(), MetadataVersion.IBP_3_3_IV3.featureLevel())). - setMetadataVersion(MetadataVersion.IBP_3_3_IV2). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP); + setMetadataVersion(MetadataVersion.IBP_3_3_IV2); @Test public void testApplyMetadataVersionChangeRecord() { @@ -363,7 +356,6 @@ public void testCanUseSafeDowngradeIfMetadataDidNotChange() { MetadataVersion.IBP_3_0_IV0.featureLevel(), MetadataVersion.IBP_3_3_IV1.featureLevel())). setMetadataVersion(MetadataVersion.IBP_3_1_IV0). setMinimumBootstrapVersion(MetadataVersion.IBP_3_0_IV0). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); assertEquals(ControllerResult.of(Collections.emptyList(), singletonMap(MetadataVersion.FEATURE_NAME, ApiError.NONE)), @@ -380,7 +372,6 @@ public void testCanotDowngradeBefore3_3_IV0() { setQuorumFeatures(features(MetadataVersion.FEATURE_NAME, MetadataVersion.IBP_3_0_IV0.featureLevel(), MetadataVersion.IBP_3_3_IV3.featureLevel())). setMetadataVersion(MetadataVersion.IBP_3_3_IV0). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); assertEquals(ControllerResult.of(Collections.emptyList(), singletonMap(MetadataVersion.FEATURE_NAME, new ApiError(Errors.INVALID_UPDATE_VERSION, @@ -400,7 +391,6 @@ public void testCreateFeatureLevelRecords() { localSupportedFeatures.put("foo", VersionRange.of(0, 2)); FeatureControlManager manager = new FeatureControlManager.Builder(). setQuorumFeatures(new QuorumFeatures(0, new ApiVersions(), localSupportedFeatures, emptyList())). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); ControllerResult> result = manager.updateFeatures( Collections.singletonMap("foo", (short) 1), diff --git a/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java index f091556cc8730..b50083169e6ab 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java @@ -32,7 +32,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.apache.kafka.controller.FeatureControlManagerTest.NO_OP_MIGRATION_BOOTSTRAP; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -54,7 +53,6 @@ public void setUp() { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(MetadataVersion.latest()). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); clusterControl = new ClusterControlManager.Builder(). setTime(time). diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java index ef519892ad76d..723f1f0a34eda 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -1369,7 +1369,7 @@ public ZkMigrationState checkBootstrapZkMigrationRecord( ) { QuorumController active = controlEnv.activeController(); return active.appendReadEvent("read migration state", OptionalLong.empty(), - () -> active.migrationControl().zkMigrationState()).get(30, TimeUnit.SECONDS); + () -> active.featureControl().zkMigrationState()).get(30, TimeUnit.SECONDS); } } } diff --git a/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java index e098ce3e3dd9f..6aa4fe508db50 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java @@ -135,7 +135,6 @@ import static org.apache.kafka.controller.ControllerRequestContextUtil.QUOTA_EXCEEDED_IN_TEST_MSG; import static org.apache.kafka.controller.ControllerRequestContextUtil.anonymousContextFor; import static org.apache.kafka.controller.ControllerRequestContextUtil.anonymousContextWithMutationQuotaExceededFor; -import static org.apache.kafka.controller.FeatureControlManagerTest.NO_OP_MIGRATION_BOOTSTRAP; import static org.apache.kafka.metadata.LeaderConstants.NO_LEADER; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -189,7 +188,6 @@ void replay(List records) throws Exception { QuorumFeatures.defaultFeatureMap(), Collections.singletonList(0))). setMetadataVersion(metadataVersion). - setZkMigrationBootstrap(NO_OP_MIGRATION_BOOTSTRAP). build(); this.clusterControl = new ClusterControlManager.Builder(). setLogContext(logContext). diff --git a/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationTest.java similarity index 82% rename from metadata/src/test/java/org/apache/kafka/controller/ZkMigrationControlManagerTest.java rename to metadata/src/test/java/org/apache/kafka/controller/ZkMigrationTest.java index 2c5275086b25e..0fa119d97705b 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationTest.java @@ -27,7 +27,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.function.Consumer; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -35,39 +34,32 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -public class ZkMigrationControlManagerTest { - public static class NoOpZkMigrationBootstrap implements ZkMigrationBootstrap { - @Override - public void bootstrapInitialMigrationState( - MetadataVersion metadataVersion, - boolean isMetadataLogEmpty, - Consumer recordConsumer - ) { +public class ZkMigrationTest { - } - } - - private ZkMigrationControlManager setupAndBootstrap( + private FeatureControlManager setupAndBootstrap( MetadataVersion metadataVersion, boolean emptyLog, boolean zkMigrationEnabled ) { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); - ZkMigrationControlManager migrationControl = new ZkMigrationControlManager( - snapshotRegistry, new LogContext(), zkMigrationEnabled - ); + FeatureControlManager featureControlManager = new FeatureControlManager.Builder() + .setSnapshotRegistry(snapshotRegistry) + .setZkMigrationEnabled(zkMigrationEnabled) + .setMetadataVersion(metadataVersion) + .build(); + List records = new ArrayList<>(); - migrationControl.bootstrapInitialMigrationState(metadataVersion, emptyLog, records::add); + featureControlManager.generateZkMigrationRecord(metadataVersion, emptyLog, records::add); records.forEach(record -> { - migrationControl.replay((ZkMigrationStateRecord) record.message()); + featureControlManager.replay((ZkMigrationStateRecord) record.message()); }); - return migrationControl; + return featureControlManager; } - private void verifyCannotBootstrapAgain(MetadataVersion metadataVersion, ZkMigrationControlManager migrationControl) { + private void verifyCannotBootstrapAgain(MetadataVersion metadataVersion, FeatureControlManager migrationControl) { // Should not be able to produce another record in any circumstance after we've bootstrapped once try { - migrationControl.bootstrapInitialMigrationState(metadataVersion, false, + migrationControl.generateZkMigrationRecord(metadataVersion, false, record -> fail("Did not expect to get another record here, but got " + record + ". State was " + migrationControl.zkMigrationState())); } catch (IllegalStateException e) { assertEquals(e.getMessage(), "Detected an invalid migration state during startup, cannot continue."); @@ -76,7 +68,7 @@ record -> fail("Did not expect to get another record here, but got " + record + @Test public void testBootstrapEmptyLog() { - ZkMigrationControlManager migrationControl; + FeatureControlManager migrationControl; migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, true, true); assertEquals(ZkMigrationState.UNINITIALIZED, migrationControl.zkMigrationState()); @@ -97,7 +89,7 @@ public void testBootstrapEmptyLog() { @Test public void testBootstrapNonEmptyLog() { - ZkMigrationControlManager migrationControl; + FeatureControlManager migrationControl; migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, false, true); assertEquals(ZkMigrationState.UNINITIALIZED, migrationControl.zkMigrationState()); @@ -114,7 +106,7 @@ public void testBootstrapNonEmptyLog() { @Test public void testMigrationStateTransitions() { - ZkMigrationControlManager migrationControl; + FeatureControlManager migrationControl; migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, true); assertEquals(ZkMigrationState.PRE_MIGRATION, migrationControl.zkMigrationState()); @@ -142,7 +134,7 @@ public void testMigrationStateTransitions() { verifyNoStateTransitionAllowed(migrationControl); } - private void verifyNoStateTransitionAllowed(ZkMigrationControlManager migrationControl) { + private void verifyNoStateTransitionAllowed(FeatureControlManager migrationControl) { assertThrows(IllegalStateException.class, () -> migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()))); assertThrows(IllegalStateException.class, () -> @@ -156,7 +148,7 @@ private void verifyNoStateTransitionAllowed(ZkMigrationControlManager migrationC @Test public void testNonMigrationStateTransitions() { // When migrations should not be present, ensure we can't change state from NONE - ZkMigrationControlManager migrationControl; + FeatureControlManager migrationControl; migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, true); assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); verifyNoStateTransitionAllowed(migrationControl); @@ -171,9 +163,10 @@ public void testNonMigrationStateTransitions() { @Test public void testFailoverToNonMigrationControllerDuringMigration() { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); - ZkMigrationControlManager migrationControl = new ZkMigrationControlManager( - snapshotRegistry, new LogContext(), true - ); + FeatureControlManager migrationControl = new FeatureControlManager.Builder() + .setSnapshotRegistry(snapshotRegistry) + .setZkMigrationEnabled(true) + .build(); // Controller has completed migration migrationControl.replay( @@ -185,22 +178,24 @@ snapshotRegistry, new LogContext(), true assertEquals(ZkMigrationState.MIGRATION, migrationControl.zkMigrationState()); // Failover to controller that does not have migrations - ZkMigrationControlManager migrationControl2 = new ZkMigrationControlManager( - snapshotRegistry, new LogContext(), false - ); + FeatureControlManager migrationControl2 = new FeatureControlManager.Builder() + .setSnapshotRegistry(snapshotRegistry) + .setZkMigrationEnabled(false) + .build(); migrationControl2.replay( new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()) .setPreMigrationSupported(true)); - assertThrows(IllegalStateException.class, () -> migrationControl2.bootstrapInitialMigrationState( + assertThrows(IllegalStateException.class, () -> migrationControl2.generateZkMigrationRecord( MetadataVersion.IBP_3_4_IV0, false, __ -> { })); } @Test public void testFailoverToNonMigrationControllerAfterMigration() { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); - ZkMigrationControlManager migrationControl = new ZkMigrationControlManager( - snapshotRegistry, new LogContext(), true - ); + FeatureControlManager migrationControl = new FeatureControlManager.Builder() + .setSnapshotRegistry(snapshotRegistry) + .setZkMigrationEnabled(true) + .build(); // Controller has completed migration migrationControl.replay( @@ -215,16 +210,18 @@ snapshotRegistry, new LogContext(), true assertEquals(ZkMigrationState.POST_MIGRATION, migrationControl.zkMigrationState()); // Failover to controller that does not have migrations - ZkMigrationControlManager migrationControl2 = new ZkMigrationControlManager( - snapshotRegistry, new LogContext(), false - ); + + FeatureControlManager migrationControl2 = new FeatureControlManager.Builder() + .setSnapshotRegistry(snapshotRegistry) + .setZkMigrationEnabled(false) + .build(); migrationControl2.replay( new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.POST_MIGRATION.value()) .setPreMigrationSupported(true)); // Bootstrap shouldn't do anything List records = new ArrayList<>(); - migrationControl2.bootstrapInitialMigrationState(MetadataVersion.IBP_3_4_IV0, false, records::add); + migrationControl2.generateZkMigrationRecord(MetadataVersion.IBP_3_4_IV0, false, records::add); assertTrue(records.isEmpty()); assertEquals(ZkMigrationState.POST_MIGRATION, migrationControl2.zkMigrationState()); } @@ -232,7 +229,7 @@ snapshotRegistry, new LogContext(), false @Test public void testMigrationDisabledStateTransitions() { // When migrations are disabled, ensure we can't change state from NONE - ZkMigrationControlManager migrationControl; + FeatureControlManager migrationControl; migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, false); assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); verifyNoStateTransitionAllowed(migrationControl); @@ -242,11 +239,12 @@ public void testMigrationDisabledStateTransitions() { verifyNoStateTransitionAllowed(migrationControl); } - private ZkMigrationControlManager verifyUpgradeFrom34(boolean migrationEnabled) { + private FeatureControlManager verifyUpgradeFrom34(boolean migrationEnabled) { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); - ZkMigrationControlManager migrationControl = new ZkMigrationControlManager( - snapshotRegistry, new LogContext(), migrationEnabled - ); + FeatureControlManager migrationControl = new FeatureControlManager.Builder() + .setSnapshotRegistry(snapshotRegistry) + .setZkMigrationEnabled(migrationEnabled) + .build(); // In 3.4, we only ever wrote PRE_MIGRATION and PreMigrationSupported tagged field wasn't present migrationControl.replay( @@ -257,7 +255,7 @@ snapshotRegistry, new LogContext(), migrationEnabled // Now bootstrap as if we're starting up in 3.5 List records = new ArrayList<>(); - migrationControl.bootstrapInitialMigrationState(MetadataVersion.IBP_3_4_IV0, false, records::add); + migrationControl.generateZkMigrationRecord(MetadataVersion.IBP_3_4_IV0, false, records::add); records.forEach(record -> { migrationControl.replay((ZkMigrationStateRecord) record.message()); }); @@ -266,14 +264,14 @@ snapshotRegistry, new LogContext(), migrationEnabled @Test public void testUpgradeFrom34MigrationEnabled() { - ZkMigrationControlManager migrationControl = verifyUpgradeFrom34(true); + FeatureControlManager migrationControl = verifyUpgradeFrom34(true); assertEquals(migrationControl.zkMigrationState(), ZkMigrationState.MIGRATION); assertFalse(migrationControl.inPreMigrationMode(MetadataVersion.IBP_3_4_IV0)); } @Test public void testUpgradeFrom34MigrationDisabled() { - ZkMigrationControlManager migrationControl = verifyUpgradeFrom34(false); + FeatureControlManager migrationControl = verifyUpgradeFrom34(false); assertEquals(migrationControl.zkMigrationState(), ZkMigrationState.POST_MIGRATION); assertFalse(migrationControl.inPreMigrationMode(MetadataVersion.IBP_3_4_IV0)); } diff --git a/metadata/src/test/java/org/apache/kafka/controller/metrics/ControllerMetadataMetricsPublisherTest.java b/metadata/src/test/java/org/apache/kafka/controller/metrics/ControllerMetadataMetricsPublisherTest.java index 09087619559bd..e05123365f6d0 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/metrics/ControllerMetadataMetricsPublisherTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/metrics/ControllerMetadataMetricsPublisherTest.java @@ -31,7 +31,6 @@ import org.apache.kafka.image.ProducerIdsImage; import org.apache.kafka.image.ScramImage; import org.apache.kafka.image.TopicsImage; -import org.apache.kafka.image.ZkMigrationStateImage; import org.apache.kafka.image.loader.LoaderManifest; import org.apache.kafka.image.loader.LogDeltaManifest; import org.apache.kafka.image.loader.SnapshotManifest; @@ -87,8 +86,7 @@ static MetadataImage fakeImageFromTopicsImage(TopicsImage topicsImage) { ClientQuotasImage.EMPTY, ProducerIdsImage.EMPTY, AclsImage.EMPTY, - ScramImage.EMPTY, - ZkMigrationStateImage.EMPTY); + ScramImage.EMPTY); } static final TopicsImage TOPICS_IMAGE1; diff --git a/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java b/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java index 8c63cbc516d48..a1fc6fa4eac5f 100644 --- a/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java @@ -21,6 +21,7 @@ import org.apache.kafka.image.writer.ImageWriterOptions; import org.apache.kafka.image.writer.RecordListWriter; import org.apache.kafka.metadata.RecordTestUtils; +import org.apache.kafka.metadata.migration.ZkMigrationState; import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; import org.junit.jupiter.api.Test; @@ -46,7 +47,7 @@ public class FeaturesImageTest { map1.put("foo", (short) 2); map1.put("bar", (short) 1); map1.put("baz", (short) 8); - IMAGE1 = new FeaturesImage(map1, MetadataVersion.latest()); + IMAGE1 = new FeaturesImage(map1, MetadataVersion.latest(), ZkMigrationState.UNINITIALIZED); DELTA1_RECORDS = new ArrayList<>(); DELTA1_RECORDS.add(new ApiMessageAndVersion(new FeatureLevelRecord(). @@ -64,7 +65,7 @@ public class FeaturesImageTest { Map map2 = new HashMap<>(); map2.put("foo", (short) 3); - IMAGE2 = new FeaturesImage(map2, MetadataVersion.latest()); + IMAGE2 = new FeaturesImage(map2, MetadataVersion.latest(), ZkMigrationState.UNINITIALIZED); } @Test diff --git a/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java b/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java index 89780e3eac30f..2a6e3e6f3e556 100644 --- a/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java @@ -44,8 +44,7 @@ public class MetadataImageTest { ClientQuotasImageTest.IMAGE1, ProducerIdsImageTest.IMAGE1, AclsImageTest.IMAGE1, - ScramImageTest.IMAGE1, - ZkMigrationStateImage.EMPTY); + ScramImageTest.IMAGE1); DELTA1 = new MetadataDelta.Builder(). setImage(IMAGE1). @@ -68,8 +67,7 @@ public class MetadataImageTest { ClientQuotasImageTest.IMAGE2, ProducerIdsImageTest.IMAGE2, AclsImageTest.IMAGE2, - ScramImageTest.IMAGE2, - ZkMigrationStateImage.EMPTY); + ScramImageTest.IMAGE2); } @Test From 71727a669dd258d4104cef2c6d6822555e5ef9f9 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Thu, 30 Mar 2023 14:28:10 -0400 Subject: [PATCH 15/27] cleanup --- .../kafka/zk/ZkMigrationIntegrationTest.scala | 12 +++++++--- .../kafka/server/ReplicaManagerTest.scala | 2 +- .../kafka/controller/QuorumController.java | 22 +++++++++---------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala b/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala index 35bdf2a04ce04..2908b01d09032 100644 --- a/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala @@ -19,7 +19,7 @@ package kafka.zk import kafka.security.authorizer.AclEntry.{WildcardHost, WildcardPrincipalString} import kafka.server.{ConfigType, KafkaConfig} import kafka.test.ClusterInstance -import kafka.test.annotation.{AutoStart, ClusterConfigProperty, ClusterTest, Type} +import kafka.test.annotation.{AutoStart, ClusterConfigProperty, ClusterTest, ClusterTests, Type} import kafka.test.junit.ClusterTestExtensions import kafka.test.junit.ZkClusterInvocationContext.ZkClusterInstance import kafka.testkit.{KafkaClusterTestKit, TestKitNodes} @@ -334,8 +334,14 @@ class ZkMigrationIntegrationTest { * Start a KRaft cluster with migrations enabled, verify that the controller does not accept metadata changes * through the RPCs */ - @ClusterTest(clusterType = Type.KRAFT, brokers = 0, controllers = 1, metadataVersion = MetadataVersion.IBP_3_4_IV0, - serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))) + @ClusterTests(Array( + new ClusterTest(clusterType = Type.KRAFT, brokers = 0, controllers = 1, metadataVersion = MetadataVersion.IBP_3_4_IV0, + serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))), + new ClusterTest(clusterType = Type.KRAFT, brokers = 0, controllers = 1, metadataVersion = MetadataVersion.IBP_3_5_IV0, + serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))), + new ClusterTest(clusterType = Type.KRAFT, brokers = 0, controllers = 1, metadataVersion = MetadataVersion.IBP_3_5_IV1, + serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))) + )) def testPreMigrationMode(clusterInstance: ClusterInstance): Unit = { val adminClient = clusterInstance.createAdminClient() val result = adminClient.createTopics( diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 9bf7cd51c30c1..0ef3127fe1a28 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -4364,7 +4364,7 @@ class ReplicaManagerTest { ClientQuotasImage.EMPTY, ProducerIdsImage.EMPTY, AclsImage.EMPTY, - ScramImage.EMPTY, + ScramImage.EMPTY ) } diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 9b48958bb2062..785ed7ef33852 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -847,18 +847,18 @@ static long appendRecords( } CompletableFuture appendWriteEvent( - String name, - OptionalLong deadlineNs, - ControllerWriteOperation op + String name, + OptionalLong deadlineNs, + ControllerWriteOperation op ) { return appendWriteEvent(name, deadlineNs, op, EnumSet.noneOf(ControllerOperationFlag.class)); } CompletableFuture appendWriteEvent( - String name, - OptionalLong deadlineNs, - ControllerWriteOperation op, - EnumSet flags + String name, + OptionalLong deadlineNs, + ControllerWriteOperation op, + EnumSet flags ) { ControllerWriteEvent event = new ControllerWriteEvent<>(name, op, flags); if (deadlineNs.isPresent()) { @@ -1263,10 +1263,10 @@ private void renounce() { } private void scheduleDeferredWriteEvent( - String name, - long deadlineNs, - ControllerWriteOperation op, - EnumSet flags + String name, + long deadlineNs, + ControllerWriteOperation op, + EnumSet flags ) { if (!flags.contains(DOES_NOT_UPDATE_QUEUE_TIME)) { throw new RuntimeException("deferred events should not update the queue time."); From 308798b1034576d3937091db986c57dfdc300edc Mon Sep 17 00:00:00 2001 From: David Arthur Date: Wed, 12 Apr 2023 18:10:59 -0400 Subject: [PATCH 16/27] PR feedback * Move logic from FeatureControlRecord to static method on QuorumController * Remove tagged field on migration record * Swap enum values for PRE_MIGRATION and MIGRATION --- .../controller/FeatureControlManager.java | 125 +++-------- .../kafka/controller/QuorumController.java | 85 +++++--- .../org/apache/kafka/image/FeaturesImage.java | 14 +- .../migration/KRaftMigrationDriver.java | 13 +- .../metadata/migration/ZkMigrationState.java | 22 +- .../common/metadata/ZkMigrationRecord.json | 4 +- .../controller/QuorumControllerTest.java | 4 +- .../kafka/controller/ZkMigrationTest.java | 194 +++++------------- .../apache/kafka/image/FeaturesImageTest.java | 4 +- 9 files changed, 165 insertions(+), 300 deletions(-) diff --git a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java index fa296ab543e01..b71247d087ec5 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map.Entry; @@ -49,34 +50,12 @@ public class FeatureControlManager { - /** - * Value class for the migration state and support for pre-migration (derived from MetadataVersion). - */ - private static class ZkMigrationControlState { - private final ZkMigrationState zkMigrationState; - private final boolean preMigrationSupported; - - private ZkMigrationControlState(ZkMigrationState zkMigrationState, boolean preMigrationSupported) { - this.zkMigrationState = zkMigrationState; - this.preMigrationSupported = preMigrationSupported; - } - - public ZkMigrationState zkMigrationState() { - return zkMigrationState; - } - - public boolean preMigrationSupported() { - return preMigrationSupported; - } - } - public static class Builder { private LogContext logContext = null; private SnapshotRegistry snapshotRegistry = null; private QuorumFeatures quorumFeatures = null; private MetadataVersion metadataVersion = MetadataVersion.latest(); private MetadataVersion minimumBootstrapVersion = MetadataVersion.MINIMUM_BOOTSTRAP_VERSION; - private boolean zkMigrationEnabled = false; Builder setLogContext(LogContext logContext) { this.logContext = logContext; @@ -103,11 +82,6 @@ Builder setMinimumBootstrapVersion(MetadataVersion minimumBootstrapVersion) { return this; } - Builder setZkMigrationEnabled(boolean zkMigrationEnabled) { - this.zkMigrationEnabled = zkMigrationEnabled; - return this; - } - public FeatureControlManager build() { if (logContext == null) logContext = new LogContext(); if (snapshotRegistry == null) snapshotRegistry = new SnapshotRegistry(logContext); @@ -120,8 +94,7 @@ public FeatureControlManager build() { quorumFeatures, snapshotRegistry, metadataVersion, - minimumBootstrapVersion, - zkMigrationEnabled + minimumBootstrapVersion ); } } @@ -143,31 +116,29 @@ public FeatureControlManager build() { */ private final TimelineObject metadataVersion; - private final TimelineObject migrationControlState; + /** + * The current ZK migration state + */ + private final TimelineObject migrationControlState; /** * The minimum bootstrap version that we can't downgrade before. */ private final MetadataVersion minimumBootstrapVersion; - private final boolean zkMigrationEnabled; - private FeatureControlManager( LogContext logContext, QuorumFeatures quorumFeatures, SnapshotRegistry snapshotRegistry, MetadataVersion metadataVersion, - MetadataVersion minimumBootstrapVersion, - boolean zkMigrationEnabled + MetadataVersion minimumBootstrapVersion ) { this.log = logContext.logger(FeatureControlManager.class); this.quorumFeatures = quorumFeatures; this.finalizedVersions = new TimelineHashMap<>(snapshotRegistry, 0); this.metadataVersion = new TimelineObject<>(snapshotRegistry, metadataVersion); this.minimumBootstrapVersion = minimumBootstrapVersion; - this.migrationControlState = new TimelineObject<>(snapshotRegistry, - new ZkMigrationControlState(ZkMigrationState.UNINITIALIZED, false)); - this.zkMigrationEnabled = zkMigrationEnabled; + this.migrationControlState = new TimelineObject<>(snapshotRegistry, ZkMigrationState.NONE); } ControllerResult> updateFeatures( @@ -195,7 +166,7 @@ MetadataVersion metadataVersion() { } ZkMigrationState zkMigrationState() { - return migrationControlState.get().zkMigrationState(); + return migrationControlState.get(); } private ApiError updateFeature( @@ -277,6 +248,7 @@ private ApiError updateMetadataVersion( Consumer recordConsumer ) { MetadataVersion currentVersion = metadataVersion(); + ZkMigrationState zkMigrationState = zkMigrationState(); final MetadataVersion newVersion; try { newVersion = MetadataVersion.fromFeatureLevel(newVersionLevel); @@ -284,6 +256,12 @@ private ApiError updateMetadataVersion( return invalidMetadataVersion(newVersionLevel, "Unknown metadata.version."); } + // Don't allow metadata.version changes while we're migrating + if (EnumSet.of(ZkMigrationState.PRE_MIGRATION, ZkMigrationState.MIGRATION).contains(zkMigrationState)) { + return invalidMetadataVersion(newVersionLevel, "Unable to modify metadata.version while a " + + "ZK migration is in progress."); + } + // We cannot set a version earlier than IBP_3_3_IV0, since that was the first version that contained // FeatureLevelRecord itself. if (newVersion.isLessThan(minimumBootstrapVersion)) { @@ -314,7 +292,7 @@ private ApiError updateMetadataVersion( // If we are moving to a version that supports migrations, we need to write the correct state into the log if (newVersion.isMigrationSupported()) { - generateZkMigrationRecord(newVersion, false, recordConsumer); + recordConsumer.accept(buildZkMigrationRecord(ZkMigrationState.NONE)); } return ApiError.NONE; } @@ -345,7 +323,7 @@ FinalizedControllerFeatures finalizedFeatures(long epoch) { * @param metadataVersion The current MetadataVersion of the log * @param isMetadataLogEmpty True if the log is being initialized from empty * @param recordConsumer A consumer for the ZkMigrationStateRecord - */ + public void generateZkMigrationRecord( MetadataVersion metadataVersion, boolean isMetadataLogEmpty, @@ -367,17 +345,9 @@ public void generateZkMigrationRecord( } else { // non-empty log String prefix = "During metadata log initialization,"; - ZkMigrationControlState state = migrationControlState.get(); - switch (state.zkMigrationState()) { - case UNINITIALIZED: - // No ZkMigrationState record has been seen yet - log.debug("{} did not read any ZkMigrationState. Writing a ZkMigrationState of NONE to the log to " + - "indicate this cluster was not migrated from ZK.", prefix); - if (zkMigrationEnabled) { - log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); - } - recordConsumer.accept(buildRecord(ZkMigrationState.NONE)); - break; + ZkMigrationState state = migrationControlState.get(); + switch (state) { + case NONE: // This is a non-migrated KRaft cluster log.debug("{} read a ZkMigrationState of NONE indicating this cluster was never migrated from ZK.", prefix); @@ -422,7 +392,7 @@ public void generateZkMigrationRecord( throw new IllegalStateException("Unsupported migration state " + state.zkMigrationState()); } } - } + }*/ /** * Tests if the controller should be preventing metadata updates due to being in the PRE_MIGRATION @@ -431,19 +401,17 @@ public void generateZkMigrationRecord( * migrations, then this method checks if the controller is in the PRE_MIGRATION state. */ boolean inPreMigrationMode(MetadataVersion metadataVersion) { - ZkMigrationControlState state = migrationControlState.get(); - if (metadataVersion.isMigrationSupported() && state.preMigrationSupported()) { - return state.zkMigrationState() == ZkMigrationState.PRE_MIGRATION; + ZkMigrationState state = migrationControlState.get(); + if (metadataVersion.isMigrationSupported()) { + return state == ZkMigrationState.PRE_MIGRATION; } else { return false; } } - private ApiMessageAndVersion buildRecord(ZkMigrationState state) { + static ApiMessageAndVersion buildZkMigrationRecord(ZkMigrationState state) { return new ApiMessageAndVersion( - new ZkMigrationStateRecord() - .setZkMigrationState(state.value()) - .setPreMigrationSupported(true), + new ZkMigrationStateRecord().setZkMigrationState(state.value()), ZkMigrationStateRecord.HIGHEST_SUPPORTED_VERSION ); } @@ -468,42 +436,11 @@ public void replay(FeatureLevelRecord record) { } } - /** - * Allowed state changes: - *
  • UNINITIALIZED -> ANY
    • - *
  • PRE_MIGRATION -> MIGRATION
    • - *
  • MIGRATION -> POST_MIGRATION
    • - */ - void replay(ZkMigrationStateRecord record) { + public void replay(ZkMigrationStateRecord record) { ZkMigrationState recordState = ZkMigrationState.of(record.zkMigrationState()); - ZkMigrationControlState currentState = migrationControlState.get(); - if (currentState.zkMigrationState().equals(ZkMigrationState.UNINITIALIZED)) { - log.info("Initializing ZK migration state as {}", recordState); - migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); - } else { - switch (currentState.zkMigrationState()) { - case NONE: - throw new IllegalStateException("Cannot ever change migration state away from NONE"); - case PRE_MIGRATION: - if (recordState.equals(ZkMigrationState.MIGRATION)) { - log.info("Transitioning ZK migration state from {} to {}", currentState.zkMigrationState(), recordState); - migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); - } else { - throw new IllegalStateException("Cannot change migration state from PRE_MIGRATION to " + recordState); - } - break; - case MIGRATION: - if (recordState.equals(ZkMigrationState.POST_MIGRATION)) { - log.info("Transitioning ZK migration state from {} to {}", currentState.zkMigrationState(), recordState); - migrationControlState.set(new ZkMigrationControlState(recordState, record.preMigrationSupported())); - } else { - throw new IllegalStateException("Cannot change migration state from MIGRATION to " + recordState); - } - break; - case POST_MIGRATION: - throw new IllegalStateException("Cannot ever change migration state from POST_MIGRATION"); - } - } + ZkMigrationState currentState = migrationControlState.get(); + log.info("Transitioning ZK migration state from {} to {}", currentState, recordState); + migrationControlState.set(recordState); } boolean isControllerId(int nodeId) { diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 785ed7ef33852..16beff2586fb1 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclBindingFilter; +import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.errors.ApiException; import org.apache.kafka.common.errors.BrokerIdNotRegisteredException; @@ -913,9 +914,7 @@ public CompletableFuture completeMigration() { ControllerWriteEvent event = new ControllerWriteEvent<>("Complete ZK Migration", new MigrationWriteOperation( Collections.singletonList( - new ApiMessageAndVersion( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()), - ZkMigrationStateRecord.LOWEST_SUPPORTED_VERSION) + FeatureControlManager.buildZkMigrationRecord(ZkMigrationState.MIGRATION) )), EnumSet.of(RUNS_IN_PREMIGRATION)); queue.append(event); @@ -1178,33 +1177,63 @@ private void claim(int epoch) { } } + public static void generateBootstrapRecords( + Logger log, + boolean isLogEmpty, + boolean zkMigrationEnabled, + BootstrapMetadata bootstrapMetadata, + FeatureControlManager featureControl, + Consumer recordConsumer + ) { + if (isLogEmpty) { + // If no records have been replayed, we need to write out the bootstrap records. + // This will include the new metadata.version, as well as things like SCRAM + // initialization, etc. + log.info("The metadata log appears to be empty. Appending {} bootstrap record(s) " + + "at metadata.version {} from {}.", bootstrapMetadata.records().size(), + bootstrapMetadata.metadataVersion(), bootstrapMetadata.source()); + bootstrapMetadata.records().forEach(recordConsumer); + + if (bootstrapMetadata.metadataVersion().isMigrationSupported()) { + if (zkMigrationEnabled) { + log.info("Writing ZkMigrationState of PRE_MIGRATION to the log, since migrations are enabled."); + recordConsumer.accept(FeatureControlManager.buildZkMigrationRecord(ZkMigrationState.PRE_MIGRATION)); + } else { + log.debug("Writing ZkMigrationState of NONE to the log, since migrations are not enabled."); + recordConsumer.accept(FeatureControlManager.buildZkMigrationRecord(ZkMigrationState.NONE)); + } + } + } else { + // Logs have been replayed. We need to initialize some things here if upgrading from older KRaft versions + if (featureControl.metadataVersion().equals(MetadataVersion.MINIMUM_KRAFT_VERSION)) { + log.info("No metadata.version feature level record was found in the log. " + + "Treating the log as version {}.", MetadataVersion.MINIMUM_KRAFT_VERSION); + } + + if (featureControl.metadataVersion().isMigrationSupported()) { + switch (featureControl.zkMigrationState()) { + case NONE: + // There may, or may not, be a NONE in the log. Since this is the default state, it will + // be persisted in a snapshot, so we don't need to explicitly write it here. + if (zkMigrationEnabled) { + throw new ConfigException("Should not have ZK migrations enabled on a cluster that was created in KRaft mode."); + } + break; + case PRE_MIGRATION: + throw new IllegalStateException("Detected an failed migration state during bootstrap, cannot continue."); + case MIGRATION: + case POST_MIGRATION: + // These states need more context, so they will be handled by KRaftMigrationDriver + break; + } + } + } + } class CompleteActivationEvent implements ControllerWriteOperation { @Override public ControllerResult generateRecordsAndResult() throws Exception { List records = new ArrayList<>(); - if (logReplayTracker.empty()) { - // If no records have been replayed, we need to write out the bootstrap records. - // This will include the new metadata.version, as well as things like SCRAM - // initialization, etc. - log.info("The metadata log appears to be empty. Appending {} bootstrap record(s) " + - "at metadata.version {} from {}.", bootstrapMetadata.records().size(), - bootstrapMetadata.metadataVersion(), bootstrapMetadata.source()); - records.addAll(bootstrapMetadata.records()); - - // Bootstrap the initial ZK Migration record - featureControl.generateZkMigrationRecord( - bootstrapMetadata.metadataVersion(), true, records::add); - } else { - // Logs have been replayed. We need to initialize some things here if upgrading from older KRaft versions - if (featureControl.metadataVersion().equals(MetadataVersion.MINIMUM_KRAFT_VERSION)) { - log.info("No metadata.version feature level record was found in the log. " + - "Treating the log as version {}.", MetadataVersion.MINIMUM_KRAFT_VERSION); - } - - // Initialize the ZK migration state as NONE, or throw an error if we find an in-progress migration - featureControl.generateZkMigrationRecord( - featureControl.metadataVersion(), false, records::add); - } + generateBootstrapRecords(log, logReplayTracker.empty(), zkMigrationEnabled, bootstrapMetadata, featureControl, records::add); return ControllerResult.atomicOf(records, null); } @@ -1712,6 +1741,8 @@ private enum ImbalanceSchedule { private final ZkRecordConsumer zkRecordConsumer; + private final boolean zkMigrationEnabled; + /** * The maximum number of records per batch to allow. */ @@ -1776,7 +1807,6 @@ private QuorumController( // are all treated as 3.0IV1. In newer versions the metadata.version will be specified // by the log. setMetadataVersion(MetadataVersion.MINIMUM_KRAFT_VERSION). - setZkMigrationEnabled(zkMigrationEnabled). build(); this.clusterControl = new ClusterControlManager.Builder(). setLogContext(logContext). @@ -1819,6 +1849,7 @@ private QuorumController( this.curClaimEpoch = -1; this.needToCompleteAuthorizerLoad = authorizer.isPresent(); this.zkRecordConsumer = new MigrationRecordConsumer(); + this.zkMigrationEnabled = zkMigrationEnabled; updateWriteOffset(-1); resetToEmptyState(); diff --git a/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java b/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java index c063549836bfc..2f67186e6cac2 100644 --- a/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java +++ b/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java @@ -43,7 +43,7 @@ public final class FeaturesImage { public static final FeaturesImage EMPTY = new FeaturesImage( Collections.emptyMap(), MetadataVersion.MINIMUM_KRAFT_VERSION, - ZkMigrationState.UNINITIALIZED + ZkMigrationState.NONE ); private final Map finalizedVersions; @@ -63,7 +63,7 @@ public FeaturesImage( } public boolean isEmpty() { - return finalizedVersions.isEmpty() && zkMigrationState.equals(ZkMigrationState.UNINITIALIZED); + return finalizedVersions.isEmpty() && zkMigrationState.equals(ZkMigrationState.NONE); } public MetadataVersion metadataVersion() { @@ -89,11 +89,11 @@ public void write(ImageWriter writer, ImageWriterOptions options) { writeFeatureLevels(writer, options); } - if (!zkMigrationState.equals(ZkMigrationState.UNINITIALIZED)) { - if (options.metadataVersion().isMigrationSupported()) { - writer.write(0, new ZkMigrationStateRecord().setZkMigrationState(zkMigrationState.value())); - } else { - options.handleLoss("the ZK Migration state"); + if (options.metadataVersion().isMigrationSupported()) { + writer.write(0, new ZkMigrationStateRecord().setZkMigrationState(zkMigrationState.value())); + } else { + if (!zkMigrationState.equals(ZkMigrationState.NONE)) { + options.handleLoss("the ZK Migration state which was " + zkMigrationState); } } } diff --git a/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java b/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java index 6868fccf84040..c443db84d20f9 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java +++ b/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java @@ -91,6 +91,7 @@ public class KRaftMigrationDriver implements MetadataPublisher { private volatile MigrationDriverState migrationState; private volatile ZkMigrationLeadershipState migrationLeadershipState; private volatile MetadataImage image; + private volatile boolean firstPublish; public KRaftMigrationDriver( int nodeId, @@ -111,6 +112,7 @@ public KRaftMigrationDriver( this.migrationLeadershipState = ZkMigrationLeadershipState.EMPTY; this.eventQueue = new KafkaEventQueue(Time.SYSTEM, logContext, "controller-" + nodeId + "-migration-driver-"); this.image = MetadataImage.EMPTY; + this.firstPublish = false; this.leaderAndEpoch = LeaderAndEpoch.UNKNOWN; this.initialZkLoadHandler = initialZkLoadHandler; this.faultHandler = faultHandler; @@ -157,8 +159,7 @@ private boolean imageDoesNotContainAllBrokers(MetadataImage image, Set } private boolean areZkBrokersReadyForMigration() { - if (image == MetadataImage.EMPTY) { - // TODO maybe add WAIT_FOR_INITIAL_METADATA_PUBLISH state to avoid this kind of check? + if (!firstPublish) { log.info("Waiting for initial metadata publish before checking if Zk brokers are registered."); return false; } @@ -417,16 +418,11 @@ class WaitForControllerQuorumEvent extends MigrationEvent { @Override public void run() throws Exception { if (migrationState.equals(MigrationDriverState.WAIT_FOR_CONTROLLER_QUORUM)) { - if (image.isEmpty()) { + if (!firstPublish) { log.trace("Waiting until we have received metadata before proceeding with migration"); return; } - if (image.features().zkMigrationState().equals(ZkMigrationState.UNINITIALIZED)) { - log.trace("Waiting until we have received metadata regarding the ZK migration before proceeding with migration"); - return; - } - if (image.features().zkMigrationState().equals(ZkMigrationState.NONE)) { // This error message is used in zookeeper_migration_test.py::TestMigration.test_pre_migration_mode_3_4 log.error("The controller's ZkMigrationState is NONE which means this cluster should not be migrated from ZooKeeper. " + @@ -570,6 +566,7 @@ class MetadataChangeEvent extends MigrationEvent { @Override public void run() throws Exception { + KRaftMigrationDriver.this.firstPublish = true; MetadataImage prevImage = KRaftMigrationDriver.this.image; KRaftMigrationDriver.this.image = image; String metadataType = isSnapshot ? "snapshot" : "delta"; diff --git a/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkMigrationState.java b/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkMigrationState.java index 8debe053b8db8..0717b9d2f0740 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkMigrationState.java +++ b/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkMigrationState.java @@ -20,22 +20,17 @@ /** * The cluster-wide ZooKeeper migration state. - * + *

    * An enumeration of the possible states of the ZkMigrationState field in ZkMigrationStateRecord. * This information is persisted in the metadata log and image. * * @see org.apache.kafka.common.metadata.ZkMigrationStateRecord */ public enum ZkMigrationState { - /** - * This is a synthetic value used internally by the controller to indicate that no decision has - * been made about the state of a ZK migration. This value should _not_ be written into the log. - */ - UNINITIALIZED((byte) -1), - /** * The cluster was created in KRaft mode. A cluster that was created in ZK mode can never attain - * this state; the endpoint of migration is POST_MIGRATION, instead. + * this state; the endpoint of migration is POST_MIGRATION, instead. This value is also used as + * the default migration state in an empty metadata log. */ NONE((byte) 0), @@ -44,15 +39,20 @@ public enum ZkMigrationState { * The controller is now awaiting the preconditions for starting the migration to KRaft. In this * state, the metadata log does not yet contain the cluster's data. There is a metadata quorum, * but it is not doing anything useful yet. + *

    + * In Kafka 3.4, PRE_MIGRATION was written out as value 1 to the log, but no MIGRATION state + * was ever written. Since this would be an invalid log state in 3.5+, we have swapped the + * enum values for PRE_MIGRATION and MIGRATION. This allows us to handle the upgrade case + * from 3.4 without adding additional fields to the migration record. */ - PRE_MIGRATION((byte) 1), + PRE_MIGRATION((byte) 2), /** * The ZK data has been migrated, and the KRaft controller is now writing metadata to both ZK - * and the metadata log. The controller will remain in this state until all of the brokers have + * and the metadata log. The controller will remain in this state until all the brokers have * been restarted in KRaft mode. */ - MIGRATION((byte) 2), + MIGRATION((byte) 1), /** * The migration from ZK has been fully completed. The cluster is running in KRaft mode. This state diff --git a/metadata/src/main/resources/common/metadata/ZkMigrationRecord.json b/metadata/src/main/resources/common/metadata/ZkMigrationRecord.json index b89d304666574..5c8085ce72f4b 100644 --- a/metadata/src/main/resources/common/metadata/ZkMigrationRecord.json +++ b/metadata/src/main/resources/common/metadata/ZkMigrationRecord.json @@ -25,8 +25,6 @@ "flexibleVersions": "0+", "fields": [ { "name": "ZkMigrationState", "type": "int8", "versions": "0+", - "about": "One of the possible migration states." }, - { "name": "PreMigrationSupported", "type": "bool", "versions": "0+", "taggedVersions": "0+", "tag": 0, - "about": "True if the cluster supports pre-migration mode" } + "about": "One of the possible migration states." } ] } \ No newline at end of file diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java index 31d0a3f58ad18..ccd2f711582c0 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -1348,10 +1348,10 @@ public void testBootstrapZkMigrationRecord() throws Exception { assertEquals(ZkMigrationState.NONE, checkBootstrapZkMigrationRecord(MetadataVersion.IBP_3_4_IV0, false)); - assertEquals(ZkMigrationState.UNINITIALIZED, + assertEquals(ZkMigrationState.NONE, checkBootstrapZkMigrationRecord(MetadataVersion.IBP_3_3_IV0, true)); - assertEquals(ZkMigrationState.UNINITIALIZED, + assertEquals(ZkMigrationState.NONE, checkBootstrapZkMigrationRecord(MetadataVersion.IBP_3_3_IV0, false)); } diff --git a/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationTest.java b/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationTest.java index 0fa119d97705b..bb74e7d06c2d8 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationTest.java @@ -17,13 +17,17 @@ package org.apache.kafka.controller; +import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.metadata.ZkMigrationStateRecord; import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.metadata.RecordTestUtils; +import org.apache.kafka.metadata.bootstrap.BootstrapMetadata; import org.apache.kafka.metadata.migration.ZkMigrationState; import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; import org.apache.kafka.timeline.SnapshotRegistry; import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -31,7 +35,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; public class ZkMigrationTest { @@ -44,47 +47,52 @@ private FeatureControlManager setupAndBootstrap( SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); FeatureControlManager featureControlManager = new FeatureControlManager.Builder() .setSnapshotRegistry(snapshotRegistry) - .setZkMigrationEnabled(zkMigrationEnabled) .setMetadataVersion(metadataVersion) .build(); List records = new ArrayList<>(); - featureControlManager.generateZkMigrationRecord(metadataVersion, emptyLog, records::add); - records.forEach(record -> { - featureControlManager.replay((ZkMigrationStateRecord) record.message()); - }); + QuorumController.generateBootstrapRecords( + LoggerFactory.getLogger(ZkMigrationTest.class), + emptyLog, + zkMigrationEnabled, + BootstrapMetadata.fromVersion(metadataVersion, "test"), + featureControlManager, + records::add); + RecordTestUtils.replayAll(featureControlManager, records); return featureControlManager; } - private void verifyCannotBootstrapAgain(MetadataVersion metadataVersion, FeatureControlManager migrationControl) { + private void verifyCannotBootstrapAgain(MetadataVersion metadataVersion, FeatureControlManager featureControl, Class expectedException) { // Should not be able to produce another record in any circumstance after we've bootstrapped once - try { - migrationControl.generateZkMigrationRecord(metadataVersion, false, - record -> fail("Did not expect to get another record here, but got " + record + ". State was " + migrationControl.zkMigrationState())); - } catch (IllegalStateException e) { - assertEquals(e.getMessage(), "Detected an invalid migration state during startup, cannot continue."); - } + assertThrows(expectedException, () -> + QuorumController.generateBootstrapRecords( + LoggerFactory.getLogger(ZkMigrationTest.class), + false, + true, + BootstrapMetadata.fromVersion(metadataVersion, "test"), + featureControl, + record -> fail("Did not expect to get another record here, but got " + record + ". State was " + featureControl.zkMigrationState()))); } @Test public void testBootstrapEmptyLog() { FeatureControlManager migrationControl; + // In 3.3, we won't write any records migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, true, true); - assertEquals(ZkMigrationState.UNINITIALIZED, migrationControl.zkMigrationState()); - verifyCannotBootstrapAgain(MetadataVersion.IBP_3_3_IV0, migrationControl); + assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, true, false); - assertEquals(ZkMigrationState.UNINITIALIZED, migrationControl.zkMigrationState()); - verifyCannotBootstrapAgain(MetadataVersion.IBP_3_3_IV0, migrationControl); + assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); + // In 3.4, we'll write something and should only be able to bootstrap once migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, true); assertEquals(ZkMigrationState.PRE_MIGRATION, migrationControl.zkMigrationState()); - verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl); + verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl, IllegalStateException.class); migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, false); assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl); + verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl, ConfigException.class); } @Test @@ -92,138 +100,37 @@ public void testBootstrapNonEmptyLog() { FeatureControlManager migrationControl; migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, false, true); - assertEquals(ZkMigrationState.UNINITIALIZED, migrationControl.zkMigrationState()); + assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, false, false); - assertEquals(ZkMigrationState.UNINITIALIZED, migrationControl.zkMigrationState()); - - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, true); assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); + // Cannot have migrations enabled for a non-migrated KRaft cluster unless we're in a MIGRATION state + assertThrows(ConfigException.class, () -> setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, true)); + migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, false); assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); } - @Test - public void testMigrationStateTransitions() { - FeatureControlManager migrationControl; - - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, true); - assertEquals(ZkMigrationState.PRE_MIGRATION, migrationControl.zkMigrationState()); - - assertThrows(IllegalStateException.class, () -> - migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()))); - assertThrows(IllegalStateException.class, () -> - migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()))); - assertThrows(IllegalStateException.class, () -> - migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.POST_MIGRATION.value()))); - - migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value())); - assertEquals(ZkMigrationState.MIGRATION, migrationControl.zkMigrationState()); - - assertThrows(IllegalStateException.class, () -> - migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()))); - assertThrows(IllegalStateException.class, () -> - migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()))); - assertThrows(IllegalStateException.class, () -> - migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()))); - - migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.POST_MIGRATION.value())); - assertEquals(ZkMigrationState.POST_MIGRATION, migrationControl.zkMigrationState()); - - verifyNoStateTransitionAllowed(migrationControl); - } - - private void verifyNoStateTransitionAllowed(FeatureControlManager migrationControl) { - assertThrows(IllegalStateException.class, () -> - migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()))); - assertThrows(IllegalStateException.class, () -> - migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.NONE.value()))); - assertThrows(IllegalStateException.class, () -> - migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()))); - assertThrows(IllegalStateException.class, () -> - migrationControl.replay(new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.POST_MIGRATION.value()))); - } - @Test public void testNonMigrationStateTransitions() { // When migrations should not be present, ensure we can't change state from NONE FeatureControlManager migrationControl; - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, true); - assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - verifyNoStateTransitionAllowed(migrationControl); - verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl); + assertThrows(ConfigException.class, () -> setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, true)); migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, false); assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - verifyNoStateTransitionAllowed(migrationControl); - verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl); + verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl, ConfigException.class); } @Test public void testFailoverToNonMigrationControllerDuringMigration() { - SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); - FeatureControlManager migrationControl = new FeatureControlManager.Builder() - .setSnapshotRegistry(snapshotRegistry) - .setZkMigrationEnabled(true) - .build(); - // Controller has completed migration - migrationControl.replay( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()) - .setPreMigrationSupported(true)); - migrationControl.replay( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()) - .setPreMigrationSupported(true)); - assertEquals(ZkMigrationState.MIGRATION, migrationControl.zkMigrationState()); - - // Failover to controller that does not have migrations - FeatureControlManager migrationControl2 = new FeatureControlManager.Builder() - .setSnapshotRegistry(snapshotRegistry) - .setZkMigrationEnabled(false) - .build(); - migrationControl2.replay( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()) - .setPreMigrationSupported(true)); - assertThrows(IllegalStateException.class, () -> migrationControl2.generateZkMigrationRecord( - MetadataVersion.IBP_3_4_IV0, false, __ -> { })); } @Test public void testFailoverToNonMigrationControllerAfterMigration() { - SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); - FeatureControlManager migrationControl = new FeatureControlManager.Builder() - .setSnapshotRegistry(snapshotRegistry) - .setZkMigrationEnabled(true) - .build(); - - // Controller has completed migration - migrationControl.replay( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()) - .setPreMigrationSupported(true)); - migrationControl.replay( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.MIGRATION.value()) - .setPreMigrationSupported(true)); - migrationControl.replay( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.POST_MIGRATION.value()) - .setPreMigrationSupported(true)); - assertEquals(ZkMigrationState.POST_MIGRATION, migrationControl.zkMigrationState()); - - // Failover to controller that does not have migrations - FeatureControlManager migrationControl2 = new FeatureControlManager.Builder() - .setSnapshotRegistry(snapshotRegistry) - .setZkMigrationEnabled(false) - .build(); - migrationControl2.replay( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.POST_MIGRATION.value()) - .setPreMigrationSupported(true)); - - // Bootstrap shouldn't do anything - List records = new ArrayList<>(); - migrationControl2.generateZkMigrationRecord(MetadataVersion.IBP_3_4_IV0, false, records::add); - assertTrue(records.isEmpty()); - assertEquals(ZkMigrationState.POST_MIGRATION, migrationControl2.zkMigrationState()); } @Test @@ -232,34 +139,36 @@ public void testMigrationDisabledStateTransitions() { FeatureControlManager migrationControl; migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, false); assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - verifyNoStateTransitionAllowed(migrationControl); migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, false); assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - verifyNoStateTransitionAllowed(migrationControl); } private FeatureControlManager verifyUpgradeFrom34(boolean migrationEnabled) { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); - FeatureControlManager migrationControl = new FeatureControlManager.Builder() + FeatureControlManager featureControl = new FeatureControlManager.Builder() .setSnapshotRegistry(snapshotRegistry) - .setZkMigrationEnabled(migrationEnabled) .build(); - // In 3.4, we only ever wrote PRE_MIGRATION and PreMigrationSupported tagged field wasn't present - migrationControl.replay( - new ZkMigrationStateRecord().setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value())); - - assertEquals(migrationControl.zkMigrationState(), ZkMigrationState.PRE_MIGRATION); - assertFalse(migrationControl.inPreMigrationMode(MetadataVersion.IBP_3_4_IV0)); + // In 3.4, we only ever wrote PRE_MIGRATION (1) + featureControl.replay(new ZkMigrationStateRecord().setZkMigrationState((byte) 1)); + assertEquals(featureControl.zkMigrationState(), ZkMigrationState.MIGRATION); + assertFalse(featureControl.inPreMigrationMode(MetadataVersion.IBP_3_4_IV0)); // Now bootstrap as if we're starting up in 3.5 List records = new ArrayList<>(); - migrationControl.generateZkMigrationRecord(MetadataVersion.IBP_3_4_IV0, false, records::add); + QuorumController.generateBootstrapRecords( + LoggerFactory.getLogger(ZkMigrationTest.class), + false, + true, + BootstrapMetadata.fromVersion(MetadataVersion.IBP_3_4_IV0, "test"), + featureControl, + records::add); + records.forEach(record -> { - migrationControl.replay((ZkMigrationStateRecord) record.message()); + featureControl.replay((ZkMigrationStateRecord) record.message()); }); - return migrationControl; + return featureControl; } @Test @@ -268,11 +177,4 @@ public void testUpgradeFrom34MigrationEnabled() { assertEquals(migrationControl.zkMigrationState(), ZkMigrationState.MIGRATION); assertFalse(migrationControl.inPreMigrationMode(MetadataVersion.IBP_3_4_IV0)); } - - @Test - public void testUpgradeFrom34MigrationDisabled() { - FeatureControlManager migrationControl = verifyUpgradeFrom34(false); - assertEquals(migrationControl.zkMigrationState(), ZkMigrationState.POST_MIGRATION); - assertFalse(migrationControl.inPreMigrationMode(MetadataVersion.IBP_3_4_IV0)); - } } diff --git a/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java b/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java index a1fc6fa4eac5f..ec86f3c79aeeb 100644 --- a/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java @@ -47,7 +47,7 @@ public class FeaturesImageTest { map1.put("foo", (short) 2); map1.put("bar", (short) 1); map1.put("baz", (short) 8); - IMAGE1 = new FeaturesImage(map1, MetadataVersion.latest(), ZkMigrationState.UNINITIALIZED); + IMAGE1 = new FeaturesImage(map1, MetadataVersion.latest(), ZkMigrationState.NONE); DELTA1_RECORDS = new ArrayList<>(); DELTA1_RECORDS.add(new ApiMessageAndVersion(new FeatureLevelRecord(). @@ -65,7 +65,7 @@ public class FeaturesImageTest { Map map2 = new HashMap<>(); map2.put("foo", (short) 3); - IMAGE2 = new FeaturesImage(map2, MetadataVersion.latest(), ZkMigrationState.UNINITIALIZED); + IMAGE2 = new FeaturesImage(map2, MetadataVersion.latest(), ZkMigrationState.NONE); } @Test From 301296bacc9433c36818280f8c2ce812ef0a7c77 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Fri, 14 Apr 2023 10:55:07 -0400 Subject: [PATCH 17/27] More PR feedback --- .../controller/FeatureControlManager.java | 108 +---------- .../kafka/controller/QuorumController.java | 49 +++-- .../metrics/ControllerMetadataMetrics.java | 5 +- .../ControllerMetadataMetricsPublisher.java | 4 +- .../org/apache/kafka/image/FeaturesImage.java | 3 +- .../metadata/migration/ZkMigrationState.java | 10 + .../controller/FeatureControlManagerTest.java | 43 +++++ .../controller/QuorumControllerTest.java | 98 ++++++++++ .../kafka/controller/ZkMigrationTest.java | 180 ------------------ .../kafka/metadata/RecordTestUtils.java | 8 + 10 files changed, 201 insertions(+), 307 deletions(-) delete mode 100644 metadata/src/test/java/org/apache/kafka/controller/ZkMigrationTest.java diff --git a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java index b71247d087ec5..f56cd483202b5 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java @@ -290,10 +290,6 @@ private ApiError updateMetadataVersion( .setName(MetadataVersion.FEATURE_NAME) .setFeatureLevel(newVersionLevel), FEATURE_LEVEL_RECORD.lowestSupportedVersion())); - // If we are moving to a version that supports migrations, we need to write the correct state into the log - if (newVersion.isMigrationSupported()) { - recordConsumer.accept(buildZkMigrationRecord(ZkMigrationState.NONE)); - } return ApiError.NONE; } @@ -312,108 +308,14 @@ FinalizedControllerFeatures finalizedFeatures(long epoch) { return new FinalizedControllerFeatures(features, epoch); } - - /** - * Optionally provides a ZkMigrationStateRecord to bootstrap into the metadata log. In the case of - * upgrades, the log will not be empty and this will return a NONE state record. For an empty log, - * this will return either a NONE or PRE_MIGRATION depending on the configuration and metadata.version. - *

    - * If the log is in PRE_MIGRATION, this will throw an error. - * - * @param metadataVersion The current MetadataVersion of the log - * @param isMetadataLogEmpty True if the log is being initialized from empty - * @param recordConsumer A consumer for the ZkMigrationStateRecord - - public void generateZkMigrationRecord( - MetadataVersion metadataVersion, - boolean isMetadataLogEmpty, - Consumer recordConsumer - ) { - if (!metadataVersion.isMigrationSupported()) { - return; - } - - if (isMetadataLogEmpty) { - // Initialize the log with a ZkMigrationState - if (zkMigrationEnabled) { - log.info("Writing ZkMigrationState of PRE_MIGRATION to the log, since migrations are enabled."); - recordConsumer.accept(buildRecord(ZkMigrationState.PRE_MIGRATION)); - } else { - log.debug("Writing ZkMigrationState of NONE to the log, since migrations are not enabled."); - recordConsumer.accept(buildRecord(ZkMigrationState.NONE)); - } - } else { - // non-empty log - String prefix = "During metadata log initialization,"; - ZkMigrationState state = migrationControlState.get(); - switch (state) { - - case NONE: - // This is a non-migrated KRaft cluster - log.debug("{} read a ZkMigrationState of NONE indicating this cluster was never migrated from ZK.", prefix); - if (zkMigrationEnabled) { - log.error("Should not have ZK migrations enabled on a cluster that was created in KRaft mode"); - } - break; - case POST_MIGRATION: - // This is a migrated KRaft cluster - log.debug("{} read a ZkMigrationState of POST_MIGRATION indicating this cluster was previously migrated from ZK.", prefix); - break; - case MIGRATION: - // This cluster is migrated, but still in dual-write mode - if (zkMigrationEnabled) { - log.debug("{} read a ZkMigrationState of MIGRATION indicating this cluster is being migrated from ZK.", prefix); - } else { - throw new IllegalStateException( - prefix + " read a ZkMigrationState of MIGRATION indicating this cluster is being migrated " + - "from ZK, but the controller does not have migrations enabled." - ); - } - break; - case PRE_MIGRATION: - if (!state.preMigrationSupported()) { - // Upgrade case from 3.4. The controller only wrote PRE_MIGRATION during migrations in that version, - // so this needs to complete that migration. - log.info("{} read a ZkMigrationState of PRE_MIGRATION from a ZK migration on Apache Kafka 3.4.", prefix); - if (zkMigrationEnabled) { - recordConsumer.accept(buildRecord(ZkMigrationState.MIGRATION)); - log.info("Writing ZkMigrationState of MIGRATION since migration mode is still active on the controller."); - } else { - recordConsumer.accept(buildRecord(ZkMigrationState.MIGRATION)); - recordConsumer.accept(buildRecord(ZkMigrationState.POST_MIGRATION)); - log.info("Writing ZkMigrationState of POST_MIGRATION since migration mode is not active on the controller."); - } - } else { - log.error("{} read a ZkMigrationState of PRE_MIGRATION indicating this cluster failed during a ZK migration.", prefix); - throw new IllegalStateException("Detected an invalid migration state during startup, cannot continue."); - } - break; - default: - throw new IllegalStateException("Unsupported migration state " + state.zkMigrationState()); - } - } - }*/ - /** * Tests if the controller should be preventing metadata updates due to being in the PRE_MIGRATION - * state. If the controller does not yet support migrations (before 3.4-IV0), then metadata updates - * are allowed in any state. Once the controller has been upgraded to a version that supports - * migrations, then this method checks if the controller is in the PRE_MIGRATION state. + * state. If the controller does not yet support migrations (before 3.4-IV0), then the migration state + * will be NONE and this will return false. Once the controller has been upgraded to a version that supports + * migrations, then this method checks if the migration state is equal to PRE_MIGRATION. */ - boolean inPreMigrationMode(MetadataVersion metadataVersion) { - ZkMigrationState state = migrationControlState.get(); - if (metadataVersion.isMigrationSupported()) { - return state == ZkMigrationState.PRE_MIGRATION; - } else { - return false; - } - } - - static ApiMessageAndVersion buildZkMigrationRecord(ZkMigrationState state) { - return new ApiMessageAndVersion( - new ZkMigrationStateRecord().setZkMigrationState(state.value()), - ZkMigrationStateRecord.HIGHEST_SUPPORTED_VERSION - ); + boolean inPreMigrationMode() { + return migrationControlState.get().equals(ZkMigrationState.PRE_MIGRATION); } public void replay(FeatureLevelRecord record) { diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 16beff2586fb1..50e8f7b4db892 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -684,8 +684,7 @@ public void run() throws Exception { if (!isActiveController(controllerEpoch)) { throw newNotControllerException(); } - if (featureControl.inPreMigrationMode(featureControl.metadataVersion()) && - !flags.contains(RUNS_IN_PREMIGRATION)) { + if (featureControl.inPreMigrationMode() && !flags.contains(RUNS_IN_PREMIGRATION)) { log.info("Cannot run write operation {} in pre-migration mode. Returning NOT_CONTROLLER.", name); throw newNotControllerException(); } @@ -913,9 +912,7 @@ public CompletableFuture completeMigration() { // TODO use KIP-868 transaction ControllerWriteEvent event = new ControllerWriteEvent<>("Complete ZK Migration", new MigrationWriteOperation( - Collections.singletonList( - FeatureControlManager.buildZkMigrationRecord(ZkMigrationState.MIGRATION) - )), + Collections.singletonList(ZkMigrationState.MIGRATION.toRecord())), EnumSet.of(RUNS_IN_PREMIGRATION)); queue.append(event); return event.future.thenApply(__ -> highestMigrationRecordOffset); @@ -1177,7 +1174,12 @@ private void claim(int epoch) { } } - public static void generateBootstrapRecords( + /** + * Generate the set of activation records. Until KIP-868 transactions are supported, these records + * are committed to the log as an atomic batch. The records will include the bootstrap metadata records + * (including the bootstrap "metadata.version") and may include a ZK migration record. + */ + public static void generateActivationRecords( Logger log, boolean isLogEmpty, boolean zkMigrationEnabled, @@ -1190,33 +1192,41 @@ public static void generateBootstrapRecords( // This will include the new metadata.version, as well as things like SCRAM // initialization, etc. log.info("The metadata log appears to be empty. Appending {} bootstrap record(s) " + - "at metadata.version {} from {}.", bootstrapMetadata.records().size(), - bootstrapMetadata.metadataVersion(), bootstrapMetadata.source()); + "at metadata.version {} from {}.", bootstrapMetadata.records().size(), + bootstrapMetadata.metadataVersion(), bootstrapMetadata.source()); bootstrapMetadata.records().forEach(recordConsumer); if (bootstrapMetadata.metadataVersion().isMigrationSupported()) { if (zkMigrationEnabled) { - log.info("Writing ZkMigrationState of PRE_MIGRATION to the log, since migrations are enabled."); - recordConsumer.accept(FeatureControlManager.buildZkMigrationRecord(ZkMigrationState.PRE_MIGRATION)); + log.info("Appending ZkMigrationState of PRE_MIGRATION to the log, since migrations are enabled."); + recordConsumer.accept(ZkMigrationState.PRE_MIGRATION.toRecord()); } else { - log.debug("Writing ZkMigrationState of NONE to the log, since migrations are not enabled."); - recordConsumer.accept(FeatureControlManager.buildZkMigrationRecord(ZkMigrationState.NONE)); + log.debug("Appending ZkMigrationState of NONE to the log, since migrations are not enabled."); + recordConsumer.accept(ZkMigrationState.NONE.toRecord()); + } + } else { + if (zkMigrationEnabled) { + throw new ConfigException("zookeeper.metadata.migration.enable", "true", + "The bootstrap metadata.version " + bootstrapMetadata.metadataVersion() + " does not support " + + "ZK migrations, cannot continue with ZK migrations enabled."); } } } else { // Logs have been replayed. We need to initialize some things here if upgrading from older KRaft versions if (featureControl.metadataVersion().equals(MetadataVersion.MINIMUM_KRAFT_VERSION)) { log.info("No metadata.version feature level record was found in the log. " + - "Treating the log as version {}.", MetadataVersion.MINIMUM_KRAFT_VERSION); + "Treating the log as version {}.", MetadataVersion.MINIMUM_KRAFT_VERSION); } if (featureControl.metadataVersion().isMigrationSupported()) { + log.info("Loaded ZK migration state of {}", featureControl.zkMigrationState()); switch (featureControl.zkMigrationState()) { case NONE: - // There may, or may not, be a NONE in the log. Since this is the default state, it will - // be persisted in a snapshot, so we don't need to explicitly write it here. + // Since this is the default state there may or may not be an actual NONE in the log. Regardless, + // it will eventually be persisted in a snapshot, so we don't need to explicitly write it here. if (zkMigrationEnabled) { - throw new ConfigException("Should not have ZK migrations enabled on a cluster that was created in KRaft mode."); + throw new ConfigException("zookeeper.metadata.migration.enable", "true", + "Should not have ZK migrations enabled on a cluster that was created in KRaft mode."); } break; case PRE_MIGRATION: @@ -1226,6 +1236,11 @@ public static void generateBootstrapRecords( // These states need more context, so they will be handled by KRaftMigrationDriver break; } + } else { + if (zkMigrationEnabled) { + throw new ConfigException("zookeeper.metadata.migration.enable", "true", + "Should not have ZK migrations enabled on a cluster running metadata.version " + featureControl.metadataVersion()); + } } } } @@ -1233,7 +1248,7 @@ class CompleteActivationEvent implements ControllerWriteOperation { @Override public ControllerResult generateRecordsAndResult() throws Exception { List records = new ArrayList<>(); - generateBootstrapRecords(log, logReplayTracker.empty(), zkMigrationEnabled, bootstrapMetadata, featureControl, records::add); + generateActivationRecords(log, logReplayTracker.empty(), zkMigrationEnabled, bootstrapMetadata, featureControl, records::add); return ControllerResult.atomicOf(records, null); } diff --git a/metadata/src/main/java/org/apache/kafka/controller/metrics/ControllerMetadataMetrics.java b/metadata/src/main/java/org/apache/kafka/controller/metrics/ControllerMetadataMetrics.java index a44208b6bc38f..84f2bc1f9bbf7 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/metrics/ControllerMetadataMetrics.java +++ b/metadata/src/main/java/org/apache/kafka/controller/metrics/ControllerMetadataMetrics.java @@ -20,7 +20,6 @@ import com.yammer.metrics.core.Gauge; import com.yammer.metrics.core.MetricName; import com.yammer.metrics.core.MetricsRegistry; -import org.apache.kafka.metadata.migration.ZkMigrationState; import org.apache.kafka.server.metrics.KafkaYammerMetrics; import java.util.Arrays; @@ -200,8 +199,8 @@ public int metadataErrorCount() { return this.metadataErrorCount.get(); } - public void setZkMigrationState(ZkMigrationState migrationState) { - this.zkMigrationState.set(migrationState.value()); + public void setZkMigrationState(byte migrationStateValue) { + this.zkMigrationState.set(migrationStateValue); } public byte zkMigrationState() { diff --git a/metadata/src/main/java/org/apache/kafka/controller/metrics/ControllerMetadataMetricsPublisher.java b/metadata/src/main/java/org/apache/kafka/controller/metrics/ControllerMetadataMetricsPublisher.java index 5d34fa22533e8..c72bdd9818f09 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/metrics/ControllerMetadataMetricsPublisher.java +++ b/metadata/src/main/java/org/apache/kafka/controller/metrics/ControllerMetadataMetricsPublisher.java @@ -116,7 +116,7 @@ private void publishDelta(MetadataDelta delta) { } changes.apply(metrics); if (delta.featuresDelta() != null) { - delta.featuresDelta().getZkMigrationStateChange().ifPresent(metrics::setZkMigrationState); + delta.featuresDelta().getZkMigrationStateChange().ifPresent(state -> metrics.setZkMigrationState(state.value())); } } @@ -150,7 +150,7 @@ private void publishSnapshot(MetadataImage newImage) { metrics.setGlobalPartitionCount(totalPartitions); metrics.setOfflinePartitionCount(offlinePartitions); metrics.setPreferredReplicaImbalanceCount(partitionsWithoutPreferredLeader); - metrics.setZkMigrationState(newImage.features().zkMigrationState()); + metrics.setZkMigrationState(newImage.features().zkMigrationState().value()); } @Override diff --git a/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java b/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java index 2f67186e6cac2..c92798dec013a 100644 --- a/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java +++ b/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java @@ -18,7 +18,6 @@ package org.apache.kafka.image; import org.apache.kafka.common.metadata.FeatureLevelRecord; -import org.apache.kafka.common.metadata.ZkMigrationStateRecord; import org.apache.kafka.image.writer.ImageWriter; import org.apache.kafka.image.writer.ImageWriterOptions; import org.apache.kafka.metadata.migration.ZkMigrationState; @@ -90,7 +89,7 @@ public void write(ImageWriter writer, ImageWriterOptions options) { } if (options.metadataVersion().isMigrationSupported()) { - writer.write(0, new ZkMigrationStateRecord().setZkMigrationState(zkMigrationState.value())); + writer.write(0, zkMigrationState.toRecord().message()); } else { if (!zkMigrationState.equals(ZkMigrationState.NONE)) { options.handleLoss("the ZK Migration state which was " + zkMigrationState); diff --git a/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkMigrationState.java b/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkMigrationState.java index 0717b9d2f0740..fe957622134c8 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkMigrationState.java +++ b/metadata/src/main/java/org/apache/kafka/metadata/migration/ZkMigrationState.java @@ -16,6 +16,9 @@ */ package org.apache.kafka.metadata.migration; +import org.apache.kafka.common.metadata.ZkMigrationStateRecord; +import org.apache.kafka.server.common.ApiMessageAndVersion; + import java.util.Optional; /** @@ -71,6 +74,13 @@ public byte value() { return value; } + public ApiMessageAndVersion toRecord() { + return new ApiMessageAndVersion( + new ZkMigrationStateRecord().setZkMigrationState(value()), + (short) 0 + ); + } + public static ZkMigrationState of(byte value) { return optionalOf(value) .orElseThrow(() -> new IllegalArgumentException(String.format("Value %s is not a valid Zk migration state", value))); diff --git a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java index d0d64ea0d2200..bd9483ff5d889 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java @@ -35,6 +35,8 @@ import org.apache.kafka.metadata.FinalizedControllerFeatures; import org.apache.kafka.metadata.RecordTestUtils; import org.apache.kafka.metadata.VersionRange; +import org.apache.kafka.metadata.bootstrap.BootstrapMetadata; +import org.apache.kafka.metadata.migration.ZkMigrationState; import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; import org.apache.kafka.timeline.SnapshotRegistry; @@ -414,4 +416,45 @@ public void testCreateFeatureLevelRecords() { RecordTestUtils.replayAll(manager, result2.records()); assertEquals(Optional.empty(), manager.finalizedFeatures(Long.MAX_VALUE).get("foo")); } + + @Test + public void testNoMetadataVersionChangeDuringMigration() { + FeatureControlManager manager = new FeatureControlManager.Builder(). + setQuorumFeatures(features(MetadataVersion.FEATURE_NAME, + MetadataVersion.IBP_3_0_IV0.featureLevel(), MetadataVersion.IBP_3_5_IV1.featureLevel())). + setMetadataVersion(MetadataVersion.IBP_3_4_IV0). + build(); + BootstrapMetadata bootstrapMetadata = BootstrapMetadata.fromVersion(MetadataVersion.IBP_3_4_IV0, "FeatureControlManagerTest"); + RecordTestUtils.replayAll(manager, bootstrapMetadata.records()); + RecordTestUtils.replayOne(manager, ZkMigrationState.PRE_MIGRATION.toRecord()); + + assertEquals(ControllerResult.of(Collections.emptyList(), + singletonMap(MetadataVersion.FEATURE_NAME, new ApiError(Errors.INVALID_UPDATE_VERSION, + "Invalid metadata.version 10. Unable to modify metadata.version while a ZK migration is in progress."))), + manager.updateFeatures( + singletonMap(MetadataVersion.FEATURE_NAME, MetadataVersion.IBP_3_5_IV1.featureLevel()), + singletonMap(MetadataVersion.FEATURE_NAME, FeatureUpdate.UpgradeType.UPGRADE), + emptyMap(), + true)); + + assertEquals(ControllerResult.of(Collections.emptyList(), + singletonMap(MetadataVersion.FEATURE_NAME, new ApiError(Errors.INVALID_UPDATE_VERSION, + "Invalid metadata.version 4. Unable to modify metadata.version while a ZK migration is in progress."))), + manager.updateFeatures( + singletonMap(MetadataVersion.FEATURE_NAME, MetadataVersion.IBP_3_3_IV0.featureLevel()), + singletonMap(MetadataVersion.FEATURE_NAME, FeatureUpdate.UpgradeType.SAFE_DOWNGRADE), + emptyMap(), + true)); + + // Complete the migration + RecordTestUtils.replayOne(manager, ZkMigrationState.POST_MIGRATION.toRecord()); + ControllerResult> result = manager.updateFeatures( + singletonMap(MetadataVersion.FEATURE_NAME, MetadataVersion.IBP_3_5_IV1.featureLevel()), + singletonMap(MetadataVersion.FEATURE_NAME, FeatureUpdate.UpgradeType.UPGRADE), + emptyMap(), + false); + assertEquals(Errors.NONE, result.response().get(MetadataVersion.FEATURE_NAME).error()); + RecordTestUtils.replayAll(manager, result.records()); + assertEquals(MetadataVersion.IBP_3_5_IV1, manager.metadataVersion()); + } } diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java index ccd2f711582c0..1426925f8932c 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -43,6 +43,7 @@ import java.util.stream.IntStream; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.BrokerIdNotRegisteredException; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.message.RequestHeaderData; @@ -51,6 +52,7 @@ import org.apache.kafka.common.metadata.UnfenceBrokerRecord; import org.apache.kafka.common.requests.AlterPartitionRequest; import org.apache.kafka.common.security.auth.KafkaPrincipal; +import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.common.config.ConfigResource; @@ -92,6 +94,7 @@ import org.apache.kafka.metadata.BrokerRegistrationReply; import org.apache.kafka.metadata.FinalizedControllerFeatures; import org.apache.kafka.metadata.PartitionRegistration; +import org.apache.kafka.metadata.RecordTestUtils; import org.apache.kafka.metadata.authorizer.StandardAuthorizer; import org.apache.kafka.metadata.bootstrap.BootstrapMetadata; import org.apache.kafka.metadata.migration.ZkMigrationState; @@ -105,6 +108,7 @@ import org.apache.kafka.snapshot.RawSnapshotReader; import org.apache.kafka.snapshot.Snapshots; import org.apache.kafka.test.TestUtils; +import org.apache.kafka.timeline.SnapshotRegistry; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -1373,4 +1377,98 @@ public ZkMigrationState checkBootstrapZkMigrationRecord( () -> active.featureControl().zkMigrationState()).get(30, TimeUnit.SECONDS); } } + + @Test + public void testUpgradeMigrationStateFrom34() throws Exception { + try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv.Builder(1).build()) { + // In 3.4, we only wrote a PRE_MIGRATION to the log. In that software version, we defined this + // as enum value 1. In 3.5+ software, this enum value is redefined as MIGRATION + BootstrapMetadata bootstrapMetadata = BootstrapMetadata.fromVersion(MetadataVersion.IBP_3_4_IV0, "test"); + List initialRecords = new ArrayList<>(bootstrapMetadata.records()); + initialRecords.add(ZkMigrationState.of((byte) 1).toRecord()); + logEnv.appendInitialRecords(initialRecords); + try ( + QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv.Builder(logEnv). + setControllerBuilderInitializer(controllerBuilder -> { + controllerBuilder.setZkMigrationEnabled(true); + }). + setBootstrapMetadata(bootstrapMetadata). + build(); + ) { + QuorumController active = controlEnv.activeController(); + assertEquals(active.featureControl().zkMigrationState(), ZkMigrationState.MIGRATION); + assertFalse(active.featureControl().inPreMigrationMode()); + } + } + } + + FeatureControlManager getActivationRecords( + MetadataVersion metadataVersion, + boolean emptyLog, + boolean zkMigrationEnabled + ) { + SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); + FeatureControlManager featureControlManager = new FeatureControlManager.Builder() + .setSnapshotRegistry(snapshotRegistry) + .setMetadataVersion(metadataVersion) + .build(); + + List records = new ArrayList<>(); + QuorumController.generateActivationRecords( + log, + emptyLog, + zkMigrationEnabled, + BootstrapMetadata.fromVersion(metadataVersion, "test"), + featureControlManager, + records::add); + RecordTestUtils.replayAll(featureControlManager, records); + return featureControlManager; + } + + @Test + public void testActivationRecords33() { + FeatureControlManager featureControl; + + assertEquals( + "Invalid value true for configuration zookeeper.metadata.migration.enable: " + + "The bootstrap metadata.version 3.3-IV0 does not support ZK migrations, cannot continue with ZK migrations enabled.", + assertThrows(ConfigException.class, () -> getActivationRecords(MetadataVersion.IBP_3_3_IV0, true, true)).getMessage() + ); + + featureControl = getActivationRecords(MetadataVersion.IBP_3_3_IV0, true, false); + assertEquals(MetadataVersion.IBP_3_3_IV0, featureControl.metadataVersion()); + assertEquals(ZkMigrationState.NONE, featureControl.zkMigrationState()); + + assertEquals( + "Invalid value true for configuration zookeeper.metadata.migration.enable: " + + "Should not have ZK migrations enabled on a cluster running metadata.version 3.3-IV0", + assertThrows(ConfigException.class, () -> getActivationRecords(MetadataVersion.IBP_3_3_IV0, false, true)).getMessage() + ); + + featureControl = getActivationRecords(MetadataVersion.IBP_3_3_IV0, false, false); + assertEquals(MetadataVersion.IBP_3_3_IV0, featureControl.metadataVersion()); + assertEquals(ZkMigrationState.NONE, featureControl.zkMigrationState()); + } + + @Test + public void testActivationRecords34() { + FeatureControlManager featureControl; + + featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, true, true); + assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); + assertEquals(ZkMigrationState.PRE_MIGRATION, featureControl.zkMigrationState()); + + featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, true, false); + assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); + assertEquals(ZkMigrationState.NONE, featureControl.zkMigrationState()); + + assertEquals( + "Invalid value true for configuration zookeeper.metadata.migration.enable: Should not have ZK migrations enabled on a cluster that was created in KRaft mode.", + assertThrows(ConfigException.class, () -> getActivationRecords(MetadataVersion.IBP_3_4_IV0, false, true)).getMessage() + ); + + featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, false, false); + assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); + assertEquals(ZkMigrationState.NONE, featureControl.zkMigrationState()); + } } diff --git a/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationTest.java b/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationTest.java deleted file mode 100644 index bb74e7d06c2d8..0000000000000 --- a/metadata/src/test/java/org/apache/kafka/controller/ZkMigrationTest.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kafka.controller; - -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.metadata.ZkMigrationStateRecord; -import org.apache.kafka.common.utils.LogContext; -import org.apache.kafka.metadata.RecordTestUtils; -import org.apache.kafka.metadata.bootstrap.BootstrapMetadata; -import org.apache.kafka.metadata.migration.ZkMigrationState; -import org.apache.kafka.server.common.ApiMessageAndVersion; -import org.apache.kafka.server.common.MetadataVersion; -import org.apache.kafka.timeline.SnapshotRegistry; -import org.junit.jupiter.api.Test; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.fail; - -public class ZkMigrationTest { - - private FeatureControlManager setupAndBootstrap( - MetadataVersion metadataVersion, - boolean emptyLog, - boolean zkMigrationEnabled - ) { - SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); - FeatureControlManager featureControlManager = new FeatureControlManager.Builder() - .setSnapshotRegistry(snapshotRegistry) - .setMetadataVersion(metadataVersion) - .build(); - - List records = new ArrayList<>(); - QuorumController.generateBootstrapRecords( - LoggerFactory.getLogger(ZkMigrationTest.class), - emptyLog, - zkMigrationEnabled, - BootstrapMetadata.fromVersion(metadataVersion, "test"), - featureControlManager, - records::add); - RecordTestUtils.replayAll(featureControlManager, records); - return featureControlManager; - } - - private void verifyCannotBootstrapAgain(MetadataVersion metadataVersion, FeatureControlManager featureControl, Class expectedException) { - // Should not be able to produce another record in any circumstance after we've bootstrapped once - assertThrows(expectedException, () -> - QuorumController.generateBootstrapRecords( - LoggerFactory.getLogger(ZkMigrationTest.class), - false, - true, - BootstrapMetadata.fromVersion(metadataVersion, "test"), - featureControl, - record -> fail("Did not expect to get another record here, but got " + record + ". State was " + featureControl.zkMigrationState()))); - } - - @Test - public void testBootstrapEmptyLog() { - FeatureControlManager migrationControl; - - // In 3.3, we won't write any records - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, true, true); - assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, true, false); - assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - - // In 3.4, we'll write something and should only be able to bootstrap once - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, true); - assertEquals(ZkMigrationState.PRE_MIGRATION, migrationControl.zkMigrationState()); - verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl, IllegalStateException.class); - - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, false); - assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl, ConfigException.class); - } - - @Test - public void testBootstrapNonEmptyLog() { - FeatureControlManager migrationControl; - - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, false, true); - assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_3_IV0, false, false); - assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - - // Cannot have migrations enabled for a non-migrated KRaft cluster unless we're in a MIGRATION state - assertThrows(ConfigException.class, () -> setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, true)); - - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, false); - assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - } - - @Test - public void testNonMigrationStateTransitions() { - // When migrations should not be present, ensure we can't change state from NONE - FeatureControlManager migrationControl; - assertThrows(ConfigException.class, () -> setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, true)); - - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, false); - assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - verifyCannotBootstrapAgain(MetadataVersion.IBP_3_4_IV0, migrationControl, ConfigException.class); - } - - @Test - public void testFailoverToNonMigrationControllerDuringMigration() { - - } - - @Test - public void testFailoverToNonMigrationControllerAfterMigration() { - - } - - @Test - public void testMigrationDisabledStateTransitions() { - // When migrations are disabled, ensure we can't change state from NONE - FeatureControlManager migrationControl; - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, true, false); - assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - - migrationControl = setupAndBootstrap(MetadataVersion.IBP_3_4_IV0, false, false); - assertEquals(ZkMigrationState.NONE, migrationControl.zkMigrationState()); - } - - private FeatureControlManager verifyUpgradeFrom34(boolean migrationEnabled) { - SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); - FeatureControlManager featureControl = new FeatureControlManager.Builder() - .setSnapshotRegistry(snapshotRegistry) - .build(); - - // In 3.4, we only ever wrote PRE_MIGRATION (1) - featureControl.replay(new ZkMigrationStateRecord().setZkMigrationState((byte) 1)); - assertEquals(featureControl.zkMigrationState(), ZkMigrationState.MIGRATION); - assertFalse(featureControl.inPreMigrationMode(MetadataVersion.IBP_3_4_IV0)); - - // Now bootstrap as if we're starting up in 3.5 - List records = new ArrayList<>(); - QuorumController.generateBootstrapRecords( - LoggerFactory.getLogger(ZkMigrationTest.class), - false, - true, - BootstrapMetadata.fromVersion(MetadataVersion.IBP_3_4_IV0, "test"), - featureControl, - records::add); - - records.forEach(record -> { - featureControl.replay((ZkMigrationStateRecord) record.message()); - }); - return featureControl; - } - - @Test - public void testUpgradeFrom34MigrationEnabled() { - FeatureControlManager migrationControl = verifyUpgradeFrom34(true); - assertEquals(migrationControl.zkMigrationState(), ZkMigrationState.MIGRATION); - assertFalse(migrationControl.inPreMigrationMode(MetadataVersion.IBP_3_4_IV0)); - } -} diff --git a/metadata/src/test/java/org/apache/kafka/metadata/RecordTestUtils.java b/metadata/src/test/java/org/apache/kafka/metadata/RecordTestUtils.java index 04da77b0fc327..20569dc117321 100644 --- a/metadata/src/test/java/org/apache/kafka/metadata/RecordTestUtils.java +++ b/metadata/src/test/java/org/apache/kafka/metadata/RecordTestUtils.java @@ -33,6 +33,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; @@ -87,6 +88,13 @@ public static void replayAll(Object target, } } + public static void replayOne( + Object target, + ApiMessageAndVersion recordAndVersion + ) { + replayAll(target, Collections.singletonList(recordAndVersion)); + } + /** * Replay a list of record batches. * From 7b93271d809f7f3d64cae6db7aedc819869a3b46 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Mon, 17 Apr 2023 10:06:33 -0400 Subject: [PATCH 18/27] Add transition to POST_MIGRATION --- .../kafka/controller/QuorumController.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 50e8f7b4db892..6b62b868b839e 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -1198,10 +1198,11 @@ public static void generateActivationRecords( if (bootstrapMetadata.metadataVersion().isMigrationSupported()) { if (zkMigrationEnabled) { - log.info("Appending ZkMigrationState of PRE_MIGRATION to the log, since migrations are enabled."); + log.info("Putting the controller into pre-migration mode. No metadata updates will be allowed until " + + "the ZK metadata has been migrated"); recordConsumer.accept(ZkMigrationState.PRE_MIGRATION.toRecord()); } else { - log.debug("Appending ZkMigrationState of NONE to the log, since migrations are not enabled."); + log.debug("Setting the ZK migration state to NONE since this is a de-novo KRaft cluster."); recordConsumer.accept(ZkMigrationState.NONE.toRecord()); } } else { @@ -1232,8 +1233,18 @@ public static void generateActivationRecords( case PRE_MIGRATION: throw new IllegalStateException("Detected an failed migration state during bootstrap, cannot continue."); case MIGRATION: + if (!zkMigrationEnabled) { + log.info("Completing the ZK migration since this controller was configured with " + + "'zookeeper.metadata.migration.enable' set to 'false'."); + recordConsumer.accept(ZkMigrationState.POST_MIGRATION.toRecord()); + } else { + log.info("Staying in the ZK migration since 'zookeeper.metadata.migration.enable' is still 'true'."); + } case POST_MIGRATION: - // These states need more context, so they will be handled by KRaftMigrationDriver + if (zkMigrationEnabled) { + log.info("Ignoring 'zookeeper.metadata.migration.enable' value of 'true' since the ZK migration" + + "has been completed."); + } break; } } else { From 5ae5363417e3f15d6549b05d294b3e8baf3f36f8 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Mon, 17 Apr 2023 16:24:59 -0400 Subject: [PATCH 19/27] Update system tests for recent changes --- .../kafka/controller/QuorumController.java | 29 ++++++++++-------- .../tests/core/zookeeper_migration_test.py | 30 ++++++++----------- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 6b62b868b839e..55ade709a3544 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclBindingFilter; -import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.errors.ApiException; import org.apache.kafka.common.errors.BrokerIdNotRegisteredException; @@ -1166,9 +1165,15 @@ private void claim(int epoch) { // Prepend the activate event. It is important that this event go at the beginning // of the queue rather than the end (hence prepend rather than append). It's also // important not to use prepend for anything else, to preserve the ordering here. - queue.prepend(new ControllerWriteEvent<>("completeActivation[" + epoch + "]", - new CompleteActivationEvent(), - EnumSet.of(DOES_NOT_UPDATE_QUEUE_TIME, RUNS_IN_PREMIGRATION))); + ControllerWriteEvent activationEvent = new ControllerWriteEvent<>("completeActivation[" + epoch + "]", + new CompleteActivationEvent(), + EnumSet.of(DOES_NOT_UPDATE_QUEUE_TIME, RUNS_IN_PREMIGRATION)); + activationEvent.future.whenComplete((__, t) -> { + if (t != null) { + fatalFaultHandler.handleFault("exception while activating controller", t); + } + }); + queue.prepend(activationEvent); } catch (Throwable e) { fatalFaultHandler.handleFault("exception while claiming leadership", e); } @@ -1207,9 +1212,8 @@ public static void generateActivationRecords( } } else { if (zkMigrationEnabled) { - throw new ConfigException("zookeeper.metadata.migration.enable", "true", - "The bootstrap metadata.version " + bootstrapMetadata.metadataVersion() + " does not support " + - "ZK migrations, cannot continue with ZK migrations enabled."); + throw new RuntimeException("The bootstrap metadata.version " + bootstrapMetadata.metadataVersion() + + " does not support ZK migrations. Cannot continue with ZK migrations enabled."); } } } else { @@ -1226,12 +1230,11 @@ public static void generateActivationRecords( // Since this is the default state there may or may not be an actual NONE in the log. Regardless, // it will eventually be persisted in a snapshot, so we don't need to explicitly write it here. if (zkMigrationEnabled) { - throw new ConfigException("zookeeper.metadata.migration.enable", "true", - "Should not have ZK migrations enabled on a cluster that was created in KRaft mode."); + throw new RuntimeException("Should not have ZK migrations enabled on a cluster that was created in KRaft mode."); } break; case PRE_MIGRATION: - throw new IllegalStateException("Detected an failed migration state during bootstrap, cannot continue."); + throw new RuntimeException("Detected an failed migration state during bootstrap, cannot continue."); case MIGRATION: if (!zkMigrationEnabled) { log.info("Completing the ZK migration since this controller was configured with " + @@ -1240,6 +1243,7 @@ public static void generateActivationRecords( } else { log.info("Staying in the ZK migration since 'zookeeper.metadata.migration.enable' is still 'true'."); } + break; case POST_MIGRATION: if (zkMigrationEnabled) { log.info("Ignoring 'zookeeper.metadata.migration.enable' value of 'true' since the ZK migration" + @@ -1249,15 +1253,14 @@ public static void generateActivationRecords( } } else { if (zkMigrationEnabled) { - throw new ConfigException("zookeeper.metadata.migration.enable", "true", - "Should not have ZK migrations enabled on a cluster running metadata.version " + featureControl.metadataVersion()); + throw new RuntimeException("Should not have ZK migrations enabled on a cluster running metadata.version " + featureControl.metadataVersion()); } } } } class CompleteActivationEvent implements ControllerWriteOperation { @Override - public ControllerResult generateRecordsAndResult() throws Exception { + public ControllerResult generateRecordsAndResult() { List records = new ArrayList<>(); generateActivationRecords(log, logReplayTracker.empty(), zkMigrationEnabled, bootstrapMetadata, featureControl, records::add); return ControllerResult.atomicOf(records, null); diff --git a/tests/kafkatest/tests/core/zookeeper_migration_test.py b/tests/kafkatest/tests/core/zookeeper_migration_test.py index 0a0a830a3ea80..245300958713c 100644 --- a/tests/kafkatest/tests/core/zookeeper_migration_test.py +++ b/tests/kafkatest/tests/core/zookeeper_migration_test.py @@ -158,26 +158,19 @@ def test_pre_migration_mode_3_4(self, metadata_quorum): node.version = DEV_BRANCH self.logger.info("Restarting controller node %s" % node.account.hostname) self.kafka.controller_quorum.start_node(node) - self.wait_until_rejoin() - self.logger.info("Successfully restarted controller node %s" % node.account.hostname) - for node in self.kafka.nodes: - self.logger.info("Stopping broker node %s" % node.account.hostname) - self.kafka.stop_node(node) - node.version = DEV_BRANCH - self.logger.info("Restarting broker node %s" % node.account.hostname) - self.kafka.start_node(node) - self.wait_until_rejoin() - self.logger.info("Successfully restarted broker node %s" % node.account.hostname) + # Controller should crash # Check the controller's logs for the error message about the migration state saw_expected_error = False for node in self.kafka.controller_quorum.nodes: + wait_until(lambda: not self.kafka.controller_quorum.alive(node), timeout_sec=60, + backoff_sec=1, err_msg="Controller did not halt in the expected amount of time") with node.account.monitor_log(KafkaService.STDOUT_STDERR_CAPTURE) as monitor: monitor.offset = 0 try: # Shouldn't have to wait too long to see this log message after startup monitor.wait_until( - "ZkMigrationState is NONE which means this cluster should not be migrated from ZooKeeper", + "Should not have ZK migrations enabled on a cluster that was created in KRaft mode.", timeout_sec=10.0, backoff_sec=.25, err_msg="" ) @@ -186,7 +179,7 @@ def test_pre_migration_mode_3_4(self, metadata_quorum): except TimeoutError: continue - assert saw_expected_error, "Did not see expected ERROR log for ZkMigrationState = NONE in the controller logs" + assert saw_expected_error, "Did not see expected ERROR log in the controller logs" def test_upgrade_after_3_4_migration(self): """ @@ -232,7 +225,8 @@ def test_upgrade_after_3_4_migration(self): } self.kafka.create_topic(topic_cfg) - # Now we're in dual-write mode. Upgrade the controller to 3.5+ + # Now we're in dual-write mode. The 3.4 controller will have written a PRE_MIGRATION record (1) into the log. + # We now upgrade the controller to 3.5+ where 1 is redefined as MIGRATION. for node in controller.nodes: self.logger.info("Stopping controller node %s" % node.account.hostname) self.kafka.controller_quorum.stop_node(node) @@ -242,21 +236,21 @@ def test_upgrade_after_3_4_migration(self): self.wait_until_rejoin() self.logger.info("Successfully restarted controller node %s" % node.account.hostname) - # Check the controller's logs for the error message about the migration state - saw_expected_error = False + # Check the controller's logs for the INFO message that we're still in the migration state + saw_expected_log = False for node in self.kafka.controller_quorum.nodes: with node.account.monitor_log(KafkaService.STDOUT_STDERR_CAPTURE) as monitor: monitor.offset = 0 try: # Shouldn't have to wait too long to see this log message after startup monitor.wait_until( - "read a ZkMigrationState of PRE_MIGRATION from a ZK migration on Apache Kafka 3.4.", + "Staying in the ZK migration", timeout_sec=10.0, backoff_sec=.25, err_msg="" ) - saw_expected_error = True + saw_expected_log = True break except TimeoutError: continue - assert saw_expected_error, "Did not see expected INFO log for detecting upgrade after 3.4 migration" + assert saw_expected_log, "Did not see expected INFO log after upgrading from a 3.4 migration" From c183eef288b0087a4e7507551f10b22e42f693b4 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Tue, 18 Apr 2023 17:13:23 -0400 Subject: [PATCH 20/27] Fix some logic in the migration driver --- checkstyle/suppressions.xml | 2 +- .../migration/KRaftMigrationDriver.java | 27 ++++++-- .../migration/KRaftMigrationDriverTest.java | 66 ++++++++++++++++--- 3 files changed, 77 insertions(+), 18 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index aa5f18fc5ea05..680bab90af6c2 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -306,7 +306,7 @@ + files="(ClientQuotasImage|KafkaEventQueue|MetadataDelta|QuorumController|ReplicationControlManager|KRaftMigrationDriver).java"/> brokerIds; public final Map> capturedConfigs = new HashMap<>(); + private ZkMigrationLeadershipState state = null; public CapturingMigrationClient(Set brokerIdsInZk) { this.brokerIds = brokerIdsInZk; @@ -89,21 +90,27 @@ public CapturingMigrationClient(Set brokerIdsInZk) { @Override public ZkMigrationLeadershipState getOrCreateMigrationRecoveryState(ZkMigrationLeadershipState initialState) { - return initialState; + if (state == null) { + state = initialState; + } + return state; } @Override public ZkMigrationLeadershipState setMigrationRecoveryState(ZkMigrationLeadershipState state) { + this.state = state; return state; } @Override public ZkMigrationLeadershipState claimControllerLeadership(ZkMigrationLeadershipState state) { + this.state = state; return state; } @Override public ZkMigrationLeadershipState releaseControllerLeadership(ZkMigrationLeadershipState state) { + this.state = state; return state; } @@ -114,6 +121,7 @@ public ZkMigrationLeadershipState createTopic( Map topicPartitions, ZkMigrationLeadershipState state ) { + this.state = state; return state; } @@ -122,6 +130,7 @@ public ZkMigrationLeadershipState updateTopicPartitions( Map> topicPartitions, ZkMigrationLeadershipState state ) { + this.state = state; return state; } @@ -132,6 +141,7 @@ public ZkMigrationLeadershipState writeConfigs( ZkMigrationLeadershipState state ) { capturedConfigs.computeIfAbsent(configResource, __ -> new HashMap<>()).putAll(configMap); + this.state = state; return state; } @@ -141,6 +151,7 @@ public ZkMigrationLeadershipState writeClientQuotas( Map quotas, ZkMigrationLeadershipState state ) { + this.state = state; return state; } @@ -149,6 +160,7 @@ public ZkMigrationLeadershipState writeProducerId( long nextProducerId, ZkMigrationLeadershipState state ) { + this.state = state; return state; } @@ -158,6 +170,7 @@ public ZkMigrationLeadershipState removeDeletedAcls( List deletedAcls, ZkMigrationLeadershipState state ) { + this.state = state; return state; } @@ -167,6 +180,7 @@ public ZkMigrationLeadershipState writeAddedAcls( List addedAcls, ZkMigrationLeadershipState state ) { + this.state = state; return state; } @@ -247,12 +261,6 @@ RegisterBrokerRecord zkBrokerRecord(int id) { return record; } - ZkMigrationStateRecord preMigrationRecord() { - ZkMigrationStateRecord record = new ZkMigrationStateRecord(); - record.setZkMigrationState(ZkMigrationState.PRE_MIGRATION.value()); - return record; - } - /** * Enqueues a metadata change event with the migration driver and returns a future that can be waited on in * the test code. The future will complete once the metadata change event executes completely. @@ -297,7 +305,7 @@ public void testOnlySendNeededRPCsToBrokers() throws Exception { MetadataDelta delta = new MetadataDelta(image); driver.start(); - delta.replay(preMigrationRecord()); + delta.replay(ZkMigrationState.PRE_MIGRATION.toRecord().message()); delta.replay(zkBrokerRecord(1)); delta.replay(zkBrokerRecord(2)); delta.replay(zkBrokerRecord(3)); @@ -379,7 +387,7 @@ public ZkMigrationLeadershipState claimControllerLeadership(ZkMigrationLeadershi MetadataDelta delta = new MetadataDelta(image); driver.start(); - delta.replay(preMigrationRecord()); + delta.replay(ZkMigrationState.PRE_MIGRATION.toRecord().message()); delta.replay(zkBrokerRecord(1)); delta.replay(zkBrokerRecord(2)); delta.replay(zkBrokerRecord(3)); @@ -402,4 +410,42 @@ public ZkMigrationLeadershipState claimControllerLeadership(ZkMigrationLeadershi } } } + + @Test + public void testSkipWaitForBrokersInDualWrite() throws Exception { + CountingMetadataPropagator metadataPropagator = new CountingMetadataPropagator(); + CapturingMigrationClient migrationClient = new CapturingMigrationClient(Collections.emptySet()); + MockFaultHandler faultHandler = new MockFaultHandler("testMigrationClientExpiration"); + try (KRaftMigrationDriver driver = new KRaftMigrationDriver( + 3000, + new NoOpRecordConsumer(), + migrationClient, + metadataPropagator, + metadataPublisher -> { }, + faultHandler + )) { + MetadataImage image = MetadataImage.EMPTY; + MetadataDelta delta = new MetadataDelta(image); + + // Fake a complete migration with ZK client + migrationClient.setMigrationRecoveryState( + ZkMigrationLeadershipState.EMPTY.withKRaftMetadataOffsetAndEpoch(100, 1)); + + driver.start(); + delta.replay(ZkMigrationState.PRE_MIGRATION.toRecord().message()); + delta.replay(zkBrokerRecord(1)); + delta.replay(zkBrokerRecord(2)); + delta.replay(zkBrokerRecord(3)); + delta.replay(ZkMigrationState.MIGRATION.toRecord().message()); + MetadataProvenance provenance = new MetadataProvenance(100, 1, 1); + image = delta.apply(provenance); + + driver.onControllerChange(new LeaderAndEpoch(OptionalInt.of(3000), 1)); + driver.onMetadataUpdate(delta, image, new LogDeltaManifest(provenance, + new LeaderAndEpoch(OptionalInt.of(3000), 1), 1, 100, 42)); + + TestUtils.waitForCondition(() -> driver.migrationState().get(1, TimeUnit.MINUTES).equals(MigrationDriverState.DUAL_WRITE), + "Waiting for KRaftMigrationDriver to enter ZK_MIGRATION state"); + } + } } \ No newline at end of file From 5107fb4605f3822d2dbf0ee34fcd1346ababa2d3 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Wed, 19 Apr 2023 10:33:22 -0400 Subject: [PATCH 21/27] Fix some tests --- .../BrokerRegistrationRequestTest.scala | 12 ++++++--- .../controller/QuorumControllerTest.java | 25 ++++++++++--------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala b/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala index 32308458ee5ba..f19b6afb38051 100644 --- a/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala @@ -146,25 +146,29 @@ class BrokerRegistrationRequestTest { } @ClusterTest(clusterType = Type.KRAFT, brokers = 0, controllers = 1, metadataVersion = MetadataVersion.IBP_3_3_IV3, - serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))) - def testRegisterZkWithKRaftOldMetadataVersion(clusterInstance: ClusterInstance): Unit = { + serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "false"))) + def testRegisterZkWith33Controller(clusterInstance: ClusterInstance): Unit = { + // Verify that a controller running an old metadata.version cannot register a ZK broker val clusterId = clusterInstance.clusterId() val channelManager = brokerToControllerChannelManager(clusterInstance) try { channelManager.start() - + // Invalid registration (isMigratingZkBroker, but MV does not support migrations) assertEquals( Errors.BROKER_ID_NOT_REGISTERED, - registerBroker(channelManager, clusterId, 100, Some(1), Some((MetadataVersion.IBP_3_3_IV0, MetadataVersion.IBP_3_3_IV0)))) + registerBroker(channelManager, clusterId, 100, Some(1), Some((MetadataVersion.IBP_3_3_IV0, MetadataVersion.IBP_3_3_IV3)))) + // No features (MV) sent with registration, controller can't verify assertEquals( Errors.BROKER_ID_NOT_REGISTERED, registerBroker(channelManager, clusterId, 100, Some(1), None)) + // Given MV is too high for controller to support assertEquals( Errors.BROKER_ID_NOT_REGISTERED, registerBroker(channelManager, clusterId, 100, Some(1), Some((MetadataVersion.IBP_3_4_IV0, MetadataVersion.IBP_3_4_IV0)))) + // Controller supports this MV and isMigratingZkBroker is false, so this one works assertEquals( Errors.NONE, registerBroker(channelManager, clusterId, 100, None, Some((MetadataVersion.IBP_3_3_IV3, MetadataVersion.IBP_3_4_IV0)))) diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java index 1fb9f1a577194..20df0e5273db3 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -43,7 +43,6 @@ import java.util.stream.IntStream; import org.apache.kafka.common.Uuid; -import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.BrokerIdNotRegisteredException; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.message.RequestHeaderData; @@ -104,6 +103,7 @@ import org.apache.kafka.raft.OffsetAndEpoch; import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; +import org.apache.kafka.server.fault.FaultHandlerException; import org.apache.kafka.snapshot.FileRawSnapshotReader; import org.apache.kafka.snapshot.RawSnapshotReader; import org.apache.kafka.snapshot.Snapshots; @@ -1352,11 +1352,14 @@ public void testBootstrapZkMigrationRecord() throws Exception { assertEquals(ZkMigrationState.NONE, checkBootstrapZkMigrationRecord(MetadataVersion.IBP_3_4_IV0, false)); - assertEquals(ZkMigrationState.NONE, - checkBootstrapZkMigrationRecord(MetadataVersion.IBP_3_3_IV0, true)); - assertEquals(ZkMigrationState.NONE, checkBootstrapZkMigrationRecord(MetadataVersion.IBP_3_3_IV0, false)); + + assertEquals( + "The bootstrap metadata.version 3.3-IV0 does not support ZK migrations. Cannot continue with ZK migrations enabled.", + assertThrows(FaultHandlerException.class, () -> + checkBootstrapZkMigrationRecord(MetadataVersion.IBP_3_3_IV0, true)).getCause().getCause().getMessage() + ); } public ZkMigrationState checkBootstrapZkMigrationRecord( @@ -1430,9 +1433,8 @@ public void testActivationRecords33() { FeatureControlManager featureControl; assertEquals( - "Invalid value true for configuration zookeeper.metadata.migration.enable: " + - "The bootstrap metadata.version 3.3-IV0 does not support ZK migrations, cannot continue with ZK migrations enabled.", - assertThrows(ConfigException.class, () -> getActivationRecords(MetadataVersion.IBP_3_3_IV0, true, true)).getMessage() + "The bootstrap metadata.version 3.3-IV0 does not support ZK migrations. Cannot continue with ZK migrations enabled.", + assertThrows(RuntimeException.class, () -> getActivationRecords(MetadataVersion.IBP_3_3_IV0, true, true)).getMessage() ); featureControl = getActivationRecords(MetadataVersion.IBP_3_3_IV0, true, false); @@ -1440,9 +1442,8 @@ public void testActivationRecords33() { assertEquals(ZkMigrationState.NONE, featureControl.zkMigrationState()); assertEquals( - "Invalid value true for configuration zookeeper.metadata.migration.enable: " + - "Should not have ZK migrations enabled on a cluster running metadata.version 3.3-IV0", - assertThrows(ConfigException.class, () -> getActivationRecords(MetadataVersion.IBP_3_3_IV0, false, true)).getMessage() + "Should not have ZK migrations enabled on a cluster running metadata.version 3.3-IV0", + assertThrows(RuntimeException.class, () -> getActivationRecords(MetadataVersion.IBP_3_3_IV0, false, true)).getMessage() ); featureControl = getActivationRecords(MetadataVersion.IBP_3_3_IV0, false, false); @@ -1463,8 +1464,8 @@ public void testActivationRecords34() { assertEquals(ZkMigrationState.NONE, featureControl.zkMigrationState()); assertEquals( - "Invalid value true for configuration zookeeper.metadata.migration.enable: Should not have ZK migrations enabled on a cluster that was created in KRaft mode.", - assertThrows(ConfigException.class, () -> getActivationRecords(MetadataVersion.IBP_3_4_IV0, false, true)).getMessage() + "Should not have ZK migrations enabled on a cluster that was created in KRaft mode.", + assertThrows(RuntimeException.class, () -> getActivationRecords(MetadataVersion.IBP_3_4_IV0, false, true)).getMessage() ); featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, false, false); From 8923c4db307d9ef0a9b3c59887ef64941d5c65b3 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Thu, 20 Apr 2023 17:52:15 -0400 Subject: [PATCH 22/27] PR feedback --- .../controller/FeatureControlManager.java | 3 +- .../kafka/controller/QuorumController.java | 26 ++++--- .../org/apache/kafka/image/FeaturesImage.java | 7 +- .../migration/KRaftMigrationDriver.java | 67 ++++++++++--------- .../metadata/migration/ZkMigrationState.java | 4 ++ .../common/metadata/ZkMigrationRecord.json | 7 +- .../controller/QuorumControllerTest.java | 6 +- .../apache/kafka/image/FeaturesImageTest.java | 14 ++++ 8 files changed, 81 insertions(+), 53 deletions(-) diff --git a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java index f56cd483202b5..319c9d537eec1 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java @@ -19,7 +19,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map.Entry; @@ -257,7 +256,7 @@ private ApiError updateMetadataVersion( } // Don't allow metadata.version changes while we're migrating - if (EnumSet.of(ZkMigrationState.PRE_MIGRATION, ZkMigrationState.MIGRATION).contains(zkMigrationState)) { + if (zkMigrationState.inProgress()) { return invalidMetadataVersion(newVersionLevel, "Unable to modify metadata.version while a " + "ZK migration is in progress."); } diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 55ade709a3544..1ab0f00391d2d 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -1126,7 +1126,7 @@ private boolean isActiveController() { return isActiveController(curClaimEpoch); } - private boolean isActiveController(int claimEpoch) { + private static boolean isActiveController(int claimEpoch) { return claimEpoch != -1; } @@ -1184,14 +1184,14 @@ private void claim(int epoch) { * are committed to the log as an atomic batch. The records will include the bootstrap metadata records * (including the bootstrap "metadata.version") and may include a ZK migration record. */ - public static void generateActivationRecords( + public static List generateActivationRecords( Logger log, boolean isLogEmpty, boolean zkMigrationEnabled, BootstrapMetadata bootstrapMetadata, - FeatureControlManager featureControl, - Consumer recordConsumer + FeatureControlManager featureControl ) { + List records = new ArrayList<>(); if (isLogEmpty) { // If no records have been replayed, we need to write out the bootstrap records. // This will include the new metadata.version, as well as things like SCRAM @@ -1199,16 +1199,16 @@ public static void generateActivationRecords( log.info("The metadata log appears to be empty. Appending {} bootstrap record(s) " + "at metadata.version {} from {}.", bootstrapMetadata.records().size(), bootstrapMetadata.metadataVersion(), bootstrapMetadata.source()); - bootstrapMetadata.records().forEach(recordConsumer); + records.addAll(bootstrapMetadata.records()); if (bootstrapMetadata.metadataVersion().isMigrationSupported()) { if (zkMigrationEnabled) { log.info("Putting the controller into pre-migration mode. No metadata updates will be allowed until " + "the ZK metadata has been migrated"); - recordConsumer.accept(ZkMigrationState.PRE_MIGRATION.toRecord()); + records.add(ZkMigrationState.PRE_MIGRATION.toRecord()); } else { log.debug("Setting the ZK migration state to NONE since this is a de-novo KRaft cluster."); - recordConsumer.accept(ZkMigrationState.NONE.toRecord()); + records.add(ZkMigrationState.NONE.toRecord()); } } else { if (zkMigrationEnabled) { @@ -1237,9 +1237,12 @@ public static void generateActivationRecords( throw new RuntimeException("Detected an failed migration state during bootstrap, cannot continue."); case MIGRATION: if (!zkMigrationEnabled) { - log.info("Completing the ZK migration since this controller was configured with " + + // This can happen if controller leadership transfers to a controller with migrations enabled + // after another controller had finalized the migration. For example, during a rolling restart + // of the controller quorum during which the migration config is being set to false. + log.warn("Completing the ZK migration since this controller was configured with " + "'zookeeper.metadata.migration.enable' set to 'false'."); - recordConsumer.accept(ZkMigrationState.POST_MIGRATION.toRecord()); + records.add(ZkMigrationState.POST_MIGRATION.toRecord()); } else { log.info("Staying in the ZK migration since 'zookeeper.metadata.migration.enable' is still 'true'."); } @@ -1257,12 +1260,13 @@ public static void generateActivationRecords( } } } + return records; } class CompleteActivationEvent implements ControllerWriteOperation { @Override public ControllerResult generateRecordsAndResult() { - List records = new ArrayList<>(); - generateActivationRecords(log, logReplayTracker.empty(), zkMigrationEnabled, bootstrapMetadata, featureControl, records::add); + List records = generateActivationRecords(log, logReplayTracker.empty(), + zkMigrationEnabled, bootstrapMetadata, featureControl); return ControllerResult.atomicOf(records, null); } diff --git a/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java b/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java index c92798dec013a..623b45a8dba33 100644 --- a/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java +++ b/metadata/src/main/java/org/apache/kafka/image/FeaturesImage.java @@ -62,7 +62,9 @@ public FeaturesImage( } public boolean isEmpty() { - return finalizedVersions.isEmpty() && zkMigrationState.equals(ZkMigrationState.NONE); + return finalizedVersions.isEmpty() && + metadataVersion.equals(MetadataVersion.MINIMUM_KRAFT_VERSION) && + zkMigrationState.equals(ZkMigrationState.NONE); } public MetadataVersion metadataVersion() { @@ -130,7 +132,7 @@ private void writeFeatureLevels(ImageWriter writer, ImageWriterOptions options) @Override public int hashCode() { - return Objects.hash(finalizedVersions, zkMigrationState); + return Objects.hash(finalizedVersions, metadataVersion, zkMigrationState); } @Override @@ -138,6 +140,7 @@ public boolean equals(Object o) { if (!(o instanceof FeaturesImage)) return false; FeaturesImage other = (FeaturesImage) o; return finalizedVersions.equals(other.finalizedVersions) && + metadataVersion.equals(other.metadataVersion) && zkMigrationState.equals(other.zkMigrationState); } diff --git a/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java b/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java index 06b18cdb1309d..879fc09585073 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java +++ b/metadata/src/main/java/org/apache/kafka/metadata/migration/KRaftMigrationDriver.java @@ -209,6 +209,11 @@ private void apply(String name, Function optionalOf(byte value) { } return Optional.empty(); } + + public boolean inProgress() { + return this == PRE_MIGRATION || this == MIGRATION; + } } diff --git a/metadata/src/main/resources/common/metadata/ZkMigrationRecord.json b/metadata/src/main/resources/common/metadata/ZkMigrationRecord.json index 5c8085ce72f4b..aaaed4f4a083f 100644 --- a/metadata/src/main/resources/common/metadata/ZkMigrationRecord.json +++ b/metadata/src/main/resources/common/metadata/ZkMigrationRecord.json @@ -18,9 +18,12 @@ "type": "metadata", "name": "ZkMigrationStateRecord", // Version 0 adds ZkMigrationState which is used by the KRaft controller to mark the beginning and end - // of the ZK to KRaft migration. Possible values are 0 (None), 1 (PreMigration), 2 (Migration), 3 (PostMigration). + // of the ZK to KRaft migration. + // + // In 3.4, the defined values are: 0 (None), 1 (PreMigration), 2 (Migration), 3 (PostMigration). + // In 3.5, the values for PreMigration and Migration were swapped: 0 (None), 2 (PreMigration), 1 (Migration), 3 (PostMigration). + // This was done to work around the fact that we never wrote Migration or PostMigration records in 3.4 // - // Version 0 (3.5) adds PreMigrationSupported tagged field to distinguish between a 3.4 migration and a 3.5+ migration "validVersions": "0", "flexibleVersions": "0+", "fields": [ diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java index 20df0e5273db3..f207cd84b5c6e 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -1416,14 +1416,12 @@ FeatureControlManager getActivationRecords( .setMetadataVersion(metadataVersion) .build(); - List records = new ArrayList<>(); - QuorumController.generateActivationRecords( + List records = QuorumController.generateActivationRecords( log, emptyLog, zkMigrationEnabled, BootstrapMetadata.fromVersion(metadataVersion, "test"), - featureControlManager, - records::add); + featureControlManager); RecordTestUtils.replayAll(featureControlManager, records); return featureControlManager; } diff --git a/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java b/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java index ec86f3c79aeeb..915c14f8f01e9 100644 --- a/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java @@ -28,11 +28,14 @@ import org.junit.jupiter.api.Timeout; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; @Timeout(value = 40) @@ -96,4 +99,15 @@ private void testToImageAndBack(FeaturesImage image) throws Throwable { FeaturesImage nextImage = delta.apply(); assertEquals(image, nextImage); } + + @Test + public void testEmpty() { + assertTrue(FeaturesImage.EMPTY.isEmpty()); + assertFalse(new FeaturesImage(Collections.singletonMap("foo", (short) 1), + FeaturesImage.EMPTY.metadataVersion(), FeaturesImage.EMPTY.zkMigrationState()).isEmpty()); + assertFalse(new FeaturesImage(FeaturesImage.EMPTY.finalizedVersions(), + MetadataVersion.IBP_3_3_IV0, FeaturesImage.EMPTY.zkMigrationState()).isEmpty()); + assertFalse(new FeaturesImage(FeaturesImage.EMPTY.finalizedVersions(), + FeaturesImage.EMPTY.metadataVersion(), ZkMigrationState.MIGRATION).isEmpty()); + } } From e43d9243536dcd2e6ec7368ba1f2387eeaa86fbd Mon Sep 17 00:00:00 2001 From: David Arthur Date: Thu, 20 Apr 2023 17:56:36 -0400 Subject: [PATCH 23/27] Fix broker count in test --- .../integration/kafka/zk/ZkMigrationIntegrationTest.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala b/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala index 2908b01d09032..04f6813139af9 100644 --- a/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala @@ -335,11 +335,11 @@ class ZkMigrationIntegrationTest { * through the RPCs */ @ClusterTests(Array( - new ClusterTest(clusterType = Type.KRAFT, brokers = 0, controllers = 1, metadataVersion = MetadataVersion.IBP_3_4_IV0, + new ClusterTest(clusterType = Type.KRAFT, brokers = 1, controllers = 1, metadataVersion = MetadataVersion.IBP_3_4_IV0, serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))), - new ClusterTest(clusterType = Type.KRAFT, brokers = 0, controllers = 1, metadataVersion = MetadataVersion.IBP_3_5_IV0, + new ClusterTest(clusterType = Type.KRAFT, brokers = 1, controllers = 1, metadataVersion = MetadataVersion.IBP_3_5_IV0, serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))), - new ClusterTest(clusterType = Type.KRAFT, brokers = 0, controllers = 1, metadataVersion = MetadataVersion.IBP_3_5_IV1, + new ClusterTest(clusterType = Type.KRAFT, brokers = 1, controllers = 1, metadataVersion = MetadataVersion.IBP_3_5_IV1, serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))) )) def testPreMigrationMode(clusterInstance: ClusterInstance): Unit = { From 816e976efd7bb2e98f99e39a9e29b7da7d5596d5 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Sat, 22 Apr 2023 19:43:30 -0400 Subject: [PATCH 24/27] Fix unit test --- .../src/test/java/org/apache/kafka/image/FeaturesImageTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java b/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java index 915c14f8f01e9..155973b4a4f37 100644 --- a/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java @@ -93,7 +93,7 @@ public void testImage2RoundTrip() throws Throwable { private void testToImageAndBack(FeaturesImage image) throws Throwable { RecordListWriter writer = new RecordListWriter(); - image.write(writer, new ImageWriterOptions.Builder().build()); + image.write(writer, new ImageWriterOptions.Builder().setMetadataVersion(image.metadataVersion()).build()); FeaturesDelta delta = new FeaturesDelta(FeaturesImage.EMPTY); RecordTestUtils.replayAll(delta, writer.records()); FeaturesImage nextImage = delta.apply(); From bb6093e8dc92ca008aa32ea025e09a30110935de Mon Sep 17 00:00:00 2001 From: David Arthur Date: Mon, 24 Apr 2023 13:09:19 -0400 Subject: [PATCH 25/27] Add another test case for old MV --- .../controller/QuorumControllerTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java index f207cd84b5c6e..b97b3194c0ea9 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -1470,4 +1470,23 @@ public void testActivationRecords34() { assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); assertEquals(ZkMigrationState.NONE, featureControl.zkMigrationState()); } + + @Test + public void testMigrationsEnabledForOldBootstrapMetadataVersion() throws Exception { + try ( + LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv.Builder(1).build(); + ) { + QuorumControllerTestEnv.Builder controlEnvBuilder = new QuorumControllerTestEnv.Builder(logEnv). + setControllerBuilderInitializer(controllerBuilder -> { + controllerBuilder.setZkMigrationEnabled(true); + }). + setBootstrapMetadata(BootstrapMetadata.fromVersion(MetadataVersion.IBP_3_3_IV0, "test")); + + QuorumControllerTestEnv controlEnv = controlEnvBuilder.build(); + QuorumController active = controlEnv.activeController(); + assertEquals(ZkMigrationState.NONE, active.appendReadEvent("read migration state", OptionalLong.empty(), + () -> active.featureControl().zkMigrationState()).get(30, TimeUnit.SECONDS)); + assertThrows(FaultHandlerException.class, controlEnv::close); + } + } } From 15608e4920bfa46b22660d9fd6541b00fe031d6d Mon Sep 17 00:00:00 2001 From: David Arthur Date: Tue, 25 Apr 2023 13:12:09 -0400 Subject: [PATCH 26/27] Don't allow KRaft brokers to register during pre-migration --- checkstyle/suppressions.xml | 4 +- .../junit/RaftClusterInvocationContext.java | 15 ++-- .../kafka/zk/ZkMigrationIntegrationTest.scala | 34 +------- .../BrokerRegistrationRequestTest.scala | 80 ++++++++++++++----- .../controller/ClusterControlManager.java | 5 ++ .../kafka/controller/QuorumController.java | 28 ++++--- 6 files changed, 100 insertions(+), 66 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 706ff8d736943..8904cb6dc114e 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -306,9 +306,9 @@ + files="(ClientQuotasImage|KafkaEventQueue|MetadataDelta|QuorumController|ReplicationControlManager|KRaftMigrationDriver|ClusterControlManager).java"/> + files="(ClientQuotasImage|KafkaEventQueue|ReplicationControlManager|FeatureControlManager|KRaftMigrationDriver|ScramControlManager|ClusterControlManager).java"/> getAdditionalExtensions() { zkReference.set(new EmbeddedZookeeper()); builder.setConfigProp("zookeeper.connect", String.format("localhost:%d", zkReference.get().port())); } - // Copy properties into the TestKit builder clusterConfig.serverProperties().forEach((key, value) -> builder.setConfigProp(key.toString(), value.toString())); // KAFKA-12512 need to pass security protocol and listener name here KafkaClusterTestKit cluster = builder.build(); clusterReference.set(cluster); cluster.format(); - cluster.startup(); - kafka.utils.TestUtils.waitUntilTrue( - () -> cluster.brokers().get(0).brokerState() == BrokerState.RUNNING, - () -> "Broker never made it to RUNNING state.", - org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, - 100L); + if (clusterConfig.isAutoStart()) { + cluster.startup(); + kafka.utils.TestUtils.waitUntilTrue( + () -> cluster.brokers().get(0).brokerState() == BrokerState.RUNNING, + () -> "Broker never made it to RUNNING state.", + org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, + 100L); + } }, (AfterTestExecutionCallback) context -> clusterInstance.stop(), new ClusterInstanceParameterResolver(clusterInstance), diff --git a/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala b/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala index 04f6813139af9..33eefe757526e 100644 --- a/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala @@ -19,19 +19,18 @@ package kafka.zk import kafka.security.authorizer.AclEntry.{WildcardHost, WildcardPrincipalString} import kafka.server.{ConfigType, KafkaConfig} import kafka.test.ClusterInstance -import kafka.test.annotation.{AutoStart, ClusterConfigProperty, ClusterTest, ClusterTests, Type} +import kafka.test.annotation.{AutoStart, ClusterConfigProperty, ClusterTest, Type} import kafka.test.junit.ClusterTestExtensions import kafka.test.junit.ZkClusterInvocationContext.ZkClusterInstance import kafka.testkit.{KafkaClusterTestKit, TestKitNodes} import kafka.utils.{PasswordEncoder, TestUtils} -import org.apache.kafka.clients.admin.{Admin, AlterClientQuotasResult, AlterConfigOp, AlterConfigsResult, ConfigEntry, CreateTopicsOptions, NewTopic} +import org.apache.kafka.clients.admin._ import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} import org.apache.kafka.common.Uuid import org.apache.kafka.common.acl.AclOperation.{DESCRIBE, READ, WRITE} import org.apache.kafka.common.acl.AclPermissionType.ALLOW import org.apache.kafka.common.acl.{AccessControlEntry, AclBinding} import org.apache.kafka.common.config.{ConfigResource, TopicConfig} -import org.apache.kafka.common.errors.TimeoutException import org.apache.kafka.common.quota.{ClientQuotaAlteration, ClientQuotaEntity} import org.apache.kafka.common.resource.PatternType.{LITERAL, PREFIXED} import org.apache.kafka.common.resource.ResourcePattern @@ -44,13 +43,13 @@ import org.apache.kafka.metadata.authorizer.StandardAcl import org.apache.kafka.metadata.migration.ZkMigrationLeadershipState import org.apache.kafka.raft.RaftConfig import org.apache.kafka.server.common.{ApiMessageAndVersion, MetadataVersion, ProducerIdsBlock} -import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertNotNull, assertThrows, assertTrue} +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertNotNull, assertTrue} import org.junit.jupiter.api.Timeout import org.junit.jupiter.api.extension.ExtendWith import org.slf4j.LoggerFactory import java.util -import java.util.concurrent.{ExecutionException, TimeUnit} +import java.util.concurrent.TimeUnit import java.util.{Properties, UUID} import scala.collection.Seq import scala.jdk.CollectionConverters._ @@ -329,29 +328,4 @@ class ZkMigrationIntegrationTest { assertTrue(firstProducerIdBlock.firstProducerId() < producerIdBlock.firstProducerId()) } } - - /** - * Start a KRaft cluster with migrations enabled, verify that the controller does not accept metadata changes - * through the RPCs - */ - @ClusterTests(Array( - new ClusterTest(clusterType = Type.KRAFT, brokers = 1, controllers = 1, metadataVersion = MetadataVersion.IBP_3_4_IV0, - serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))), - new ClusterTest(clusterType = Type.KRAFT, brokers = 1, controllers = 1, metadataVersion = MetadataVersion.IBP_3_5_IV0, - serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))), - new ClusterTest(clusterType = Type.KRAFT, brokers = 1, controllers = 1, metadataVersion = MetadataVersion.IBP_3_5_IV1, - serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))) - )) - def testPreMigrationMode(clusterInstance: ClusterInstance): Unit = { - val adminClient = clusterInstance.createAdminClient() - val result = adminClient.createTopics( - java.util.Collections.singleton(new NewTopic("test", 10, 1.asInstanceOf[Short])), - new CreateTopicsOptions().timeoutMs(1000) - ) - - assertEquals( - classOf[TimeoutException], - assertThrows(classOf[ExecutionException], () => result.all().get(5, TimeUnit.MINUTES)).getCause.getClass - ) - } } diff --git a/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala b/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala index f19b6afb38051..5d46687b58eec 100644 --- a/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala @@ -15,28 +15,28 @@ * limitations under the License. */ -package unit.kafka.server +package kafka.server -import kafka.server.{BrokerToControllerChannelManager, ControllerInformation, ControllerNodeProvider, ControllerRequestCompletionHandler} import kafka.test.ClusterInstance -import kafka.test.annotation.{ClusterConfigProperty, ClusterTest, Type} +import kafka.test.annotation._ import kafka.test.junit.ClusterTestExtensions +import kafka.test.junit.RaftClusterInvocationContext.RaftClusterInstance import org.apache.kafka.clients.ClientResponse -import org.apache.kafka.common.errors.TimeoutException -import org.apache.kafka.common.message.{BrokerRegistrationRequestData, BrokerRegistrationResponseData} +import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic +import org.apache.kafka.common.message.{BrokerRegistrationRequestData, CreateTopicsRequestData} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.requests.{BrokerRegistrationRequest, BrokerRegistrationResponse} +import org.apache.kafka.common.requests._ import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.utils.Time import org.apache.kafka.common.{Node, Uuid} import org.apache.kafka.server.common.MetadataVersion -import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.{assertEquals, assertThrows} import org.junit.jupiter.api.extension.ExtendWith import org.junit.jupiter.api.{Tag, Timeout} -import java.util.concurrent.{CompletableFuture, TimeUnit} +import java.util.concurrent.{CompletableFuture, TimeUnit, TimeoutException} /** * This test simulates a broker registering with the KRaft quorum under different configurations. @@ -75,18 +75,19 @@ class BrokerRegistrationRequestTest { ) } - def sendAndRecieve( + def sendAndReceive[T <: AbstractRequest, R <: AbstractResponse]( channelManager: BrokerToControllerChannelManager, - req: BrokerRegistrationRequestData - ): BrokerRegistrationResponseData = { - val responseFuture = new CompletableFuture[BrokerRegistrationResponseData]() - channelManager.sendRequest(new BrokerRegistrationRequest.Builder(req), new ControllerRequestCompletionHandler() { + reqBuilder: AbstractRequest.Builder[T], + timeoutMs: Int + ): R = { + val responseFuture = new CompletableFuture[R]() + channelManager.sendRequest(reqBuilder, new ControllerRequestCompletionHandler() { override def onTimeout(): Unit = responseFuture.completeExceptionally(new TimeoutException()) override def onComplete(response: ClientResponse): Unit = - responseFuture.complete(response.responseBody().asInstanceOf[BrokerRegistrationResponse].data()) + responseFuture.complete(response.responseBody().asInstanceOf[R]) }) - responseFuture.get(30, TimeUnit.SECONDS) + responseFuture.get(timeoutMs, TimeUnit.MILLISECONDS) } def registerBroker( @@ -114,7 +115,22 @@ class BrokerRegistrationRequestTest { .setIsMigratingZkBroker(zkEpoch.isDefined) .setFeatures(features) - Errors.forCode(sendAndRecieve(channelManager, req).errorCode()) + val resp = sendAndReceive[BrokerRegistrationRequest, BrokerRegistrationResponse]( + channelManager, new BrokerRegistrationRequest.Builder(req), 30000) + Errors.forCode(resp.data().errorCode()) + } + + + def createTopics(channelManager: BrokerToControllerChannelManager, + topicName: String): Errors = { + val createTopics = new CreateTopicsRequestData() + createTopics.setTopics(new CreateTopicsRequestData.CreatableTopicCollection()) + createTopics.topics().add(new CreatableTopic().setName(topicName).setNumPartitions(10).setReplicationFactor(1)) + createTopics.setTimeoutMs(500) + + val req = new CreateTopicsRequest.Builder(createTopics) + val resp = sendAndReceive[CreateTopicsRequest, CreateTopicsResponse](channelManager, req, 3000).data() + Errors.forCode(resp.topics().find(topicName).errorCode()) } @ClusterTest(clusterType = Type.KRAFT, brokers = 0, controllers = 1, metadataVersion = MetadataVersion.IBP_3_4_IV0, @@ -177,9 +193,16 @@ class BrokerRegistrationRequestTest { } } - @ClusterTest(clusterType = Type.KRAFT, brokers = 0, controllers = 1, metadataVersion = MetadataVersion.IBP_3_4_IV0, + @ClusterTest( + clusterType = Type.KRAFT, + brokers = 1, + controllers = 1, + metadataVersion = MetadataVersion.IBP_3_4_IV0, + autoStart = AutoStart.NO, serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))) def testRegisterZkWithKRaftMigrationEnabled(clusterInstance: ClusterInstance): Unit = { + clusterInstance.asInstanceOf[RaftClusterInstance].controllers().forEach(_.startup()) + val clusterId = clusterInstance.clusterId() val channelManager = brokerToControllerChannelManager(clusterInstance) try { @@ -197,11 +220,32 @@ class BrokerRegistrationRequestTest { Errors.UNSUPPORTED_VERSION, registerBroker(channelManager, clusterId, 100, Some(1), Some((MetadataVersion.IBP_3_3_IV3, MetadataVersion.IBP_3_3_IV3)))) + // Cannot register KRaft broker when in pre-migration assertEquals( - Errors.NONE, + Errors.BROKER_ID_NOT_REGISTERED, registerBroker(channelManager, clusterId, 100, None, Some((MetadataVersion.IBP_3_4_IV0, MetadataVersion.IBP_3_4_IV0)))) } finally { channelManager.shutdown() } } + + /** + * Start a KRaft cluster with migrations enabled, verify that the controller does not accept metadata changes + * through the RPCs. The migration never proceeds past pre-migration since no ZK brokers are registered. + */ + @ClusterTests(Array( + new ClusterTest(clusterType = Type.KRAFT, autoStart = AutoStart.NO, controllers = 1, metadataVersion = MetadataVersion.IBP_3_4_IV0, + serverProperties = Array(new ClusterConfigProperty(key = "zookeeper.metadata.migration.enable", value = "true"))) + )) + def testNoMetadataChangesInPreMigrationMode(clusterInstance: ClusterInstance): Unit = { + clusterInstance.asInstanceOf[RaftClusterInstance].controllers().forEach(_.startup()) + + val channelManager = brokerToControllerChannelManager(clusterInstance) + try { + channelManager.start() + assertThrows(classOf[TimeoutException], () => createTopics(channelManager, "test-pre-migration")) + } finally { + channelManager.shutdown() + } + } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java index 98981b4c1f6a9..f0986f1d1a253 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java @@ -340,6 +340,11 @@ public ControllerResult registerBroker( throw new BrokerIdNotRegisteredException("Controller does not support registering ZK brokers."); } + if (!request.isMigratingZkBroker() && featureControl.inPreMigrationMode()) { + throw new BrokerIdNotRegisteredException("Controller is in pre-migration mode and cannot register KRaft " + + "brokers until the metadata migration is complete."); + } + RegisterBrokerRecord record = new RegisterBrokerRecord(). setBrokerId(brokerId). setIsMigratingZkBroker(request.isMigratingZkBroker()). diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 1ab0f00391d2d..23ee53e215006 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -415,7 +415,16 @@ private NotControllerException newNotControllerException() { OptionalInt latestController = raftClient.leaderAndEpoch().leaderId(); if (latestController.isPresent()) { return new NotControllerException(ACTIVE_CONTROLLER_EXCEPTION_TEXT_PREFIX + - latestController.getAsInt()); + latestController.getAsInt() + "."); + } else { + return new NotControllerException("No controller appears to be active."); + } + } + + private NotControllerException newPreMigrationException() { + OptionalInt latestController = raftClient.leaderAndEpoch().leaderId(); + if (latestController.isPresent()) { + return new NotControllerException("The controller is in pre-migration mode."); } else { return new NotControllerException("No controller appears to be active."); } @@ -685,7 +694,7 @@ public void run() throws Exception { } if (featureControl.inPreMigrationMode() && !flags.contains(RUNS_IN_PREMIGRATION)) { log.info("Cannot run write operation {} in pre-migration mode. Returning NOT_CONTROLLER.", name); - throw newNotControllerException(); + throw newPreMigrationException(); } startProcessingTimeNs = OptionalLong.of(now); ControllerResult result = op.generateRecordsAndResult(); @@ -1165,13 +1174,14 @@ private void claim(int epoch) { // Prepend the activate event. It is important that this event go at the beginning // of the queue rather than the end (hence prepend rather than append). It's also // important not to use prepend for anything else, to preserve the ordering here. - ControllerWriteEvent activationEvent = new ControllerWriteEvent<>("completeActivation[" + epoch + "]", - new CompleteActivationEvent(), - EnumSet.of(DOES_NOT_UPDATE_QUEUE_TIME, RUNS_IN_PREMIGRATION)); - activationEvent.future.whenComplete((__, t) -> { - if (t != null) { - fatalFaultHandler.handleFault("exception while activating controller", t); - } + ControllerWriteEvent activationEvent = new ControllerWriteEvent<>( + "completeActivation[" + epoch + "]", + new CompleteActivationEvent(), + EnumSet.of(DOES_NOT_UPDATE_QUEUE_TIME, RUNS_IN_PREMIGRATION) + ); + activationEvent.future.exceptionally(t -> { + fatalFaultHandler.handleFault("exception while activating controller", t); + return null; }); queue.prepend(activationEvent); } catch (Throwable e) { From a4422d60ce313052d4c2b3d8424f3fb859ebb777 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Tue, 25 Apr 2023 16:20:26 -0400 Subject: [PATCH 27/27] Fix one more activation case, add tests --- .../kafka/controller/QuorumController.java | 5 +- .../controller/QuorumControllerTest.java | 52 +++++++++++++++---- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 23ee53e215006..3364a321e8d32 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -928,7 +928,7 @@ public CompletableFuture completeMigration() { @Override public void abortMigration() { - log.error("Aborting ZK Migration"); + fatalFaultHandler.handleFault("Aborting the ZK migration"); // TODO use KIP-868 transaction } } @@ -1244,7 +1244,8 @@ public static List generateActivationRecords( } break; case PRE_MIGRATION: - throw new RuntimeException("Detected an failed migration state during bootstrap, cannot continue."); + log.warn("Activating pre-migration controller without empty log. There may be a partial migration"); + break; case MIGRATION: if (!zkMigrationEnabled) { // This can happen if controller leadership transfers to a controller with migrations enabled diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java index b97b3194c0ea9..4f3a39b118d67 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -49,6 +49,7 @@ import org.apache.kafka.common.metadata.BrokerRegistrationChangeRecord; import org.apache.kafka.common.metadata.ConfigRecord; import org.apache.kafka.common.metadata.UnfenceBrokerRecord; +import org.apache.kafka.common.metadata.ZkMigrationStateRecord; import org.apache.kafka.common.requests.AlterPartitionRequest; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.utils.LogContext; @@ -1407,7 +1408,7 @@ public void testUpgradeMigrationStateFrom34() throws Exception { FeatureControlManager getActivationRecords( MetadataVersion metadataVersion, - boolean emptyLog, + Optional stateInLog, boolean zkMigrationEnabled ) { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); @@ -1416,9 +1417,12 @@ FeatureControlManager getActivationRecords( .setMetadataVersion(metadataVersion) .build(); + stateInLog.ifPresent(zkMigrationState -> + featureControlManager.replay((ZkMigrationStateRecord) zkMigrationState.toRecord().message())); + List records = QuorumController.generateActivationRecords( log, - emptyLog, + !stateInLog.isPresent(), zkMigrationEnabled, BootstrapMetadata.fromVersion(metadataVersion, "test"), featureControlManager); @@ -1432,19 +1436,19 @@ public void testActivationRecords33() { assertEquals( "The bootstrap metadata.version 3.3-IV0 does not support ZK migrations. Cannot continue with ZK migrations enabled.", - assertThrows(RuntimeException.class, () -> getActivationRecords(MetadataVersion.IBP_3_3_IV0, true, true)).getMessage() + assertThrows(RuntimeException.class, () -> getActivationRecords(MetadataVersion.IBP_3_3_IV0, Optional.empty(), true)).getMessage() ); - featureControl = getActivationRecords(MetadataVersion.IBP_3_3_IV0, true, false); + featureControl = getActivationRecords(MetadataVersion.IBP_3_3_IV0, Optional.empty(), false); assertEquals(MetadataVersion.IBP_3_3_IV0, featureControl.metadataVersion()); assertEquals(ZkMigrationState.NONE, featureControl.zkMigrationState()); assertEquals( "Should not have ZK migrations enabled on a cluster running metadata.version 3.3-IV0", - assertThrows(RuntimeException.class, () -> getActivationRecords(MetadataVersion.IBP_3_3_IV0, false, true)).getMessage() + assertThrows(RuntimeException.class, () -> getActivationRecords(MetadataVersion.IBP_3_3_IV0, Optional.of(ZkMigrationState.NONE), true)).getMessage() ); - featureControl = getActivationRecords(MetadataVersion.IBP_3_3_IV0, false, false); + featureControl = getActivationRecords(MetadataVersion.IBP_3_3_IV0, Optional.of(ZkMigrationState.NONE), false); assertEquals(MetadataVersion.IBP_3_3_IV0, featureControl.metadataVersion()); assertEquals(ZkMigrationState.NONE, featureControl.zkMigrationState()); } @@ -1453,24 +1457,52 @@ public void testActivationRecords33() { public void testActivationRecords34() { FeatureControlManager featureControl; - featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, true, true); + featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, Optional.empty(), true); assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); assertEquals(ZkMigrationState.PRE_MIGRATION, featureControl.zkMigrationState()); - featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, true, false); + featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, Optional.empty(), false); assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); assertEquals(ZkMigrationState.NONE, featureControl.zkMigrationState()); assertEquals( "Should not have ZK migrations enabled on a cluster that was created in KRaft mode.", - assertThrows(RuntimeException.class, () -> getActivationRecords(MetadataVersion.IBP_3_4_IV0, false, true)).getMessage() + assertThrows(RuntimeException.class, () -> getActivationRecords(MetadataVersion.IBP_3_4_IV0, Optional.of(ZkMigrationState.NONE), true)).getMessage() ); - featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, false, false); + featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, Optional.of(ZkMigrationState.NONE), false); assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); assertEquals(ZkMigrationState.NONE, featureControl.zkMigrationState()); + + featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, Optional.of(ZkMigrationState.PRE_MIGRATION), true); + assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); + assertEquals(ZkMigrationState.PRE_MIGRATION, featureControl.zkMigrationState()); + + featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, Optional.of(ZkMigrationState.MIGRATION), true); + assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); + assertEquals(ZkMigrationState.MIGRATION, featureControl.zkMigrationState()); + + featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, Optional.of(ZkMigrationState.MIGRATION), false); + assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); + assertEquals(ZkMigrationState.POST_MIGRATION, featureControl.zkMigrationState()); + + featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, Optional.of(ZkMigrationState.POST_MIGRATION), true); + assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); + assertEquals(ZkMigrationState.POST_MIGRATION, featureControl.zkMigrationState()); + + featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, Optional.of(ZkMigrationState.POST_MIGRATION), false); + assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); + assertEquals(ZkMigrationState.POST_MIGRATION, featureControl.zkMigrationState()); } + @Test + public void testActivationRecordsNonEmptyLog() { + FeatureControlManager featureControl; + + featureControl = getActivationRecords(MetadataVersion.IBP_3_4_IV0, Optional.empty(), true); + assertEquals(MetadataVersion.IBP_3_4_IV0, featureControl.metadataVersion()); + assertEquals(ZkMigrationState.PRE_MIGRATION, featureControl.zkMigrationState()); } + @Test public void testMigrationsEnabledForOldBootstrapMetadataVersion() throws Exception { try (