Skip to content

KAFKA-16968: Make 3.8-IV0 a stable MetadataVersion and create 3.9-IV0 #16347

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cmccabe merged 16 commits into from
Jun 27, 2024
Merged

KAFKA-16968: Make 3.8-IV0 a stable MetadataVersion and create 3.9-IV0 #16347

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
fbb566f
KAFKA-16968: Make 3.8-IV0 a stable MetadataVersion and create 3.9-IV0
cmccabe Jun 15, 2024
ea90660
re-add 4.0 and add a comment about unstable MVs
cmccabe Jun 17, 2024
43daca5
fix test
cmccabe Jun 18, 2024
12370b4
Bump ListOffsetsRequest to v9, mark latest version unstable
cmccabe Jun 18, 2024
f96af84
address review comments
cmccabe Jun 18, 2024
4e9c025
fixes
cmccabe Jun 18, 2024
68d1265
more updates
cmccabe Jun 20, 2024
84ea68c
address review comments
cmccabe Jun 24, 2024
77e8b86
Merge branch 'trunk' into KAFKA-16968
cmccabe Jun 24, 2024
95e1a49
Add comment
cmccabe Jun 24, 2024
933aed5
fix tests
cmccabe Jun 25, 2024
a2c7f0a
Merge branch 'trunk' into KAFKA-16968
cmccabe Jun 25, 2024
c06985d
fix checkstyle
cmccabe Jun 25, 2024
c183b4d
test fixes
cmccabe Jun 26, 2024
1545da7
Fix ApiVersionsRequestTest
cmccabe Jun 27, 2024
fccf7cb
Fix SaslApiVersionsRequestTest
cmccabe Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/src/test/java/kafka/test/ClusterTestExtensionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ public void testNoAutoStart() {

@ClusterTest
public void testDefaults(ClusterInstance clusterInstance) {
Assertions.assertEquals(MetadataVersion.IBP_4_0_IV0, clusterInstance.config().metadataVersion());
Assertions.assertEquals(MetadataVersion.latestTesting(), clusterInstance.config().metadataVersion());
}

@ClusterTests({
Expand Down
2 changes: 1 addition & 1 deletion core/src/test/java/kafka/test/annotation/ClusterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
AutoStart autoStart() default AutoStart.DEFAULT;
SecurityProtocol securityProtocol() default SecurityProtocol.PLAINTEXT;
String listener() default "";
MetadataVersion metadataVersion() default MetadataVersion.IBP_4_0_IV0;
MetadataVersion metadataVersion() default MetadataVersion.IBP_3_9_IV0;
ClusterConfigProperty[] serverProperties() default {};
// users can add tags that they want to display in test
String[] tags() default {};
Expand Down
2 changes: 1 addition & 1 deletion core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ object ZkMigrationIntegrationTest {
MetadataVersion.IBP_3_7_IV2,
MetadataVersion.IBP_3_7_IV4,
MetadataVersion.IBP_3_8_IV0,
MetadataVersion.IBP_4_0_IV0
MetadataVersion.IBP_3_9_IV0
Comment thread
cmccabe marked this conversation as resolved.
Outdated
).map { mv =>
val serverProperties = new util.HashMap[String, String]()
serverProperties.put("inter.broker.listener.name", "EXTERNAL")
Expand Down
4 changes: 2 additions & 2 deletions core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ object ApiVersionsRequestTest {
List(ClusterConfig.defaultBuilder()
.setTypes(java.util.Collections.singleton(Type.ZK))
.setServerProperties(serverProperties)
.setMetadataVersion(MetadataVersion.IBP_4_0_IV0)
.setMetadataVersion(MetadataVersion.IBP_3_9_IV0)
Comment thread
cmccabe marked this conversation as resolved.
Outdated
.build()).asJava
}

Expand Down Expand Up @@ -83,7 +83,7 @@ object ApiVersionsRequestTest {
class ApiVersionsRequestTest(cluster: ClusterInstance) extends AbstractApiVersionsRequestTest(cluster) {

@ClusterTemplate("testApiVersionsRequestTemplate")
@ClusterTest(types = Array(Type.KRAFT, Type.CO_KRAFT), metadataVersion = MetadataVersion.IBP_4_0_IV0, serverProperties = Array(
@ClusterTest(types = Array(Type.KRAFT, Type.CO_KRAFT), metadataVersion = MetadataVersion.IBP_3_9_IV0, serverProperties = Array(
new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "false"),
new ClusterConfigProperty(key = "unstable.feature.versions.enable", value = "true")
))
Expand Down
2 changes: 1 addition & 1 deletion core/src/test/scala/unit/kafka/server/ReplicationQuotasTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ class ReplicationQuotasTest extends QuorumTestHarness {
features.add(new BrokerRegistrationRequestData.Feature()
.setName(MetadataVersion.FEATURE_NAME)
.setMinSupportedVersion(MetadataVersion.IBP_3_0_IV1.featureLevel())
.setMaxSupportedVersion(MetadataVersion.IBP_4_0_IV0.featureLevel()))
.setMaxSupportedVersion(MetadataVersion.latestTesting().featureLevel()))
controllerServer.controller.registerBroker(
ControllerRequestContextUtil.ANONYMOUS_CONTEXT,
new BrokerRegistrationRequestData()
Expand Down
2 changes: 1 addition & 1 deletion metadata/src/test/java/org/apache/kafka/controller/PartitionChangeBuilderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ private static MetadataVersion metadataVersionForPartitionChangeRecordVersion(sh
case (short) 1:
return MetadataVersion.IBP_3_7_IV2;
case (short) 2:
return MetadataVersion.IBP_3_8_IV0;
return MetadataVersion.IBP_3_9_IV0;
default:
throw new RuntimeException("Unknown PartitionChangeRecord version " + version);
}
Expand Down
16 changes: 8 additions & 8 deletions metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ public void testConfigurationOperations() throws Throwable {
) {
controlEnv.activeController().registerBroker(ANONYMOUS_CONTEXT,
new BrokerRegistrationRequestData().
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_4_0_IV0)).
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.latestTesting())).
setBrokerId(0).
setLogDirs(Collections.singletonList(Uuid.fromString("iiaQjkRPQcuMULNII0MUeA"))).
setClusterId(logEnv.clusterId())).get();
Expand Down Expand Up @@ -240,7 +240,7 @@ public void testDelayedConfigurationOperations() throws Throwable {
) {
controlEnv.activeController().registerBroker(ANONYMOUS_CONTEXT,
new BrokerRegistrationRequestData().
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_4_0_IV0)).
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.latestTesting())).
setBrokerId(0).
setLogDirs(Collections.singletonList(Uuid.fromString("sTbzRAMnTpahIyIPNjiLhw"))).
setClusterId(logEnv.clusterId())).get();
Expand Down Expand Up @@ -298,7 +298,7 @@ public void testFenceMultipleBrokers() throws Throwable {
new BrokerRegistrationRequestData().
setBrokerId(brokerId).
setClusterId(active.clusterId()).
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_4_0_IV0)).
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.latestTesting())).
setIncarnationId(Uuid.randomUuid()).
setListeners(listeners));
brokerEpochs.put(brokerId, reply.get().epoch());
Expand Down Expand Up @@ -380,7 +380,7 @@ public void testUncleanShutdownBroker() throws Throwable {
}).
setSessionTimeoutMillis(OptionalLong.of(sessionTimeoutMillis)).

setBootstrapMetadata(BootstrapMetadata.fromVersion(MetadataVersion.IBP_3_8_IV0, "test-provided bootstrap ELR enabled")).
setBootstrapMetadata(BootstrapMetadata.fromVersion(MetadataVersion.IBP_3_9_IV0, "test-provided bootstrap ELR enabled")).
build()
) {
ListenerCollection listeners = new ListenerCollection();
Expand All @@ -394,7 +394,7 @@ public void testUncleanShutdownBroker() throws Throwable {
new BrokerRegistrationRequestData().
setBrokerId(brokerId).
setClusterId(active.clusterId()).
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_8_IV0)).
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_9_IV0)).
setIncarnationId(Uuid.randomUuid()).
setLogDirs(Collections.singletonList(Uuid.randomUuid())).
setListeners(listeners));
Expand Down Expand Up @@ -463,7 +463,7 @@ public void testUncleanShutdownBroker() throws Throwable {
new BrokerRegistrationRequestData().
setBrokerId(brokerToUncleanShutdown).
setClusterId(active.clusterId()).
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_8_IV0)).
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_9_IV0)).
setIncarnationId(Uuid.randomUuid()).
setLogDirs(Collections.singletonList(Uuid.randomUuid())).
setListeners(listeners)).get();
Expand All @@ -476,7 +476,7 @@ public void testUncleanShutdownBroker() throws Throwable {
new BrokerRegistrationRequestData().
setBrokerId(lastKnownElr[0]).
setClusterId(active.clusterId()).
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_8_IV0)).
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_9_IV0)).
setIncarnationId(Uuid.randomUuid()).
setLogDirs(Collections.singletonList(Uuid.randomUuid())).
setListeners(listeners)).get();
Expand Down Expand Up @@ -717,7 +717,7 @@ public void testUnregisterBroker() throws Throwable {
setBrokerId(0).
setClusterId(active.clusterId()).
setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwBA")).
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_4_0_IV0)).
setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.latestTesting())).
setLogDirs(Collections.singletonList(Uuid.fromString("vBpaRsZVSaGsQT53wtYGtg"))).
setListeners(listeners));
assertEquals(5L, reply.get().epoch());
Expand Down
2 changes: 1 addition & 1 deletion metadata/src/test/java/org/apache/kafka/metadata/PartitionRegistrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,7 @@ public void testPartitionRegistrationToRecord_ElrShouldBeNullIfEmpty() {
setPartitionEpoch(0);
List<UnwritableMetadataException> exceptions = new ArrayList<>();
ImageWriterOptions options = new ImageWriterOptions.Builder().
setMetadataVersion(MetadataVersion.IBP_3_8_IV0).
setMetadataVersion(MetadataVersion.IBP_3_9_IV0).
Comment thread
cmccabe marked this conversation as resolved.
Outdated
setLossHandler(exceptions::add).
build();
assertEquals(new ApiMessageAndVersion(expectRecord, (short) 2), partitionRegistration.toRecord(topicID, 0, options));
Expand Down
2 changes: 1 addition & 1 deletion server-common/src/main/java/org/apache/kafka/server/common/GroupVersion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
public enum GroupVersion implements FeatureVersion {

// Version 1 enables the consumer rebalance protocol (KIP-848).
GV_1(1, MetadataVersion.IBP_4_0_IV0, Collections.emptyMap());
GV_1(1, MetadataVersion.IBP_3_9_IV0, Collections.emptyMap());
Comment thread
cmccabe marked this conversation as resolved.
Outdated

public static final String FEATURE_NAME = "group.version";

Expand Down
11 changes: 6 additions & 5 deletions server-common/src/main/java/org/apache/kafka/server/common/MetadataVersion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,11 +202,12 @@ public enum MetadataVersion {
// Add new fetch request version for KIP-951
IBP_3_7_IV4(19, "3.7", "IV4", false),

// Add ELR related supports (KIP-966).
IBP_3_8_IV0(20, "3.8", "IV0", true),
// New version for the Kafka 3.8.0 release.
IBP_3_8_IV0(20, "3.8", "IV0", false),

// Add ELR related supports (KIP-966).
Comment thread
cmccabe marked this conversation as resolved.
// Introduce version 1 of the GroupVersion feature (KIP-848).
IBP_4_0_IV0(21, "4.0", "IV0", false);
IBP_3_9_IV0(21, "3.9", "IV0", true);

// NOTES when adding a new version:
// Update the default version in @ClusterTest annotation to point to the latest version
Expand All @@ -232,7 +233,7 @@ public enum MetadataVersion {
* <strong>Think carefully before you update this value. ONCE A METADATA VERSION IS PRODUCTION,
* IT CANNOT BE CHANGED.</strong>
*/
public static final MetadataVersion LATEST_PRODUCTION = IBP_3_7_IV4;
public static final MetadataVersion LATEST_PRODUCTION = IBP_3_8_IV0;
Comment thread
cmccabe marked this conversation as resolved.

/**
* An array containing all of the MetadataVersion entries.
Expand Down Expand Up @@ -331,7 +332,7 @@ public boolean isDirectoryAssignmentSupported() {
}

public boolean isElrSupported() {
return this.isAtLeast(IBP_3_8_IV0);
return this.isAtLeast(IBP_3_9_IV0);
Comment thread
cmccabe marked this conversation as resolved.
Outdated
Comment thread
cmccabe marked this conversation as resolved.
Outdated
}

public boolean isKRaftSupported() {
Expand Down
4 changes: 2 additions & 2 deletions server-common/src/main/java/org/apache/kafka/server/common/TestFeatureVersion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ public enum TestFeatureVersion implements FeatureVersion {

// TEST_1 released right before MV 3.7-IVO was released, and it has no dependencies
TEST_1(1, MetadataVersion.IBP_3_7_IV0, Collections.emptyMap()),
// TEST_2 released right before MV 3.8-IVO was released, and it depends on this metadata version
TEST_2(2, MetadataVersion.IBP_3_8_IV0, Collections.singletonMap(MetadataVersion.FEATURE_NAME, MetadataVersion.IBP_3_8_IV0.featureLevel()));
// TEST_2 released right before MV 3.9-IVO was released, and it depends on this metadata version
TEST_2(2, MetadataVersion.IBP_3_9_IV0, Collections.singletonMap(MetadataVersion.FEATURE_NAME, MetadataVersion.IBP_3_9_IV0.featureLevel()));

private final short featureLevel;
private final MetadataVersion metadataVersionMapping;
Expand Down
2 changes: 1 addition & 1 deletion server-common/src/test/java/org/apache/kafka/server/common/FeaturesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public void testLatestProductionMapsToLatestMetadataVersion(Features features) {
@EnumSource(MetadataVersion.class)
public void testDefaultTestVersion(MetadataVersion metadataVersion) {
short expectedVersion;
if (!metadataVersion.isLessThan(MetadataVersion.IBP_3_8_IV0)) {
if (!metadataVersion.isLessThan(MetadataVersion.IBP_3_9_IV0)) {
Comment thread
cmccabe marked this conversation as resolved.
expectedVersion = 2;
} else if (!metadataVersion.isLessThan(MetadataVersion.IBP_3_7_IV0)) {
expectedVersion = 1;
Expand Down
9 changes: 5 additions & 4 deletions server-common/src/test/java/org/apache/kafka/server/common/MetadataVersionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,9 +184,10 @@ public void testFromVersionString() {
assertEquals(IBP_3_7_IV3, MetadataVersion.fromVersionString("3.7-IV3"));
assertEquals(IBP_3_7_IV4, MetadataVersion.fromVersionString("3.7-IV4"));

assertEquals(IBP_3_8_IV0, MetadataVersion.fromVersionString("3.8"));
Comment thread
cmccabe marked this conversation as resolved.
assertEquals(IBP_3_8_IV0, MetadataVersion.fromVersionString("3.8-IV0"));

assertEquals(IBP_4_0_IV0, MetadataVersion.fromVersionString("4.0-IV0"));
assertEquals(IBP_3_9_IV0, MetadataVersion.fromVersionString("3.9-IV0"));
}

@Test
Expand Down Expand Up @@ -247,7 +248,7 @@ public void testShortVersion() {
assertEquals("3.7", IBP_3_7_IV3.shortVersion());
assertEquals("3.7", IBP_3_7_IV4.shortVersion());
assertEquals("3.8", IBP_3_8_IV0.shortVersion());
assertEquals("4.0", IBP_4_0_IV0.shortVersion());
assertEquals("3.9", IBP_3_9_IV0.shortVersion());
}

@Test
Expand Down Expand Up @@ -297,7 +298,7 @@ public void testVersion() {
assertEquals("3.7-IV3", IBP_3_7_IV3.version());
assertEquals("3.7-IV4", IBP_3_7_IV4.version());
assertEquals("3.8-IV0", IBP_3_8_IV0.version());
assertEquals("4.0-IV0", IBP_4_0_IV0.version());
assertEquals("3.9-IV0", IBP_3_9_IV0.version());
}

@Test
Expand Down Expand Up @@ -365,7 +366,7 @@ public void testDirectoryAssignmentSupported(MetadataVersion metadataVersion) {
@ParameterizedTest
@EnumSource(value = MetadataVersion.class)
public void testIsElrSupported(MetadataVersion metadataVersion) {
assertEquals(metadataVersion.isAtLeast(IBP_3_8_IV0), metadataVersion.isElrSupported());
assertEquals(metadataVersion.isAtLeast(IBP_3_9_IV0), metadataVersion.isElrSupported());
}

@ParameterizedTest
Expand Down