From c1d9ff49c379f9ad81d01a19a1cafc8c77626aaa Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Sun, 3 Jan 2021 20:58:13 -0700 Subject: [PATCH] generate definitions / kversion: partial KIP-516 support Some of KIP-516 was added to Kafka; this ports what was added: - metadata - leader and ISR - update metadata --- generate/definitions/03_metadata | 16 +++++++++++++--- generate/definitions/04_leader_and_isr | 20 ++++++++++++++------ generate/definitions/06_update_metadata | 5 +++-- pkg/kversion/kversion.go | 7 +++++++ 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/generate/definitions/03_metadata b/generate/definitions/03_metadata index fbcfb0fb..dcf3dc97 100644 --- a/generate/definitions/03_metadata +++ b/generate/definitions/03_metadata @@ -1,11 +1,16 @@ // MetadataRequest requests metadata from Kafka. -MetadataRequest => key 3, max version 9, flexible v9+ +MetadataRequest => key 3, max version 10, flexible v9+ // Topics is a list of topics to return metadata about. If this is null // in v1+, all topics are included. If this is empty, no topics are. // For v0 (] - // Topic is the topic to request metadata for. - Topic: string + // The topic ID. Only one of either topic ID or topic name should be used. + // If using the topic name, this should just be the default empty value. + TopicID: uuid // v10+ + // Topic is the topic to request metadata for. Version 10 switched this + // from a string to a nullable string; if using a topic ID, this field + // should be null. + Topic: nullable-string-v10+ // AllowAutoTopicCreation, introduced in Kafka 0.11.0.0, allows topic // auto creation of the topics in this request if they do not exist. AllowAutoTopicCreation: bool // v4+ @@ -59,6 +64,8 @@ MetadataResponse => ErrorCode: int16 // Topic is the topic this metadata corresponds to. Topic: string + // The topic ID. + TopicID: uuid // v10+ // IsInternal signifies whether this topic is a Kafka internal topic. IsInternal: bool // v1+ // Partitions contains metadata about partitions for a topic. @@ -74,6 +81,9 @@ MetadataResponse => // // REPLICA_NOT_AVAILABLE is returned in v0 responses if any replica is // unavailable. + // + // UNKNOWN_TOPIC_ID is returned if using a topic ID and the ID does not + // exist. ErrorCode: int16 // Partition is a partition number for a topic. Partition: int32 diff --git a/generate/definitions/04_leader_and_isr b/generate/definitions/04_leader_and_isr index dbd5c6e2..dfafb8c4 100644 --- a/generate/definitions/04_leader_and_isr +++ b/generate/definitions/04_leader_and_isr @@ -1,6 +1,6 @@ // LeaderAndISRRequestTopicPartition is a common struct that is used across // different versions of LeaderAndISRRequest. -LeaderAndISRRequestTopicPartition => not top level, no encoding +LeaderAndISRRequestTopicPartition => not top level, no encoding, flexible v4+ Topic: string // v0-v1 Partition: int32 ControllerEpoch: int32 @@ -13,6 +13,12 @@ LeaderAndISRRequestTopicPartition => not top level, no encoding RemovingReplicas: [int32] // v3+ IsNew: bool // v1+ +// LeaderAndISRResponseTopicPartition is a common struct that is used across +// different versions of LeaderAndISRResponse. +LeaderAndISRResponseTopicPartition => not top level, no encoding, flexible v4+ + Topic: string // v0-v4 + Partition: int32 + ErrorCode: int16 // LeaderAndISRRequest is an advanced request that controller brokers use // to broadcast state to other brokers. Manually using this request is a @@ -24,13 +30,15 @@ LeaderAndISRRequestTopicPartition => not top level, no encoding // Kafka 1.0.0 introduced version 1. Kafka 2.2.0 introduced version 2, proposed // in KIP-380, which changed the layout of the struct to be more memory // efficient. Kafka 2.4.0 introduced version 3 with KIP-455. -LeaderAndISRRequest => key 4, max version 4, flexible v4+ +LeaderAndISRRequest => key 4, max version 5, flexible v4+ ControllerID: int32 ControllerEpoch: int32 BrokerEpoch: int64(-1) // v2+ + Type: int8 // v5+ PartitionStates: [LeaderAndISRRequestTopicPartition] // v0-v1 TopicStates: [=>] // v2+ Topic: string + TopicID: uuid // v5+ PartitionStates: [LeaderAndISRRequestTopicPartition] LiveLeaders: [=>] BrokerID: int32 @@ -40,7 +48,7 @@ LeaderAndISRRequest => key 4, max version 4, flexible v4+ // LeaderAndISRResponse is returned from a LeaderAndISRRequest. LeaderAndISRResponse => ErrorCode: int16 - Partitions: [=>] - Topic: string - Partition: int32 - ErrorCode: int16 + Partitions: [LeaderAndISRResponseTopicPartition] // v0-v4 + Topics: [=>] // v5+ + TopicID: uuid + Partitions: [LeaderAndISRResponseTopicPartition] diff --git a/generate/definitions/06_update_metadata b/generate/definitions/06_update_metadata index 632838a4..b8dce73f 100644 --- a/generate/definitions/06_update_metadata +++ b/generate/definitions/06_update_metadata @@ -1,4 +1,4 @@ -UpdateMetadataRequestTopicPartition => not top level, no encoding +UpdateMetadataRequestTopicPartition => not top level, no encoding, flexible v6+ Topic: string // v0-v4 Partition: int32 ControllerEpoch: int32 @@ -19,13 +19,14 @@ UpdateMetadataRequestTopicPartition => not top level, no encoding // // Kafka 2.2.0 introduced version 5, proposed in KIP-380, which changed the // layout of the struct to be more memory efficient. -UpdateMetadataRequest => key 6, max version 6, flexible v6+ +UpdateMetadataRequest => key 6, max version 7, flexible v6+ ControllerID: int32 ControllerEpoch: int32 BrokerEpoch: int64(-1) // v5+ PartitionStates: [UpdateMetadataRequestTopicPartition] // v0-v4 TopicStates: [=>] // v5+ Topic: string + TopicID: uuid // v7+ PartitionStates: [UpdateMetadataRequestTopicPartition] LiveBrokers: [=>] ID: int32 diff --git a/pkg/kversion/kversion.go b/pkg/kversion/kversion.go index 38ed737f..2eab54bf 100644 --- a/pkg/kversion/kversion.go +++ b/pkg/kversion/kversion.go @@ -454,5 +454,12 @@ func Tip() Versions { v[48]++ // 1 describe client quotas v[49]++ // 1 alter client quotas + // KAFKA-10547 5c921afa4a593478f7d1c49e5db9d787558d0d5e KIP-516 + v[3]++ // 10 metadata + v[6]++ // 7 update metadata + + // KAFKA-10545 1dd1e7f945d7a8c1dc177223cd88800680f1ff46 KIP-516 + v[4]++ // 5 leader and isr + return v }