From cc0247bf535e79ffa4b55204a7ff15340354b566 Mon Sep 17 00:00:00 2001 From: Geordie Date: Fri, 4 Dec 2020 23:56:01 +0800 Subject: [PATCH 01/14] MINOR: Leaves lock() outside the try block (#9687) Reviewers: Chia-Ping Tsai --- .../org/apache/kafka/trogdor/workload/RoundTripWorker.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java index 643555a8e95ee..6f448cc2ed31c 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java @@ -254,8 +254,8 @@ public void run() { spec.valueGenerator().generate(messageIndex)); producer.send(record, (metadata, exception) -> { if (exception == null) { + lock.lock(); try { - lock.lock(); unackedSends -= 1; if (unackedSends <= 0) unackedSendsAreZero.signalAll(); @@ -272,8 +272,8 @@ public void run() { } catch (Throwable e) { WorkerUtils.abort(log, "ProducerRunnable", e, doneFuture); } finally { + lock.lock(); try { - lock.lock(); log.info("{}: ProducerRunnable is exiting. messagesSent={}; uniqueMessagesSent={}; " + "ackedSends={}/{}.", id, messagesSent, uniqueMessagesSent, spec.maxMessages() - unackedSends, spec.maxMessages()); @@ -359,8 +359,8 @@ public void run() { if (toReceiveTracker.removePending(messageIndex)) { uniqueMessagesReceived++; if (uniqueMessagesReceived >= spec.maxMessages()) { + lock.lock(); try { - lock.lock(); log.info("{}: Consumer received the full count of {} unique messages. " + "Waiting for all {} sends to be acked...", id, spec.maxMessages(), unackedSends); while (unackedSends > 0) From 4f2f08eb006cdf3dc848b34b4b17cf27a023e9da Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Fri, 4 Dec 2020 12:48:23 -0500 Subject: [PATCH 02/14] KAFKA-10792: Prevent source task shutdown from blocking herder thread (#9669) Changes the `WorkerSourceTask` class to only call `SourceTask::stop` from the task thread when the task is actually stopped (via `Source:task::close` just before `WorkerTask::run` completes), and only if an attempt has been made to start the task (which will not be the case if it was created in the paused state and then shut down before being started). This prevents `SourceTask::stop` from being indirectly invoked on the herder's thread, which can have adverse effects if the task is unable to shut down promptly. Unit tests are tweaked where necessary to account for this new logic, which covers some edge cases mentioned in PR #5020 that were unaddressed up until now. The existing integration tests for blocking connectors are expanded to also include cases for blocking source and sink tasks. Full coverage of every source/sink task method is intentionally omitted from these expanded tests in order to avoid inflating test runtime (each one adds an extra 5 seconds at minimum) and because the tests that are added here were sufficient to reproduce the bug with source task shutdown. Author: Chris Egerton Reviewers: Nigel Liang , Tom Bentley , Randall Hauch --- .../connect/runtime/WorkerSourceTask.java | 44 +- .../integration/BlockingConnectorTest.java | 566 ++++++++++++++---- .../runtime/ErrorHandlingTaskTest.java | 6 - ...rrorHandlingTaskWithTopicCreationTest.java | 6 - 4 files changed, 472 insertions(+), 150 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java index 5874e3c26e25f..bc7df97476b30 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java @@ -102,9 +102,7 @@ class WorkerSourceTask extends WorkerTask { private CountDownLatch stopRequestedLatch; private Map taskConfig; - private boolean finishedStart = false; - private boolean startedShutdownBeforeStartCompleted = false; - private boolean stopped = false; + private boolean started = false; public WorkerSourceTask(ConnectorTaskId id, SourceTask task, @@ -166,8 +164,12 @@ public void initialize(TaskConfig taskConfig) { @Override protected void close() { - if (!shouldPause()) { - tryStop(); + if (started) { + try { + task.stop(); + } catch (Throwable t) { + log.warn("Could not stop task", t); + } } if (producer != null) { try { @@ -206,39 +208,21 @@ public void cancel() { public void stop() { super.stop(); stopRequestedLatch.countDown(); - synchronized (this) { - if (finishedStart) - tryStop(); - else - startedShutdownBeforeStartCompleted = true; - } - } - - private synchronized void tryStop() { - if (!stopped) { - try { - task.stop(); - stopped = true; - } catch (Throwable t) { - log.warn("Could not stop task", t); - } - } } @Override public void execute() { try { + // If we try to start the task at all by invoking initialize, then count this as + // "started" and expect a subsequent call to the task's stop() method + // to properly clean up any resources allocated by its initialize() or + // start() methods. If the task throws an exception during stop(), + // the worst thing that happens is another exception gets logged for an already- + // failed task + started = true; task.initialize(new WorkerSourceTaskContext(offsetReader, this, configState)); task.start(taskConfig); log.info("{} Source task finished initialization and start", this); - synchronized (this) { - if (startedShutdownBeforeStartCompleted) { - tryStop(); - return; - } - finishedStart = true; - } - while (!isStopping()) { if (shouldPause()) { onPause(); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/BlockingConnectorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/BlockingConnectorTest.java index 9a8e1fa76aeb7..abc9a938c6606 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/BlockingConnectorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/BlockingConnectorTest.java @@ -16,17 +16,26 @@ */ package org.apache.kafka.connect.integration; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.AbstractConfig; import org.apache.kafka.common.config.Config; import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.connect.connector.Connector; import org.apache.kafka.connect.connector.ConnectorContext; import org.apache.kafka.connect.connector.Task; import org.apache.kafka.connect.runtime.Worker; import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException; import org.apache.kafka.connect.runtime.rest.resources.ConnectorsResource; +import org.apache.kafka.connect.sink.SinkConnector; +import org.apache.kafka.connect.sink.SinkRecord; +import org.apache.kafka.connect.sink.SinkTask; +import org.apache.kafka.connect.sink.SinkTaskContext; import org.apache.kafka.connect.source.SourceConnector; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.source.SourceTaskContext; import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster; import org.apache.kafka.test.IntegrationTest; import org.junit.After; @@ -36,16 +45,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG; import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG; +import static org.apache.kafka.connect.runtime.SinkConnectorConfig.TOPICS_CONFIG; import static org.apache.kafka.test.TestUtils.waitForCondition; import static org.junit.Assert.assertThrows; @@ -67,6 +81,33 @@ public class BlockingConnectorTest { private static final long RECORD_TRANSFER_DURATION_MS = TimeUnit.SECONDS.toMillis(30); private static final long REST_REQUEST_TIMEOUT = Worker.CONNECTOR_GRACEFUL_SHUTDOWN_TIMEOUT_MS * 2; + private static final String CONNECTOR_INITIALIZE = "Connector::initialize"; + private static final String CONNECTOR_INITIALIZE_WITH_TASK_CONFIGS = "Connector::initializeWithTaskConfigs"; + private static final String CONNECTOR_START = "Connector::start"; + private static final String CONNECTOR_RECONFIGURE = "Connector::reconfigure"; + private static final String CONNECTOR_TASK_CLASS = "Connector::taskClass"; + private static final String CONNECTOR_TASK_CONFIGS = "Connector::taskConfigs"; + private static final String CONNECTOR_STOP = "Connector::stop"; + private static final String CONNECTOR_VALIDATE = "Connector::validate"; + private static final String CONNECTOR_CONFIG = "Connector::config"; + private static final String CONNECTOR_VERSION = "Connector::version"; + private static final String TASK_START = "Task::start"; + private static final String TASK_STOP = "Task::stop"; + private static final String TASK_VERSION = "Task::version"; + private static final String SINK_TASK_INITIALIZE = "SinkTask::initialize"; + private static final String SINK_TASK_PUT = "SinkTask::put"; + private static final String SINK_TASK_FLUSH = "SinkTask::flush"; + private static final String SINK_TASK_PRE_COMMIT = "SinkTask::preCommit"; + private static final String SINK_TASK_OPEN = "SinkTask::open"; + private static final String SINK_TASK_ON_PARTITIONS_ASSIGNED = "SinkTask::onPartitionsAssigned"; + private static final String SINK_TASK_CLOSE = "SinkTask::close"; + private static final String SINK_TASK_ON_PARTITIONS_REVOKED = "SinkTask::onPartitionsRevoked"; + private static final String SOURCE_TASK_INITIALIZE = "SourceTask::initialize"; + private static final String SOURCE_TASK_POLL = "SourceTask::poll"; + private static final String SOURCE_TASK_COMMIT = "SourceTask::commit"; + private static final String SOURCE_TASK_COMMIT_RECORD = "SourceTask::commitRecord"; + private static final String SOURCE_TASK_COMMIT_RECORD_WITH_METADATA = "SourceTask::commitRecordWithMetadata"; + private EmbeddedConnectCluster connect; private ConnectorHandle normalConnectorHandle; @@ -101,18 +142,18 @@ public void close() { // stop all Connect, Kafka and Zk threads. connect.stop(); ConnectorsResource.resetRequestTimeout(); - BlockingConnector.resetBlockLatch(); + Block.resetBlockLatch(); } @Test public void testBlockInConnectorValidate() throws Exception { log.info("Starting test testBlockInConnectorValidate"); - assertThrows(ConnectRestException.class, () -> createConnectorWithBlock(ValidateBlockingConnector.class)); + assertThrows(ConnectRestException.class, () -> createConnectorWithBlock(ValidateBlockingConnector.class, CONNECTOR_VALIDATE)); // Will NOT assert that connector has failed, since the request should fail before it's even created // Connector should already be blocked so this should return immediately, but check just to // make sure that it actually did block - BlockingConnector.waitForBlock(); + Block.waitForBlock(); createNormalConnector(); verifyNormalConnector(); @@ -121,12 +162,12 @@ public void testBlockInConnectorValidate() throws Exception { @Test public void testBlockInConnectorConfig() throws Exception { log.info("Starting test testBlockInConnectorConfig"); - assertThrows(ConnectRestException.class, () -> createConnectorWithBlock(ConfigBlockingConnector.class)); + assertThrows(ConnectRestException.class, () -> createConnectorWithBlock(ConfigBlockingConnector.class, CONNECTOR_CONFIG)); // Will NOT assert that connector has failed, since the request should fail before it's even created // Connector should already be blocked so this should return immediately, but check just to // make sure that it actually did block - BlockingConnector.waitForBlock(); + Block.waitForBlock(); createNormalConnector(); verifyNormalConnector(); @@ -135,8 +176,8 @@ public void testBlockInConnectorConfig() throws Exception { @Test public void testBlockInConnectorInitialize() throws Exception { log.info("Starting test testBlockInConnectorInitialize"); - createConnectorWithBlock(InitializeBlockingConnector.class); - BlockingConnector.waitForBlock(); + createConnectorWithBlock(InitializeBlockingConnector.class, CONNECTOR_INITIALIZE); + Block.waitForBlock(); createNormalConnector(); verifyNormalConnector(); @@ -145,8 +186,8 @@ public void testBlockInConnectorInitialize() throws Exception { @Test public void testBlockInConnectorStart() throws Exception { log.info("Starting test testBlockInConnectorStart"); - createConnectorWithBlock(BlockingConnector.START); - BlockingConnector.waitForBlock(); + createConnectorWithBlock(BlockingConnector.class, CONNECTOR_START); + Block.waitForBlock(); createNormalConnector(); verifyNormalConnector(); @@ -155,10 +196,54 @@ public void testBlockInConnectorStart() throws Exception { @Test public void testBlockInConnectorStop() throws Exception { log.info("Starting test testBlockInConnectorStop"); - createConnectorWithBlock(BlockingConnector.STOP); + createConnectorWithBlock(BlockingConnector.class, CONNECTOR_STOP); + waitForConnectorStart(BLOCKING_CONNECTOR_NAME); + connect.deleteConnector(BLOCKING_CONNECTOR_NAME); + Block.waitForBlock(); + + createNormalConnector(); + verifyNormalConnector(); + } + + @Test + public void testBlockInSourceTaskStart() throws Exception { + log.info("Starting test testBlockInSourceTaskStart"); + createConnectorWithBlock(BlockingSourceConnector.class, TASK_START); + Block.waitForBlock(); + + createNormalConnector(); + verifyNormalConnector(); + } + + @Test + public void testBlockInSourceTaskStop() throws Exception { + log.info("Starting test testBlockInSourceTaskStop"); + createConnectorWithBlock(BlockingSourceConnector.class, TASK_STOP); waitForConnectorStart(BLOCKING_CONNECTOR_NAME); connect.deleteConnector(BLOCKING_CONNECTOR_NAME); - BlockingConnector.waitForBlock(); + Block.waitForBlock(); + + createNormalConnector(); + verifyNormalConnector(); + } + + @Test + public void testBlockInSinkTaskStart() throws Exception { + log.info("Starting test testBlockInSinkTaskStart"); + createConnectorWithBlock(BlockingSinkConnector.class, TASK_START); + Block.waitForBlock(); + + createNormalConnector(); + verifyNormalConnector(); + } + + @Test + public void testBlockInSinkTaskStop() throws Exception { + log.info("Starting test testBlockInSinkTaskStop"); + createConnectorWithBlock(BlockingSinkConnector.class, TASK_STOP); + waitForConnectorStart(BLOCKING_CONNECTOR_NAME); + connect.deleteConnector(BLOCKING_CONNECTOR_NAME); + Block.waitForBlock(); createNormalConnector(); verifyNormalConnector(); @@ -167,50 +252,41 @@ public void testBlockInConnectorStop() throws Exception { @Test public void testWorkerRestartWithBlockInConnectorStart() throws Exception { log.info("Starting test testWorkerRestartWithBlockInConnectorStart"); - createConnectorWithBlock(BlockingConnector.START); + createConnectorWithBlock(BlockingConnector.class, CONNECTOR_START); // First instance of the connector should block on startup - BlockingConnector.waitForBlock(); + Block.waitForBlock(); createNormalConnector(); connect.removeWorker(); connect.addWorker(); // After stopping the only worker and restarting it, a new instance of the blocking // connector should be created and we can ensure that it blocks again - BlockingConnector.waitForBlock(); + Block.waitForBlock(); verifyNormalConnector(); } @Test public void testWorkerRestartWithBlockInConnectorStop() throws Exception { log.info("Starting test testWorkerRestartWithBlockInConnectorStop"); - createConnectorWithBlock(BlockingConnector.STOP); + createConnectorWithBlock(BlockingConnector.class, CONNECTOR_STOP); waitForConnectorStart(BLOCKING_CONNECTOR_NAME); createNormalConnector(); waitForConnectorStart(NORMAL_CONNECTOR_NAME); connect.removeWorker(); - BlockingConnector.waitForBlock(); + Block.waitForBlock(); connect.addWorker(); waitForConnectorStart(BLOCKING_CONNECTOR_NAME); verifyNormalConnector(); } - private void createConnectorWithBlock(String block) { - Map props = baseBlockingConnectorProps(); - props.put(BlockingConnector.BLOCK_CONFIG, block); - log.info("Creating connector with block during {}", block); - try { - connect.configureConnector(BLOCKING_CONNECTOR_NAME, props); - } catch (RuntimeException e) { - log.info("Failed to create connector", e); - throw e; - } - } - - private void createConnectorWithBlock(Class connectorClass) { - Map props = baseBlockingConnectorProps(); + private void createConnectorWithBlock(Class connectorClass, String block) { + Map props = new HashMap<>(); props.put(CONNECTOR_CLASS_CONFIG, connectorClass.getName()); - log.info("Creating blocking connector of type {}", connectorClass.getSimpleName()); + props.put(TASKS_MAX_CONFIG, "1"); + props.put(TOPICS_CONFIG, "t1"); // Required for sink connectors + props.put(Block.BLOCK_CONFIG, Objects.requireNonNull(block)); + log.info("Creating blocking connector of type {} with block in {}", connectorClass.getSimpleName(), block); try { connect.configureConnector(BLOCKING_CONNECTOR_NAME, props); } catch (RuntimeException e) { @@ -219,13 +295,6 @@ private void createConnectorWithBlock(Class connect } } - private Map baseBlockingConnectorProps() { - Map result = new HashMap<>(); - result.put(CONNECTOR_CLASS_CONFIG, BlockingConnector.class.getName()); - result.put(TASKS_MAX_CONFIG, "1"); - return result; - } - private void createNormalConnector() { connect.kafka().createTopic(TEST_TOPIC, 3); @@ -263,63 +332,43 @@ private void verifyNormalConnector() throws InterruptedException { normalConnectorHandle.awaitCommits(RECORD_TRANSFER_DURATION_MS); } - public static class BlockingConnector extends SourceConnector { - + private static class Block { private static CountDownLatch blockLatch; - private String block; + private final String block; public static final String BLOCK_CONFIG = "block"; - public static final String INITIALIZE = "initialize"; - public static final String INITIALIZE_WITH_TASK_CONFIGS = "initializeWithTaskConfigs"; - public static final String START = "start"; - public static final String RECONFIGURE = "reconfigure"; - public static final String TASK_CLASS = "taskClass"; - public static final String TASK_CONFIGS = "taskConfigs"; - public static final String STOP = "stop"; - public static final String VALIDATE = "validate"; - public static final String CONFIG = "config"; - public static final String VERSION = "version"; - - private static final ConfigDef CONFIG_DEF = new ConfigDef() - .define( - BLOCK_CONFIG, - ConfigDef.Type.STRING, - "", - ConfigDef.Importance.MEDIUM, - "Where to block indefinitely, e.g., 'start', 'initialize', 'taskConfigs', 'version'" - ); - - // No-args constructor required by the framework - public BlockingConnector() { - this(null); - } - - protected BlockingConnector(String block) { - this.block = block; - synchronized (BlockingConnector.class) { - if (blockLatch != null) { - blockLatch.countDown(); - } - blockLatch = new CountDownLatch(1); - } + private static ConfigDef config() { + return new ConfigDef() + .define( + BLOCK_CONFIG, + ConfigDef.Type.STRING, + "", + ConfigDef.Importance.MEDIUM, + "Where to block indefinitely, e.g., 'Connector::start', 'Connector::initialize', " + + "'Connector::taskConfigs', 'Task::version', 'SinkTask::put', 'SourceTask::poll'" + ); } public static void waitForBlock() throws InterruptedException { - synchronized (BlockingConnector.class) { + synchronized (Block.class) { if (blockLatch == null) { throw new IllegalArgumentException("No connector has been created yet"); } } - + log.debug("Waiting for connector to block"); blockLatch.await(); log.debug("Connector should now be blocked"); } + // Note that there is only ever at most one global block latch at a time, which makes tests that + // use blocks in multiple places impossible. If necessary, this can be addressed in the future by + // adding support for multiple block latches at a time, possibly identifiable by a connector/task + // ID, the location of the expected block, or both. public static void resetBlockLatch() { - synchronized (BlockingConnector.class) { + synchronized (Block.class) { if (blockLatch != null) { blockLatch.countDown(); blockLatch = null; @@ -327,81 +376,114 @@ public static void resetBlockLatch() { } } + public Block(Map props) { + this(new AbstractConfig(config(), props).getString(BLOCK_CONFIG)); + } + + public Block(String block) { + this.block = block; + synchronized (Block.class) { + if (blockLatch != null) { + blockLatch.countDown(); + } + blockLatch = new CountDownLatch(1); + } + } + + public Map taskConfig() { + return Collections.singletonMap(BLOCK_CONFIG, block); + } + + public void maybeBlockOn(String block) { + if (block.equals(this.block)) { + log.info("Will block on {}", block); + blockLatch.countDown(); + while (true) { + try { + Thread.sleep(Long.MAX_VALUE); + } catch (InterruptedException e) { + // No-op. Just keep blocking. + } + } + } else { + log.debug("Will not block on {}", block); + } + } + } + + // Used to test blocks in Connector (as opposed to Task) methods + public static class BlockingConnector extends SourceConnector { + + private Block block; + + // No-args constructor required by the framework + public BlockingConnector() { + this(null); + } + + protected BlockingConnector(String block) { + this.block = new Block(block); + } + @Override public void initialize(ConnectorContext ctx) { - maybeBlockOn(INITIALIZE); + block.maybeBlockOn(CONNECTOR_INITIALIZE); super.initialize(ctx); } @Override public void initialize(ConnectorContext ctx, List> taskConfigs) { - maybeBlockOn(INITIALIZE_WITH_TASK_CONFIGS); + block.maybeBlockOn(CONNECTOR_INITIALIZE_WITH_TASK_CONFIGS); super.initialize(ctx, taskConfigs); } @Override public void start(Map props) { - this.block = new AbstractConfig(CONFIG_DEF, props).getString(BLOCK_CONFIG); - maybeBlockOn(START); + this.block = new Block(props); + block.maybeBlockOn(CONNECTOR_START); } @Override public void reconfigure(Map props) { + block.maybeBlockOn(CONNECTOR_RECONFIGURE); super.reconfigure(props); - maybeBlockOn(RECONFIGURE); } @Override public Class taskClass() { - maybeBlockOn(TASK_CLASS); + block.maybeBlockOn(CONNECTOR_TASK_CLASS); return BlockingTask.class; } @Override public List> taskConfigs(int maxTasks) { - maybeBlockOn(TASK_CONFIGS); + block.maybeBlockOn(CONNECTOR_TASK_CONFIGS); return Collections.singletonList(Collections.emptyMap()); } @Override public void stop() { - maybeBlockOn(STOP); + block.maybeBlockOn(CONNECTOR_STOP); } @Override public Config validate(Map connectorConfigs) { - maybeBlockOn(VALIDATE); + block.maybeBlockOn(CONNECTOR_VALIDATE); return super.validate(connectorConfigs); } @Override public ConfigDef config() { - maybeBlockOn(CONFIG); - return CONFIG_DEF; + block.maybeBlockOn(CONNECTOR_CONFIG); + return Block.config(); } @Override public String version() { - maybeBlockOn(VERSION); + block.maybeBlockOn(CONNECTOR_VERSION); return "0.0.0"; } - protected void maybeBlockOn(String block) { - if (block.equals(this.block)) { - log.info("Will block on {}", block); - blockLatch.countDown(); - while (true) { - try { - Thread.sleep(Long.MAX_VALUE); - } catch (InterruptedException e) { - // No-op. Just keep blocking. - } - } - } else { - log.debug("Will not block on {}", block); - } - } - public static class BlockingTask extends SourceTask { @Override public void start(Map props) { @@ -426,19 +508,287 @@ public String version() { // Some methods are called before Connector::start, so we use this as a workaround public static class InitializeBlockingConnector extends BlockingConnector { public InitializeBlockingConnector() { - super(INITIALIZE); + super(CONNECTOR_INITIALIZE); } } public static class ConfigBlockingConnector extends BlockingConnector { public ConfigBlockingConnector() { - super(CONFIG); + super(CONNECTOR_CONFIG); } } public static class ValidateBlockingConnector extends BlockingConnector { public ValidateBlockingConnector() { - super(VALIDATE); + super(CONNECTOR_VALIDATE); + } + } + + // Used to test blocks in SourceTask methods + public static class BlockingSourceConnector extends SourceConnector { + + private Map props; + private final Class taskClass; + + // No-args constructor required by the framework + public BlockingSourceConnector() { + this(BlockingSourceTask.class); + } + + protected BlockingSourceConnector(Class taskClass) { + this.taskClass = taskClass; + } + + @Override + public void start(Map props) { + this.props = props; + } + + @Override + public Class taskClass() { + return taskClass; + } + + @Override + public List> taskConfigs(int maxTasks) { + return IntStream.range(0, maxTasks) + .mapToObj(i -> new HashMap<>(props)) + .collect(Collectors.toList()); + } + + @Override + public void stop() { + } + + @Override + public Config validate(Map connectorConfigs) { + return super.validate(connectorConfigs); + } + + @Override + public ConfigDef config() { + return Block.config(); + } + + @Override + public String version() { + return "0.0.0"; + } + + public static class BlockingSourceTask extends SourceTask { + private Block block; + + // No-args constructor required by the framework + public BlockingSourceTask() { + this(null); + } + + protected BlockingSourceTask(String block) { + this.block = new Block(block); + } + + @Override + public void start(Map props) { + this.block = new Block(props); + block.maybeBlockOn(TASK_START); + } + + @Override + public List poll() { + block.maybeBlockOn(SOURCE_TASK_POLL); + return null; + } + + @Override + public void stop() { + block.maybeBlockOn(TASK_STOP); + } + + @Override + public String version() { + block.maybeBlockOn(TASK_VERSION); + return "0.0.0"; + } + + @Override + public void initialize(SourceTaskContext context) { + block.maybeBlockOn(SOURCE_TASK_INITIALIZE); + super.initialize(context); + } + + @Override + public void commit() throws InterruptedException { + block.maybeBlockOn(SOURCE_TASK_COMMIT); + super.commit(); + } + + @Override + @SuppressWarnings("deprecation") + public void commitRecord(SourceRecord record) throws InterruptedException { + block.maybeBlockOn(SOURCE_TASK_COMMIT_RECORD); + super.commitRecord(record); + } + + @Override + public void commitRecord(SourceRecord record, RecordMetadata metadata) throws InterruptedException { + block.maybeBlockOn(SOURCE_TASK_COMMIT_RECORD_WITH_METADATA); + super.commitRecord(record, metadata); + } + } + } + + public static class TaskInitializeBlockingSourceConnector extends BlockingSourceConnector { + public TaskInitializeBlockingSourceConnector() { + super(InitializeBlockingSourceTask.class); + } + + public static class InitializeBlockingSourceTask extends BlockingSourceTask { + public InitializeBlockingSourceTask() { + super(SOURCE_TASK_INITIALIZE); + } + } + } + + // Used to test blocks in SinkTask methods + public static class BlockingSinkConnector extends SinkConnector { + + private Map props; + private final Class taskClass; + + // No-args constructor required by the framework + public BlockingSinkConnector() { + this(BlockingSinkTask.class); + } + + protected BlockingSinkConnector(Class taskClass) { + this.taskClass = taskClass; + } + + @Override + public void start(Map props) { + this.props = props; + } + + @Override + public Class taskClass() { + return taskClass; + } + + @Override + public List> taskConfigs(int maxTasks) { + return IntStream.rangeClosed(0, maxTasks) + .mapToObj(i -> new HashMap<>(props)) + .collect(Collectors.toList()); + } + + @Override + public void stop() { + } + + @Override + public Config validate(Map connectorConfigs) { + return super.validate(connectorConfigs); + } + + @Override + public ConfigDef config() { + return Block.config(); + } + + @Override + public String version() { + return "0.0.0"; + } + + public static class BlockingSinkTask extends SinkTask { + private Block block; + + // No-args constructor required by the framework + public BlockingSinkTask() { + this(null); + } + + protected BlockingSinkTask(String block) { + this.block = new Block(block); + } + + @Override + public void start(Map props) { + this.block = new Block(props); + block.maybeBlockOn(TASK_START); + } + + @Override + public void put(Collection records) { + block.maybeBlockOn(SINK_TASK_PUT); + } + + @Override + public void stop() { + block.maybeBlockOn(TASK_STOP); + } + + @Override + public String version() { + block.maybeBlockOn(TASK_VERSION); + return "0.0.0"; + } + + @Override + public void initialize(SinkTaskContext context) { + block.maybeBlockOn(SINK_TASK_INITIALIZE); + super.initialize(context); + } + + @Override + public void flush(Map currentOffsets) { + block.maybeBlockOn(SINK_TASK_FLUSH); + super.flush(currentOffsets); + } + + @Override + public Map preCommit(Map currentOffsets) { + block.maybeBlockOn(SINK_TASK_PRE_COMMIT); + return super.preCommit(currentOffsets); + } + + @Override + public void open(Collection partitions) { + block.maybeBlockOn(SINK_TASK_OPEN); + super.open(partitions); + } + + @Override + @SuppressWarnings("deprecation") + public void onPartitionsAssigned(Collection partitions) { + block.maybeBlockOn(SINK_TASK_ON_PARTITIONS_ASSIGNED); + super.onPartitionsAssigned(partitions); + } + + @Override + public void close(Collection partitions) { + block.maybeBlockOn(SINK_TASK_CLOSE); + super.close(partitions); + } + + @Override + @SuppressWarnings("deprecation") + public void onPartitionsRevoked(Collection partitions) { + block.maybeBlockOn(SINK_TASK_ON_PARTITIONS_REVOKED); + super.onPartitionsRevoked(partitions); + } + } + } + + public static class TaskInitializeBlockingSinkConnector extends BlockingSinkConnector { + public TaskInitializeBlockingSinkConnector() { + super(InitializeBlockingSinkTask.class); + } + + public static class InitializeBlockingSinkTask extends BlockingSinkTask { + public InitializeBlockingSinkTask() { + super(SINK_TASK_INITIALIZE); + } } } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java index c471b0381cbe3..70bbfc6590cd1 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java @@ -236,9 +236,6 @@ public void testSourceTasksCloseErrorReporters() { createSourceTask(initialState, retryWithToleranceOperator); - sourceTask.stop(); - PowerMock.expectLastCall(); - expectClose(); reporter.close(); @@ -263,9 +260,6 @@ public void testCloseErrorReportersExceptionPropagation() { createSourceTask(initialState, retryWithToleranceOperator); - sourceTask.stop(); - PowerMock.expectLastCall(); - expectClose(); // Even though the reporters throw exceptions, they should both still be closed. diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskWithTopicCreationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskWithTopicCreationTest.java index aba64457dbc7c..9c115ac517821 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskWithTopicCreationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskWithTopicCreationTest.java @@ -250,9 +250,6 @@ public void testSourceTasksCloseErrorReporters() { createSourceTask(initialState, retryWithToleranceOperator); - sourceTask.stop(); - PowerMock.expectLastCall(); - expectClose(); reporter.close(); @@ -277,9 +274,6 @@ public void testCloseErrorReportersExceptionPropagation() { createSourceTask(initialState, retryWithToleranceOperator); - sourceTask.stop(); - PowerMock.expectLastCall(); - expectClose(); // Even though the reporters throw exceptions, they should both still be closed. From 4e9c7fc8a5db6e59e43a67586867c3b1c9fbe567 Mon Sep 17 00:00:00 2001 From: Rohit Deshpande Date: Fri, 4 Dec 2020 10:51:12 -0800 Subject: [PATCH 03/14] KAFKA-10629: TopologyTestDriver should not require a Properties argument (#9660) Implements KIP-680. Reviewers: Chia-Ping Tsai , Matthias J. Sax --- .../kafka/streams/StreamsBuilderTest.java | 2 +- .../apache/kafka/streams/TopologyTest.java | 7 +-- .../kstream/internals/AbstractStreamTest.java | 10 ++-- .../internals/KStreamTransformTest.java | 1 - .../internals/ProcessorNodeTest.java | 4 +- .../kafka/streams/TopologyTestDriver.java | 26 ++++++++- .../apache/kafka/streams/TestTopicsTest.java | 3 +- .../kafka/streams/TopologyTestDriverTest.java | 54 +++++++++---------- 8 files changed, 60 insertions(+), 47 deletions(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java index e24ad1d427d33..8a9c165c5cb3a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java @@ -112,7 +112,7 @@ public void process(final Record record) { } } ); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), new Properties())) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build())) { final TestInputTopic inputTopic = driver.createInputTopic("topic", new StringSerializer(), new StringSerializer()); inputTopic.pipeInput("hey", "there"); diff --git a/streams/src/test/java/org/apache/kafka/streams/TopologyTest.java b/streams/src/test/java/org/apache/kafka/streams/TopologyTest.java index 3fe80f90f4b1e..f42e0b9f14800 100644 --- a/streams/src/test/java/org/apache/kafka/streams/TopologyTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/TopologyTest.java @@ -37,7 +37,6 @@ import org.apache.kafka.test.MockApiProcessorSupplier; import org.apache.kafka.test.MockKeyValueStore; import org.apache.kafka.test.MockProcessorSupplier; -import org.apache.kafka.test.TestUtils; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; @@ -48,9 +47,6 @@ import java.util.Set; import java.util.regex.Pattern; -import static org.apache.kafka.common.utils.Utils.mkEntry; -import static org.apache.kafka.common.utils.Utils.mkMap; -import static org.apache.kafka.common.utils.Utils.mkProperties; import static java.time.Duration.ofMillis; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; @@ -353,8 +349,7 @@ public void shouldThrowOnUnassignedStateStoreAccess() { .addProcessor(badNodeName, new LocalMockProcessorSupplier(), sourceNodeName); try { - new TopologyTestDriver(topology, mkProperties( - mkMap(mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath())))); + new TopologyTestDriver(topology); fail("Should have thrown StreamsException"); } catch (final StreamsException e) { final String error = e.toString(); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java index 8eb1b4dc58e4f..4bc7779f5ed6a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java @@ -22,7 +22,6 @@ import org.apache.kafka.test.NoopValueTransformer; import org.apache.kafka.test.NoopValueTransformerWithKey; import org.apache.kafka.streams.StreamsBuilder; -import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.TestInputTopic; import org.apache.kafka.streams.TopologyTestDriver; import org.apache.kafka.streams.kstream.Consumed; @@ -35,8 +34,8 @@ import org.apache.kafka.streams.processor.Processor; import org.apache.kafka.streams.processor.ProcessorSupplier; import org.apache.kafka.test.MockProcessorSupplier; -import org.apache.kafka.test.TestUtils; import org.junit.Test; + import java.util.Random; import static org.easymock.EasyMock.createMock; @@ -44,9 +43,6 @@ import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertTrue; -import static org.apache.kafka.common.utils.Utils.mkEntry; -import static org.apache.kafka.common.utils.Utils.mkMap; -import static org.apache.kafka.common.utils.Utils.mkProperties; public class AbstractStreamTest { @@ -86,8 +82,8 @@ public void testShouldBeExtensible() { stream.randomFilter().process(supplier); - final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), mkProperties( - mkMap(mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath())))); + final TopologyTestDriver driver = new TopologyTestDriver(builder.build()); + final TestInputTopic inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer()); for (final int expectedKey : expectedKeys) { inputTopic.pipeInput(expectedKey, "V" + expectedKey); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java index 2b2b9ba947153..20b1d3a896676 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java @@ -79,7 +79,6 @@ public void close() { } try (final TopologyTestDriver driver = new TopologyTestDriver( builder.build(), - new Properties(), Instant.ofEpochMilli(0L))) { final TestInputTopic inputTopic = driver.createInputTopic(TOPIC_NAME, new IntegerSerializer(), new IntegerSerializer()); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java index 398b3a45c0417..ed445a747ee91 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java @@ -36,7 +36,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Properties; + import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ROLLUP_VALUE; import static org.hamcrest.CoreMatchers.containsString; @@ -171,7 +171,7 @@ public void testTopologyLevelClassCastException() { }); final Topology topology = builder.build(); - final TopologyTestDriver testDriver = new TopologyTestDriver(topology, new Properties()); + final TopologyTestDriver testDriver = new TopologyTestDriver(topology); final TestInputTopic topic = testDriver.createInputTopic("streams-plaintext-input", new StringSerializer(), new StringSerializer()); final StreamsException se = assertThrows(StreamsException.class, () -> topic.pipeInput("a-key", "a value")); diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java index 876746a0f2998..8a9007651b8a8 100644 --- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java +++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java @@ -99,6 +99,7 @@ import java.util.Properties; import java.util.Queue; import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; @@ -242,6 +243,16 @@ public void onBatchRestored(final TopicPartition topicPartition, final String st public void onRestoreEnd(final TopicPartition topicPartition, final String storeName, final long totalRestored) {} }; + /** + * Create a new test diver instance. + * Default test properties are used to initialize the driver instance + * + * @param topology the topology to be tested + */ + public TopologyTestDriver(final Topology topology) { + this(topology, new Properties()); + } + /** * Create a new test diver instance. * Initialized the internally mocked wall-clock time with {@link System#currentTimeMillis() current system time}. @@ -254,6 +265,18 @@ public TopologyTestDriver(final Topology topology, this(topology, config, null); } + /** + * Create a new test diver instance. + * + * @param topology the topology to be tested + * @param initialWallClockTimeMs the initial value of internally mocked wall-clock time + */ + public TopologyTestDriver(final Topology topology, + final Instant initialWallClockTimeMs) { + this(topology, new Properties(), initialWallClockTimeMs); + } + + /** * Create a new test diver instance. * @@ -299,7 +322,8 @@ private TopologyTestDriver(final InternalTopologyBuilder builder, final Properties configCopy = new Properties(); configCopy.putAll(config); configCopy.putIfAbsent(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy-bootstrap-host:0"); - configCopy.putIfAbsent(StreamsConfig.APPLICATION_ID_CONFIG, "dummy-topology-test-driver-app-id"); + // provide randomized dummy app-id if it's not specified + configCopy.putIfAbsent(StreamsConfig.APPLICATION_ID_CONFIG, "dummy-topology-test-driver-app-id-" + ThreadLocalRandom.current().nextInt()); final StreamsConfig streamsConfig = new ClientUtils.QuietStreamsConfig(configCopy); logIfTaskIdleEnabled(streamsConfig); diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java index 463bd3fac1ca0..6d24f1f45bea4 100644 --- a/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java +++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java @@ -42,7 +42,6 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import java.util.Properties; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; @@ -75,7 +74,7 @@ public void setup() { final KStream source = builder.stream(INPUT_TOPIC_MAP, Consumed.with(longSerde, stringSerde)); final KStream mapped = source.map((key, value) -> new KeyValue<>(value, key)); mapped.to(OUTPUT_TOPIC_MAP, Produced.with(stringSerde, longSerde)); - testDriver = new TopologyTestDriver(builder.build(), new Properties()); + testDriver = new TopologyTestDriver(builder.build()); } @After diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java index 6dc3a3705d375..6cdb65840462c 100644 --- a/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java +++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java @@ -471,7 +471,7 @@ public void shouldCloseProcessor() { @Test public void shouldThrowForUnknownTopic() { - testDriver = new TopologyTestDriver(new Topology(), config); + testDriver = new TopologyTestDriver(new Topology()); assertThrows( IllegalArgumentException.class, () -> testDriver.pipeRecord( @@ -485,7 +485,7 @@ public void shouldThrowForUnknownTopic() { @Test public void shouldThrowForMissingTime() { - testDriver = new TopologyTestDriver(new Topology(), config); + testDriver = new TopologyTestDriver(new Topology()); assertThrows( IllegalStateException.class, () -> testDriver.pipeRecord( @@ -506,7 +506,7 @@ public void shouldThrowForUnknownTopicDeprecated() { new ByteArraySerializer(), new ByteArraySerializer()); - testDriver = new TopologyTestDriver(new Topology(), config); + testDriver = new TopologyTestDriver(new Topology()); try { testDriver.pipeInput(consumerRecordFactory.create((byte[]) null)); fail("Should have throw IllegalArgumentException"); @@ -517,7 +517,7 @@ public void shouldThrowForUnknownTopicDeprecated() { @Test public void shouldThrowNoSuchElementExceptionForUnusedOutputTopicWithDynamicRouting() { - testDriver = new TopologyTestDriver(setupSourceSinkTopology(), config); + testDriver = new TopologyTestDriver(setupSourceSinkTopology()); final TestOutputTopic outputTopic = new TestOutputTopic<>( testDriver, "unused-topic", @@ -531,7 +531,7 @@ public void shouldThrowNoSuchElementExceptionForUnusedOutputTopicWithDynamicRout @Test public void shouldCaptureSinkTopicNamesIfWrittenInto() { - testDriver = new TopologyTestDriver(setupSourceSinkTopology(), config); + testDriver = new TopologyTestDriver(setupSourceSinkTopology()); assertThat(testDriver.producedTopicNames(), is(Collections.emptySet())); @@ -577,7 +577,7 @@ public void shouldCaptureGlobalTopicNameIfWrittenInto() { builder.globalTable(SOURCE_TOPIC_1, Materialized.as("globalTable")); builder.stream(SOURCE_TOPIC_2).to(SOURCE_TOPIC_1); - testDriver = new TopologyTestDriver(builder.build(), config); + testDriver = new TopologyTestDriver(builder.build()); assertThat(testDriver.producedTopicNames(), is(Collections.emptySet())); @@ -590,7 +590,7 @@ public void shouldCaptureGlobalTopicNameIfWrittenInto() { @Test public void shouldProcessRecordForTopic() { - testDriver = new TopologyTestDriver(setupSourceSinkTopology(), config); + testDriver = new TopologyTestDriver(setupSourceSinkTopology()); pipeRecord(SOURCE_TOPIC_1, testRecord1); final ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1); @@ -602,7 +602,7 @@ public void shouldProcessRecordForTopic() { @Test public void shouldSetRecordMetadata() { - testDriver = new TopologyTestDriver(setupSingleProcessorTopology(), config); + testDriver = new TopologyTestDriver(setupSingleProcessorTopology()); pipeRecord(SOURCE_TOPIC_1, testRecord1); @@ -624,7 +624,7 @@ private void pipeRecord(final String topic, final TestRecord rec //Test not migrated to non-deprecated methods, topic handling not based on record any more @Test public void shouldSendRecordViaCorrectSourceTopicDeprecated() { - testDriver = new TopologyTestDriver(setupMultipleSourceTopology(SOURCE_TOPIC_1, SOURCE_TOPIC_2), config); + testDriver = new TopologyTestDriver(setupMultipleSourceTopology(SOURCE_TOPIC_1, SOURCE_TOPIC_2)); final List processedRecords1 = mockProcessors.get(0).processedRecords; final List processedRecords2 = mockProcessors.get(1).processedRecords; @@ -677,7 +677,7 @@ public void shouldUseSourceSpecificDeserializersDeprecated() { }, processor); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); final org.apache.kafka.streams.test.ConsumerRecordFactory source1Factory = new org.apache.kafka.streams.test.ConsumerRecordFactory<>( @@ -739,7 +739,7 @@ public void shouldUseSourceSpecificDeserializers() { }, processor); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); final Long source1Key = 42L; final String source1Value = "anyString"; @@ -772,7 +772,7 @@ public void shouldUseSourceSpecificDeserializers() { @Test public void shouldPassRecordHeadersIntoSerializersAndDeserializers() { - testDriver = new TopologyTestDriver(setupSourceSinkTopology(), config); + testDriver = new TopologyTestDriver(setupSourceSinkTopology()); final AtomicBoolean passedHeadersToKeySerializer = new AtomicBoolean(false); final AtomicBoolean passedHeadersToValueSerializer = new AtomicBoolean(false); @@ -832,7 +832,7 @@ public void shouldUseSinkSpecificSerializers() { topology.addSink("sink-1", SINK_TOPIC_1, Serdes.Long().serializer(), Serdes.String().serializer(), sourceName1); topology.addSink("sink-2", SINK_TOPIC_2, Serdes.Integer().serializer(), Serdes.Double().serializer(), sourceName2); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); final Long source1Key = 42L; final String source1Value = "anyString"; @@ -867,7 +867,7 @@ public void shouldUseSinkSpecificSerializers() { //Test not migrated to non-deprecated methods, List processing now in TestInputTopic @Test public void shouldProcessConsumerRecordList() { - testDriver = new TopologyTestDriver(setupMultipleSourceTopology(SOURCE_TOPIC_1, SOURCE_TOPIC_2), config); + testDriver = new TopologyTestDriver(setupMultipleSourceTopology(SOURCE_TOPIC_1, SOURCE_TOPIC_2)); final List processedRecords1 = mockProcessors.get(0).processedRecords; final List processedRecords2 = mockProcessors.get(1).processedRecords; @@ -892,7 +892,7 @@ record = processedRecords2.get(0); @Test public void shouldForwardRecordsFromSubtopologyToSubtopology() { - testDriver = new TopologyTestDriver(setupTopologyWithTwoSubtopologies(), config); + testDriver = new TopologyTestDriver(setupTopologyWithTwoSubtopologies()); pipeRecord(SOURCE_TOPIC_1, testRecord1); @@ -909,7 +909,7 @@ public void shouldForwardRecordsFromSubtopologyToSubtopology() { @Test public void shouldPopulateGlobalStore() { - testDriver = new TopologyTestDriver(setupGlobalStoreTopology(SOURCE_TOPIC_1), config); + testDriver = new TopologyTestDriver(setupGlobalStoreTopology(SOURCE_TOPIC_1)); final KeyValueStore globalStore = testDriver.getKeyValueStore(SOURCE_TOPIC_1 + "-globalStore"); Assert.assertNotNull(globalStore); @@ -924,8 +924,8 @@ public void shouldPopulateGlobalStore() { public void shouldPunctuateOnStreamsTime() { final MockPunctuator mockPunctuator = new MockPunctuator(); testDriver = new TopologyTestDriver( - setupSingleProcessorTopology(10L, PunctuationType.STREAM_TIME, mockPunctuator), - config); + setupSingleProcessorTopology(10L, PunctuationType.STREAM_TIME, mockPunctuator) + ); final List expectedPunctuations = new LinkedList<>(); @@ -1052,7 +1052,7 @@ public void shouldReturnAllStores() { "globalProcessorName", voidProcessorSupplier); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); final Set expectedStoreNames = new HashSet<>(); expectedStoreNames.add("store"); @@ -1096,7 +1096,7 @@ private void shouldReturnCorrectStoreTypeOnly(final boolean persistent) { globalTimestampedKeyValueStoreName); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); // verify state stores assertNotNull(testDriver.getKeyValueStore(keyValueStoreName)); @@ -1175,7 +1175,7 @@ private void shouldThrowIfBuiltInStoreIsAccessedWithUntypedMethod(final boolean globalTimestampedKeyValueStoreName); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); { final IllegalArgumentException e = assertThrows( @@ -1367,7 +1367,7 @@ public void shouldReturnAllStoresNames() { "globalProcessorName", voidProcessorSupplier); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); final Set expectedStoreNames = new HashSet<>(); expectedStoreNames.add("store"); @@ -1515,7 +1515,7 @@ public void shouldAllowPrePopulatingStatesStoresWithCachingEnabled() { Serdes.Long()).withCachingEnabled(), // intentionally turn on caching to achieve better test coverage "aggregator"); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); store = testDriver.getKeyValueStore("aggStore"); store.put("a", 21L); @@ -1655,7 +1655,7 @@ public void shouldProcessFromSourceThatMatchPattern() { topology.addSource(sourceName, pattern2Source1); topology.addSink("sink", SINK_TOPIC_1, sourceName); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); pipeRecord(SOURCE_TOPIC_1, testRecord1); final ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1); @@ -1674,7 +1674,7 @@ public void shouldThrowPatternNotValidForTopicNameException() { topology.addSource(sourceName, pattern2Source1); topology.addSink("sink", SINK_TOPIC_1, sourceName); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); try { pipeRecord(SOURCE_TOPIC_1, testRecord1); } catch (final TopologyException exception) { @@ -1737,7 +1737,7 @@ public void process(final Record record) { topology.addSink("recursiveSink", "input", new StringSerializer(), new StringSerializer(), "recursiveProcessor"); topology.addSink("sink", "output", new StringSerializer(), new StringSerializer(), "recursiveProcessor"); - try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(topology, new Properties())) { + try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(topology)) { final TestInputTopic in = topologyTestDriver.createInputTopic("input", new StringSerializer(), new StringSerializer()); final TestOutputTopic out = topologyTestDriver.createOutputTopic("output", new StringDeserializer(), new StringDeserializer()); @@ -1809,7 +1809,7 @@ public void process(final Record record) { topology.addSink("sink", "output", new StringSerializer(), new StringSerializer(), "recursiveProcessor"); topology.addSink("globalSink", "global-topic", new StringSerializer(), new StringSerializer(), "recursiveProcessor"); - try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(topology, new Properties())) { + try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(topology)) { final TestInputTopic in = topologyTestDriver.createInputTopic("input", new StringSerializer(), new StringSerializer()); final TestOutputTopic globalTopic = topologyTestDriver.createOutputTopic("global-topic", new StringDeserializer(), new StringDeserializer()); From b9640a71c42ad117ed418209f87dc2962173469d Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Sat, 5 Dec 2020 02:55:42 +0800 Subject: [PATCH 04/14] HOTFIX: fix failed build caused by StreamThreadTest (#9691) Reviewer: Matthias J. Sax --- .../kafka/streams/processor/internals/StreamThreadTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java index 59bb66f9f210d..157f9e5efcd58 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java @@ -2518,7 +2518,8 @@ public void runAndVerifyFailedStreamThreadRecording(final boolean shouldFail) { new AtomicInteger(), new AtomicLong(Long.MAX_VALUE), null, - e -> { } + e -> { }, + null ) { @Override void runOnce() { From 9ece7fe372230e848920d843af2d0d5c2fbaac4d Mon Sep 17 00:00:00 2001 From: Walker Carlson <18128741+wcarlson5@users.noreply.github.com> Date: Fri, 4 Dec 2020 11:21:03 -0800 Subject: [PATCH 05/14] KAFKA-10500: Allow people to add new StreamThread at runtime (#9615) Part of KIP-663. Reviewers: Bruno Cadonna , A. Sophie Blee-Goldman , Matthias J. Sax --- .../apache/kafka/streams/KafkaStreams.java | 146 +++++++++++++----- .../streams/state/internals/ThreadCache.java | 3 + .../kafka/streams/KafkaStreamsTest.java | 44 ++++++ .../AdjustStreamThreadCountTest.java | 111 +++++++++++++ 4 files changed, 262 insertions(+), 42 deletions(-) create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/AdjustStreamThreadCountTest.java diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java index 44ce3079ac457..4fb2b2fbefea8 100644 --- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java +++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java @@ -84,6 +84,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.TreeMap; @@ -161,11 +162,20 @@ public class KafkaStreams implements AutoCloseable { private final ProcessorTopology taskTopology; private final ProcessorTopology globalTaskTopology; private final long totalCacheSize; + private final StreamStateListener streamStateListener; + private final StateRestoreListener delegatingStateRestoreListener; + private final Map threadState; + private final ArrayList storeProviders; + private final UUID processId; + private final KafkaClientSupplier clientSupplier; + private final InternalTopologyBuilder internalTopologyBuilder; GlobalStreamThread globalStreamThread; private KafkaStreams.StateListener stateListener; private StateRestoreListener globalStateRestoreListener; private boolean oldHandler; + private java.util.function.Consumer streamsUncaughtExceptionHandler; + private final Object changeThreadCount = new Object(); // container states /** @@ -397,6 +407,7 @@ public void setUncaughtExceptionHandler(final StreamsUncaughtExceptionHandler st final Consumer handler = exception -> handleStreamsUncaughtException(exception, streamsUncaughtExceptionHandler); synchronized (stateLock) { if (state == State.CREATED) { + this.streamsUncaughtExceptionHandler = handler; Objects.requireNonNull(streamsUncaughtExceptionHandler); for (final StreamThread thread : threads) { thread.setStreamsUncaughtExceptionHandler(handler); @@ -755,7 +766,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, this.config = config; this.time = time; // The application ID is a required config and hence should always have value - final UUID processId = UUID.randomUUID(); + processId = UUID.randomUUID(); final String userClientId = config.getString(StreamsConfig.CLIENT_ID_CONFIG); final String applicationId = config.getString(StreamsConfig.APPLICATION_ID_CONFIG); if (userClientId.length() <= 0) { @@ -765,7 +776,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, } final LogContext logContext = new LogContext(String.format("stream-client [%s] ", clientId)); this.log = logContext.logger(getClass()); - + this.clientSupplier = clientSupplier; final MetricConfig metricConfig = new MetricConfig() .samples(config.getInt(StreamsConfig.METRICS_NUM_SAMPLES_CONFIG)) .recordLevel(Sensor.RecordingLevel.forName(config.getString(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG))) @@ -792,7 +803,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, ClientMetrics.addStateMetric(streamsMetrics, (metricsConfig, now) -> state); log.info("Kafka Streams version: {}", ClientMetrics.version()); log.info("Kafka Streams commit ID: {}", ClientMetrics.commitId()); - + this.internalTopologyBuilder = internalTopologyBuilder; // re-write the physical topology according to the config internalTopologyBuilder.rewriteTopology(config); @@ -820,18 +831,18 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, throw new TopologyException("Topology has no stream threads and no global threads, " + "must subscribe to at least one source topic or global table."); } - + oldHandler = false; totalCacheSize = config.getLong(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG); final long cacheSizePerThread = getCacheSizePerThread(numStreamThreads); final boolean hasPersistentStores = taskTopology.hasPersistentLocalStore() || (hasGlobalTopology && globalTaskTopology.hasPersistentGlobalStore()); - + streamsUncaughtExceptionHandler = this::defaultStreamsUncaughtExceptionHandler; try { stateDirectory = new StateDirectory(config, time, hasPersistentStores); } catch (final ProcessorStateException fatal) { throw new StreamsException(fatal); } - final StateRestoreListener delegatingStateRestoreListener = new DelegatingStateRestoreListener(); + delegatingStateRestoreListener = new DelegatingStateRestoreListener(); GlobalStreamThread.State globalThreadState = null; if (hasGlobalTopology) { final String globalThreadId = clientId + "-GlobalStreamThread"; @@ -845,7 +856,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, time, globalThreadId, delegatingStateRestoreListener, - this::defaultStreamsUncaughtExceptionHandler + streamsUncaughtExceptionHandler ); globalThreadState = globalStreamThread.state(); } @@ -853,59 +864,110 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, // use client id instead of thread client id since this admin client may be shared among threads adminClient = clientSupplier.getAdmin(config.getAdminConfigs(ClientUtils.getSharedAdminClientId(clientId))); - final Map threadState = new HashMap<>(numStreamThreads); - final ArrayList storeProviders = new ArrayList<>(); - for (int i = 0; i < numStreamThreads; i++) { - final StreamThread streamThread = StreamThread.create( - internalTopologyBuilder, - config, - clientSupplier, - adminClient, - processId, - clientId, - streamsMetrics, - time, - streamsMetadataState, - cacheSizePerThread, - stateDirectory, - delegatingStateRestoreListener, - i + 1, - KafkaStreams.this::closeToError, - this::defaultStreamsUncaughtExceptionHandler - ); - threads.add(streamThread); - threadState.put(streamThread.getId(), streamThread.state()); - storeProviders.add(new StreamThreadStateStoreProvider(streamThread)); - } - - ClientMetrics.addNumAliveStreamThreadMetric(streamsMetrics, (metricsConfig, now) -> - Math.toIntExact(threads.stream().filter(thread -> thread.state().isAlive()).count())); - - final StreamStateListener streamStateListener = new StreamStateListener(threadState, globalThreadState); + threadState = new HashMap<>(numStreamThreads); + storeProviders = new ArrayList<>(); + streamStateListener = new StreamStateListener(threadState, globalThreadState); if (hasGlobalTopology) { globalStreamThread.setStateListener(streamStateListener); } - for (final StreamThread thread : threads) { - thread.setStateListener(streamStateListener); + for (int i = 1; i <= numStreamThreads; i++) { + createStreamThread(cacheSizePerThread, i); } + ClientMetrics.addNumAliveStreamThreadMetric(streamsMetrics, (metricsConfig, now) -> + Math.toIntExact(threads.stream().filter(thread -> thread.state().isAlive()).count())); + final GlobalStateStoreProvider globalStateStoreProvider = new GlobalStateStoreProvider(internalTopologyBuilder.globalStateStores()); queryableStoreProvider = new QueryableStoreProvider(storeProviders, globalStateStoreProvider); stateDirCleaner = setupStateDirCleaner(); - oldHandler = false; maybeWarnAboutCodeInRocksDBConfigSetter(log, config); rocksDBMetricsRecordingService = maybeCreateRocksDBMetricsRecordingService(clientId, config); } + private StreamThread createStreamThread(final long cacheSizePerThread, final int threadIdx) { + final StreamThread streamThread = StreamThread.create( + internalTopologyBuilder, + config, + clientSupplier, + adminClient, + processId, + clientId, + streamsMetrics, + time, + streamsMetadataState, + cacheSizePerThread, + stateDirectory, + delegatingStateRestoreListener, + threadIdx, + KafkaStreams.this::closeToError, + streamsUncaughtExceptionHandler + ); + streamThread.setStateListener(streamStateListener); + threads.add(streamThread); + threadState.put(streamThread.getId(), streamThread.state()); + storeProviders.add(new StreamThreadStateStoreProvider(streamThread)); + return streamThread; + } + + /** + * Adds and starts a stream thread in addition to the stream threads that are already running in this + * Kafka Streams client. + *

+ * Since the number of stream threads increases, the sizes of the caches in the new stream thread + * and the existing stream threads are adapted so that the sum of the cache sizes over all stream + * threads does not exceed the total cache size specified in configuration + * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG}. + *

+ * Stream threads can only be added if this Kafka Streams client is in state RUNNING or REBALANCING. + * + * @return name of the added stream thread or empty if a new stream thread could not be added + */ + public Optional addStreamThread() { + synchronized (changeThreadCount) { + if (isRunningOrRebalancing()) { + final int threadIdx = getNextThreadIndex(); + final long cacheSizePerThread = getCacheSizePerThread(threads.size() + 1); + resizeThreadCache(cacheSizePerThread); + final StreamThread streamThread = createStreamThread(cacheSizePerThread, threadIdx); + synchronized (stateLock) { + if (isRunningOrRebalancing()) { + streamThread.start(); + return Optional.of(streamThread.getName()); + } else { + streamThread.shutdown(); + threads.remove(streamThread); + resizeThreadCache(getCacheSizePerThread(threads.size())); + return Optional.empty(); + } + } + } + } + return Optional.empty(); + } + + private int getNextThreadIndex() { + final HashSet names = new HashSet<>(); + for (final StreamThread streamThread: threads) { + names.add(streamThread.getName()); + } + final String baseName = clientId + "-StreamThread-"; + for (int i = 1; i <= threads.size(); i++) { + final String name = baseName + i; + if (!names.contains(name)) { + return i; + } + } + return threads.size() + 1; + } + private long getCacheSizePerThread(final int numStreamThreads) { return totalCacheSize / (numStreamThreads + ((globalTaskTopology != null) ? 1 : 0)); } - private void resizeThreadCache(final int numStreamThreads) { - final long cacheSizePreThread = getCacheSizePerThread(numStreamThreads); + private void resizeThreadCache(final long cacheSizePerThread) { for (final StreamThread streamThread: threads) { - streamThread.resizeCache(cacheSizePreThread); + streamThread.resizeCache(cacheSizePerThread); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java index af3d767011329..9e99d3b26ecf0 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java @@ -76,6 +76,9 @@ public void resize(final long newCacheSizeBytes) { final boolean shrink = newCacheSizeBytes < maxCacheSizeBytes; maxCacheSizeBytes = newCacheSizeBytes; if (shrink) { + if (caches.values().isEmpty()) { + return; + } final CircularIterator circularIterator = new CircularIterator<>(caches.values()); while (sizeBytes() > maxCacheSizeBytes) { final NamedCache cache = circularIterator.next(); diff --git a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java index b2e397191a05c..20de1047b57e8 100644 --- a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java @@ -78,6 +78,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Properties; import java.util.UUID; import java.util.concurrent.Executors; @@ -312,6 +313,9 @@ private void prepareStreamThread(final StreamThread thread, final boolean termin StreamThread.State.PARTITIONS_ASSIGNED); return null; }).anyTimes(); + thread.resizeCache(EasyMock.anyLong()); + EasyMock.expectLastCall().anyTimes(); + EasyMock.expect(thread.getName()).andStubReturn("newThread"); thread.shutdown(); EasyMock.expectLastCall().andAnswer(() -> { supplier.consumer.close(); @@ -588,6 +592,46 @@ public void testCloseIsIdempotent() { closeCount, MockMetricsReporter.CLOSE_COUNT.get()); } + @Test + public void shouldAddThreadWhenRunning() throws InterruptedException { + props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 1); + final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time); + streams.start(); + final int oldSize = streams.threads.size(); + TestUtils.waitForCondition(() -> streams.state() == KafkaStreams.State.RUNNING, 15L, "wait until running"); + assertThat(streams.addStreamThread(), equalTo(Optional.of("newThread"))); + assertThat(streams.threads.size(), equalTo(oldSize + 1)); + } + + @Test + public void shouldNotAddThreadWhenCreated() { + final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time); + final int oldSize = streams.threads.size(); + assertThat(streams.addStreamThread(), equalTo(Optional.empty())); + assertThat(streams.threads.size(), equalTo(oldSize)); + } + + @Test + public void shouldNotAddThreadWhenClosed() { + final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time); + final int oldSize = streams.threads.size(); + streams.close(); + assertThat(streams.addStreamThread(), equalTo(Optional.empty())); + assertThat(streams.threads.size(), equalTo(oldSize)); + } + + @Test + public void shouldNotAddThreadWhenError() { + final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time); + final int oldSize = streams.threads.size(); + streams.start(); + streamThreadOne.shutdown(); + streamThreadTwo.shutdown(); + assertThat(streams.addStreamThread(), equalTo(Optional.empty())); + assertThat(streams.threads.size(), equalTo(oldSize)); + } + + @Test public void testCannotStartOnceClosed() { final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/AdjustStreamThreadCountTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/AdjustStreamThreadCountTest.java new file mode 100644 index 0000000000000..649fc86b82511 --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/integration/AdjustStreamThreadCountTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.integration; + +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; +import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; +import org.apache.kafka.streams.processor.ThreadMetadata; +import org.apache.kafka.test.IntegrationTest; +import org.apache.kafka.test.StreamsTestUtils; +import org.apache.kafka.test.TestUtils; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +import java.io.IOException; +import java.util.Optional; +import java.util.Properties; + +import static org.apache.kafka.common.utils.Utils.mkEntry; +import static org.apache.kafka.common.utils.Utils.mkMap; +import static org.apache.kafka.common.utils.Utils.mkObjectProperties; +import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.purgeLocalStreamsState; +import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.safeUniqueTestName; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +@Category(IntegrationTest.class) +public class AdjustStreamThreadCountTest { + + @ClassRule + public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(1); + + @Rule + public TestName testName = new TestName(); + + private static String inputTopic; + private static StreamsBuilder builder; + private static Properties properties; + private static String appId = ""; + + @Before + public void setup() { + final String testId = safeUniqueTestName(getClass(), testName); + appId = "appId_" + testId; + inputTopic = "input" + testId; + IntegrationTestUtils.cleanStateBeforeTest(CLUSTER, inputTopic); + + builder = new StreamsBuilder(); + builder.stream(inputTopic); + + properties = mkObjectProperties( + mkMap( + mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()), + mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, appId), + mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()), + mkEntry(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 2), + mkEntry(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class), + mkEntry(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class) + ) + ); + } + + @After + public void teardown() throws IOException { + purgeLocalStreamsState(properties); + } + + @Test + public void shouldAddStreamThread() throws Exception { + try (final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), properties)) { + StreamsTestUtils.startKafkaStreamsAndWaitForRunningState(kafkaStreams); + final int oldThreadCount = kafkaStreams.localThreadsMetadata().size(); + assertThat(kafkaStreams.localThreadsMetadata().stream().map(t -> t.threadName().split("-StreamThread-")[1]).sorted().toArray(), equalTo(new String[] {"1", "2"})); + + final Optional name = kafkaStreams.addStreamThread(); + + assertThat(name, CoreMatchers.not(Optional.empty())); + TestUtils.waitForCondition( + () -> kafkaStreams.localThreadsMetadata().stream().sequential() + .map(ThreadMetadata::threadName).anyMatch(t -> t.equals(name.orElse(""))), + "Wait for the thread to be added" + ); + assertThat(kafkaStreams.localThreadsMetadata().size(), equalTo(oldThreadCount + 1)); + assertThat(kafkaStreams.localThreadsMetadata().stream().map(t -> t.threadName().split("-StreamThread-")[1]).sorted().toArray(), equalTo(new String[] {"1", "2", "3"})); + TestUtils.waitForCondition(() -> kafkaStreams.state() == KafkaStreams.State.RUNNING, "wait for running"); + } + } +} From 88c818095752f39488fd5aff232d4773e8b385b3 Mon Sep 17 00:00:00 2001 From: "high.lee" Date: Sat, 5 Dec 2020 04:56:59 +0900 Subject: [PATCH 06/14] KAFKA-8147: Update upgrade notes for KIP-446 (#8965) Reviewer: Matthias J. Sax , John Roesler --- docs/streams/upgrade-guide.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html index 758ea746b1183..f24d065f57412 100644 --- a/docs/streams/upgrade-guide.html +++ b/docs/streams/upgrade-guide.html @@ -156,6 +156,11 @@

Streams API We added a --force option in StreamsResetter to force remove left-over members on broker side when long session time out was configured as per KIP-571.

+

+ We added Suppressed.withLoggingDisabled() and Suppressed.withLoggingEnabled(config) + methods to allow disabling or configuring of the changelog topic and allows for configuration of the changelog topic + as per KIP-446. +

Streams API changes in 2.5.0

From a57486e75084d70f1fbf22df9d3ffc3462302add Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Fri, 4 Dec 2020 14:02:17 -0800 Subject: [PATCH 07/14] MINOR: Do not print log4j for memberId required (#9667) For MemberIdRequiredException, we would not print the exception at INFO with a full exception message since it may introduce more confusion that clearance. Reviewers: Chia-Ping Tsai , Boyang Chen --- .../clients/consumer/internals/AbstractCoordinator.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java index af6e262078d7a..4ca71bb626a06 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java @@ -465,7 +465,13 @@ boolean joinGroupIfNeeded(final Timer timer) { } } else { final RuntimeException exception = future.exception(); - log.info("Rebalance failed.", exception); + + // we do not need to log error for memberId required, + // since it is not really an error and is transient + if (!(exception instanceof MemberIdRequiredException)) { + log.info("Rebalance failed.", exception); + } + resetJoinGroupFuture(); if (exception instanceof UnknownMemberIdException || exception instanceof RebalanceInProgressException || From 8db3b1a09af0bad274e07161336994610d616b35 Mon Sep 17 00:00:00 2001 From: Randall Hauch Date: Mon, 7 Dec 2020 09:34:34 -0600 Subject: [PATCH 08/14] KAFKA-10811: Correct the MirrorConnectorsIntegrationTest to correctly mask the exit procedures (#9698) Normally the `EmbeddedConnectCluster` class masks the `Exit` procedures using within the Connect worker. This normally works great when a single instance of the embedded cluster is used. However, the `MirrorConnectorsIntegrationTest` uses two `EmbeddedConnectCluster` instances, and when the first one is stopped it would reset the (static) exit procedures, and any problems during shutdown of the second embedded Connect cluster would cause the worker to shut down the JVM running the tests. Instead, the `MirrorConnectorsIntegrationTest` class should mask the `Exit` procedures and instruct the `EmbeddedConnectClusters` instances (via the existing builder method) to not mask the procedures. Author: Randall Hauch Reviewers: Mickael Maison , Chia-Ping Tsai --- .../MirrorConnectorsIntegrationTest.java | 74 ++++++++++++++----- .../util/clusters/EmbeddedConnectCluster.java | 6 +- 2 files changed, 61 insertions(+), 19 deletions(-) diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java index a71b28c0aa9a1..d0715dc8452a9 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster; import org.apache.kafka.connect.util.clusters.EmbeddedKafkaCluster; +import org.apache.kafka.connect.util.clusters.UngracefulShutdownException; import org.apache.kafka.test.IntegrationTest; import org.apache.kafka.common.utils.Exit; import org.junit.After; @@ -44,13 +45,11 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.apache.kafka.test.TestUtils.waitForCondition; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -75,14 +74,45 @@ public class MirrorConnectorsIntegrationTest { private static final int RECORD_CONSUME_DURATION_MS = 20_000; private static final int OFFSET_SYNC_DURATION_MS = 30_000; - private final AtomicBoolean exited = new AtomicBoolean(false); + private volatile boolean shuttingDown; private Map mm2Props; private MirrorMakerConfig mm2Config; private EmbeddedConnectCluster primary; private EmbeddedConnectCluster backup; + private Exit.Procedure exitProcedure; + private Exit.Procedure haltProcedure; + @Before public void setup() throws InterruptedException { + shuttingDown = false; + exitProcedure = (code, message) -> { + if (shuttingDown) { + // ignore this since we're shutting down Connect and Kafka and timing isn't always great + return; + } + if (code != 0) { + String exitMessage = "Abrupt service exit with code " + code + " and message " + message; + log.warn(exitMessage); + throw new UngracefulShutdownException(exitMessage); + } + }; + haltProcedure = (code, message) -> { + if (shuttingDown) { + // ignore this since we're shutting down Connect and Kafka and timing isn't always great + return; + } + if (code != 0) { + String haltMessage = "Abrupt service halt with code " + code + " and message " + message; + log.warn(haltMessage); + throw new UngracefulShutdownException(haltMessage); + } + }; + // Override the exit and halt procedure that Connect and Kafka will use. For these integration tests, + // we don't want to exit the JVM and instead simply want to fail the test + Exit.setExitProcedure(exitProcedure); + Exit.setHaltProcedure(haltProcedure); + Properties brokerProps = new Properties(); brokerProps.put("auto.create.topics.enable", "false"); @@ -116,6 +146,7 @@ public void setup() throws InterruptedException { .numBrokers(1) .brokerProps(brokerProps) .workerProps(primaryWorkerProps) + .maskExitProcedures(false) .build(); backup = new EmbeddedConnectCluster.Builder() @@ -124,6 +155,7 @@ public void setup() throws InterruptedException { .numBrokers(1) .brokerProps(brokerProps) .workerProps(backupWorkerProps) + .maskExitProcedures(false) .build(); primary.start(); @@ -164,8 +196,6 @@ public void setup() throws InterruptedException { mm2Props.put("primary.bootstrap.servers", primary.kafka().bootstrapServers()); mm2Props.put("backup.bootstrap.servers", backup.kafka().bootstrapServers()); mm2Config = new MirrorMakerConfig(mm2Props); - - Exit.setExitProcedure((status, errorCode) -> exited.set(true)); } @@ -194,20 +224,27 @@ private void waitUntilMirrorMakerIsRunning(EmbeddedConnectCluster connectCluster @After public void close() { - for (String x : primary.connectors()) { - primary.deleteConnector(x); - } - for (String x : backup.connectors()) { - backup.deleteConnector(x); - } - deleteAllTopics(primary.kafka()); - deleteAllTopics(backup.kafka()); - primary.stop(); - backup.stop(); try { - assertFalse(exited.get()); + for (String x : primary.connectors()) { + primary.deleteConnector(x); + } + for (String x : backup.connectors()) { + backup.deleteConnector(x); + } + deleteAllTopics(primary.kafka()); + deleteAllTopics(backup.kafka()); } finally { - Exit.resetExitProcedure(); + shuttingDown = true; + try { + try { + primary.stop(); + } finally { + backup.stop(); + } + } finally { + Exit.resetExitProcedure(); + Exit.resetHaltProcedure(); + } } } @@ -305,6 +342,9 @@ public void testReplication() throws InterruptedException { Map primaryOffsets = primaryClient.remoteConsumerOffsets(consumerGroupName, "backup", Duration.ofMillis(CHECKPOINT_DURATION_MS)); + primaryClient.close(); + backupClient.close(); + // Failback consumer group to primary cluster primaryConsumer = primary.kafka().createConsumer(consumerProps); primaryConsumer.assign(allPartitions("test-topic-1", "backup.test-topic-1")); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java index 170ad3490974f..4c270d25f16b3 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java @@ -153,8 +153,10 @@ public void stop() { log.error("Could not stop kafka", e); throw new RuntimeException("Could not stop brokers", e); } finally { - Exit.resetExitProcedure(); - Exit.resetHaltProcedure(); + if (maskExitProcedures) { + Exit.resetExitProcedure(); + Exit.resetHaltProcedure(); + } } } From b8ebcc2a93a4e0525afb4f5ff6690ef5b7529c57 Mon Sep 17 00:00:00 2001 From: Rajini Sivaram Date: Mon, 7 Dec 2020 16:12:18 +0000 Subject: [PATCH 09/14] KAFKA-10798; Ensure response is delayed for failed SASL authentication with connection close delay (#9678) Reviewers: Manikumar Reddy --- .../SaslServerAuthenticator.java | 2 +- .../kafka/common/network/NioEchoServer.java | 2 +- .../SaslAuthenticatorFailureDelayTest.java | 37 ++++++++++++++++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java index 924ae4431d9ad..befbe4ec0a360 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java @@ -470,7 +470,7 @@ private void handleSaslToken(byte[] clientToken) throws IOException { String errorMessage = "Authentication failed during " + reauthInfo.authenticationOrReauthenticationText() + " due to invalid credentials with SASL mechanism " + saslMechanism; - sendKafkaResponse(requestContext, new SaslAuthenticateResponse( + buildResponseOnAuthenticateFailure(requestContext, new SaslAuthenticateResponse( new SaslAuthenticateResponseData() .setErrorCode(Errors.SASL_AUTHENTICATION_FAILED.code()) .setErrorMessage(errorMessage))); diff --git a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java index b66d66baba628..1d6894b4dacd0 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java +++ b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java @@ -91,7 +91,7 @@ public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtoco public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtocol, AbstractConfig config, String serverHost, ChannelBuilder channelBuilder, CredentialCache credentialCache, int failedAuthenticationDelayMs, Time time) throws Exception { - this(listenerName, securityProtocol, config, serverHost, channelBuilder, credentialCache, 100, time, + this(listenerName, securityProtocol, config, serverHost, channelBuilder, credentialCache, failedAuthenticationDelayMs, time, new DelegationTokenCache(ScramMechanism.mechanismNames())); } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java index 599345a110ac3..19003ed56da90 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -55,7 +56,7 @@ public class SaslAuthenticatorFailureDelayTest { private static final int BUFFER_SIZE = 4 * 1024; - private final MockTime time = new MockTime(10); + private final MockTime time = new MockTime(1); private NioEchoServer server; private Selector selector; private ChannelBuilder channelBuilder; @@ -118,6 +119,38 @@ public void testInvalidPasswordSaslPlain() throws Exception { server.verifyAuthenticationMetrics(0, 1); } + /** + * Tests that SASL/SCRAM clients with invalid password fail authentication with + * connection close delay if configured. + */ + @Test + public void testInvalidPasswordSaslScram() throws Exception { + String node = "0"; + SecurityProtocol securityProtocol = SecurityProtocol.SASL_SSL; + TestJaasConfig jaasConfig = configureMechanisms("SCRAM-SHA-256", Collections.singletonList("SCRAM-SHA-256")); + jaasConfig.setClientOptions("SCRAM-SHA-256", TestJaasConfig.USERNAME, "invalidpassword"); + + server = createEchoServer(securityProtocol); + createAndCheckClientAuthenticationFailure(securityProtocol, node, "SCRAM-SHA-256", null); + server.verifyAuthenticationMetrics(0, 1); + } + + /** + * Tests that clients with disabled SASL mechanism fail authentication with + * connection close delay if configured. + */ + @Test + public void testDisabledSaslMechanism() throws Exception { + String node = "0"; + SecurityProtocol securityProtocol = SecurityProtocol.SASL_SSL; + TestJaasConfig jaasConfig = configureMechanisms("SCRAM-SHA-256", Collections.singletonList("SCRAM-SHA-256")); + jaasConfig.setClientOptions("PLAIN", TestJaasConfig.USERNAME, "invalidpassword"); + + server = createEchoServer(securityProtocol); + createAndCheckClientAuthenticationFailure(securityProtocol, node, "SCRAM-SHA-256", null); + server.verifyAuthenticationMetrics(0, 1); + } + /** * Tests client connection close before response for authentication failure is sent. */ @@ -215,7 +248,7 @@ private void createAndCheckClientAuthenticationFailure(SecurityProtocol security Exception exception = finalState.exception(); assertTrue("Invalid exception class " + exception.getClass(), exception instanceof SaslAuthenticationException); if (expectedErrorMessage == null) - expectedErrorMessage = "Authentication failed due to invalid credentials with SASL mechanism " + mechanism; + expectedErrorMessage = "Authentication failed during authentication due to invalid credentials with SASL mechanism " + mechanism; assertEquals(expectedErrorMessage, exception.getMessage()); } From ab0807dd858887d934d2be520608d20bf765b609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Armando=20Garc=C3=ADa=20Sancio?= Date: Mon, 7 Dec 2020 14:06:25 -0800 Subject: [PATCH 10/14] KAFKA-10394: Add classes to read and write snapshot for KIP-630 (#9512) This PR adds support for generating snapshot for KIP-630. 1. Adds the interfaces `RawSnapshotWriter` and `RawSnapshotReader` and the implementations `FileRawSnapshotWriter` and `FileRawSnapshotReader` respectively. These interfaces and implementations are low level API for writing and reading snapshots. They are internal to the Raft implementation and are not exposed to the users of `RaftClient`. They operation at the `Record` level. These types are exposed to the `RaftClient` through the `ReplicatedLog` interface. 2. Adds a buffered snapshot writer: `SnapshotWriter`. This type is a higher-level type and it is exposed through the `RaftClient` interface. A future PR will add the related `SnapshotReader`, which will be used by the state machine to load a snapshot. Reviewers: Jason Gustafson --- checkstyle/import-control.xml | 7 + .../org/apache/kafka/common/utils/Utils.java | 5 + .../scala/kafka/raft/KafkaMetadataLog.scala | 17 ++ .../apache/kafka/raft/KafkaRaftClient.java | 17 +- .../org/apache/kafka/raft/QuorumState.java | 14 +- .../org/apache/kafka/raft/RaftClient.java | 13 + .../org/apache/kafka/raft/ReplicatedLog.java | 26 ++ .../kafka/snapshot/FileRawSnapshotReader.java | 81 +++++ .../kafka/snapshot/FileRawSnapshotWriter.java | 117 +++++++ .../kafka/snapshot/RawSnapshotReader.java | 52 ++++ .../kafka/snapshot/RawSnapshotWriter.java | 74 +++++ .../apache/kafka/snapshot/SnapshotWriter.java | 156 ++++++++++ .../org/apache/kafka/snapshot/Snapshots.java | 66 ++++ .../java/org/apache/kafka/raft/MockLog.java | 107 +++++++ .../kafka/raft/RaftClientTestContext.java | 10 +- .../kafka/snapshot/FileRawSnapshotTest.java | 289 ++++++++++++++++++ .../kafka/snapshot/SnapshotWriterTest.java | 116 +++++++ 17 files changed, 1153 insertions(+), 14 deletions(-) create mode 100644 raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotReader.java create mode 100644 raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotWriter.java create mode 100644 raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotReader.java create mode 100644 raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotWriter.java create mode 100644 raft/src/main/java/org/apache/kafka/snapshot/SnapshotWriter.java create mode 100644 raft/src/main/java/org/apache/kafka/snapshot/Snapshots.java create mode 100644 raft/src/test/java/org/apache/kafka/snapshot/FileRawSnapshotTest.java create mode 100644 raft/src/test/java/org/apache/kafka/snapshot/SnapshotWriterTest.java diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index ceee02adffa50..5368ef217bfbf 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -318,6 +318,7 @@ + @@ -329,6 +330,12 @@ + + + + + + diff --git a/clients/src/main/java/org/apache/kafka/common/utils/Utils.java b/clients/src/main/java/org/apache/kafka/common/utils/Utils.java index 5a8aefec8a00c..3f4a9312fdc3e 100755 --- a/clients/src/main/java/org/apache/kafka/common/utils/Utils.java +++ b/clients/src/main/java/org/apache/kafka/common/utils/Utils.java @@ -1321,4 +1321,9 @@ public static long getDateTime(String timestamp) throws ParseException, IllegalA return date.getTime(); } } + + @SuppressWarnings("unchecked") + public static Iterator covariantCast(Iterator iterator) { + return (Iterator) iterator; + } } diff --git a/core/src/main/scala/kafka/raft/KafkaMetadataLog.scala b/core/src/main/scala/kafka/raft/KafkaMetadataLog.scala index c5225953b1500..039b299481dd4 100644 --- a/core/src/main/scala/kafka/raft/KafkaMetadataLog.scala +++ b/core/src/main/scala/kafka/raft/KafkaMetadataLog.scala @@ -16,6 +16,7 @@ */ package kafka.raft +import java.nio.file.NoSuchFileException import java.util.Optional import kafka.log.{AppendOrigin, Log} @@ -24,6 +25,10 @@ import org.apache.kafka.common.record.{MemoryRecords, Records} import org.apache.kafka.common.{KafkaException, TopicPartition} import org.apache.kafka.raft import org.apache.kafka.raft.{LogAppendInfo, LogFetchInfo, LogOffsetMetadata, Isolation, ReplicatedLog} +import org.apache.kafka.snapshot.FileRawSnapshotReader +import org.apache.kafka.snapshot.FileRawSnapshotWriter +import org.apache.kafka.snapshot.RawSnapshotReader +import org.apache.kafka.snapshot.RawSnapshotWriter import scala.compat.java8.OptionConverters._ @@ -141,6 +146,18 @@ class KafkaMetadataLog( topicPartition } + override def createSnapshot(snapshotId: raft.OffsetAndEpoch): RawSnapshotWriter = { + FileRawSnapshotWriter.create(log.dir.toPath, snapshotId) + } + + override def readSnapshot(snapshotId: raft.OffsetAndEpoch): Optional[RawSnapshotReader] = { + try { + Optional.of(FileRawSnapshotReader.open(log.dir.toPath, snapshotId)) + } catch { + case e: NoSuchFileException => Optional.empty() + } + } + override def close(): Unit = { log.close() } diff --git a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java index 66553f66f3332..37c1846047af5 100644 --- a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java @@ -23,14 +23,14 @@ import org.apache.kafka.common.message.BeginQuorumEpochRequestData; import org.apache.kafka.common.message.BeginQuorumEpochResponseData; import org.apache.kafka.common.message.DescribeQuorumRequestData; -import org.apache.kafka.common.message.DescribeQuorumResponseData; import org.apache.kafka.common.message.DescribeQuorumResponseData.ReplicaState; +import org.apache.kafka.common.message.DescribeQuorumResponseData; import org.apache.kafka.common.message.EndQuorumEpochRequestData; import org.apache.kafka.common.message.EndQuorumEpochResponseData; import org.apache.kafka.common.message.FetchRequestData; import org.apache.kafka.common.message.FetchResponseData; -import org.apache.kafka.common.message.LeaderChangeMessage; import org.apache.kafka.common.message.LeaderChangeMessage.Voter; +import org.apache.kafka.common.message.LeaderChangeMessage; import org.apache.kafka.common.message.VoteRequestData; import org.apache.kafka.common.message.VoteResponseData; import org.apache.kafka.common.metrics.Metrics; @@ -61,6 +61,7 @@ import org.apache.kafka.raft.internals.MemoryBatchReader; import org.apache.kafka.raft.internals.RecordsBatchReader; import org.apache.kafka.raft.internals.ThresholdPurgatory; +import org.apache.kafka.snapshot.SnapshotWriter; import org.slf4j.Logger; import java.io.IOException; @@ -1815,6 +1816,18 @@ public CompletableFuture shutdown(int timeoutMs) { return shutdownComplete; } + @Override + public SnapshotWriter createSnapshot(OffsetAndEpoch snapshotId) throws IOException { + return new SnapshotWriter<>( + log.createSnapshot(snapshotId), + MAX_BATCH_SIZE, + memoryPool, + time, + CompressionType.NONE, + serde + ); + } + private void close() { kafkaRaftMetrics.close(); } diff --git a/raft/src/main/java/org/apache/kafka/raft/QuorumState.java b/raft/src/main/java/org/apache/kafka/raft/QuorumState.java index d1eea74dbb5e2..3764308bddde8 100644 --- a/raft/src/main/java/org/apache/kafka/raft/QuorumState.java +++ b/raft/src/main/java/org/apache/kafka/raft/QuorumState.java @@ -35,26 +35,26 @@ * only valid state transitions. Below we define the possible state transitions and * how they are triggered: * - * Unattached|Resigned => + * Unattached|Resigned transitions to: * Unattached: After learning of a new election with a higher epoch * Voted: After granting a vote to a candidate * Candidate: After expiration of the election timeout * Follower: After discovering a leader with an equal or larger epoch * - * Voted => + * Voted transitions to: * Unattached: After learning of a new election with a higher epoch * Candidate: After expiration of the election timeout * - * Candidate => + * Candidate transitions to: * Unattached: After learning of a new election with a higher epoch * Candidate: After expiration of the election timeout * Leader: After receiving a majority of votes * - * Leader => + * Leader transitions to: * Unattached: After learning of a new election with a higher epoch * Resigned: When shutting down gracefully * - * Follower => + * Follower transitions to: * Unattached: After learning of a new election with a higher epoch * Candidate: After expiration of the fetch timeout * Follower: After discovering a leader with a larger epoch @@ -63,11 +63,11 @@ * states are not possible for observers, so the only transitions that are possible * are between Unattached and Follower. * - * Unattached => + * Unattached transitions to: * Unattached: After learning of a new election with a higher epoch * Follower: After discovering a leader with an equal or larger epoch * - * Follower => + * Follower transitions to: * Unattached: After learning of a new election with a higher epoch * Follower: After discovering a leader with a larger epoch * diff --git a/raft/src/main/java/org/apache/kafka/raft/RaftClient.java b/raft/src/main/java/org/apache/kafka/raft/RaftClient.java index 1763bd36a05fb..ba58cb2cfb5c8 100644 --- a/raft/src/main/java/org/apache/kafka/raft/RaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/RaftClient.java @@ -16,6 +16,8 @@ */ package org.apache.kafka.raft; +import org.apache.kafka.snapshot.SnapshotWriter; + import java.io.IOException; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -100,4 +102,15 @@ default void handleResign() {} */ CompletableFuture shutdown(int timeoutMs); + /** + * Create a writable snapshot file for a given offset and epoch. + * + * The RaftClient assumes that the snapshot return will contain the records up to but + * not including the end offset in the snapshot id. See {@link SnapshotWriter} for + * details on how to use this object. + * + * @param snapshotId the end offset and epoch that identifies the snapshot + * @return a writable snapshot + */ + SnapshotWriter createSnapshot(OffsetAndEpoch snapshotId) throws IOException; } diff --git a/raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java b/raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java index d0572947b5436..b56e9bdbff9a9 100644 --- a/raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java +++ b/raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java @@ -18,8 +18,11 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.record.Records; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RawSnapshotWriter; import java.io.Closeable; +import java.io.IOException; import java.util.Optional; import java.util.OptionalLong; @@ -149,6 +152,29 @@ default OptionalLong truncateToEndOffset(OffsetAndEpoch endOffset) { return OptionalLong.of(truncationOffset); } + /** + * Create a writable snapshot for the given snapshot id. + * + * See {@link RawSnapshotWriter} for details on how to use this object. + * + * @param snapshotId the end offset and epoch that identifies the snapshot + * @return a writable snapshot + */ + RawSnapshotWriter createSnapshot(OffsetAndEpoch snapshotId) throws IOException; + + /** + * Opens a readable snapshot for the given snapshot id. + * + * Returns an Optional with a readable snapshot, if the snapshot exists, otherwise + * returns an empty Optional. See {@link RawSnapshotReader} for details on how to + * use this object. + * + * @param snapshotId the end offset and epoch that identifies the snapshot + * @return an Optional with a readable snapshot, if the snapshot exists, otherwise + * returns an empty Optional + */ + Optional readSnapshot(OffsetAndEpoch snapshotId) throws IOException; + default void close() {} } diff --git a/raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotReader.java b/raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotReader.java new file mode 100644 index 0000000000000..d517006d4d68f --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotReader.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.snapshot; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.Iterator; +import org.apache.kafka.common.record.FileRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.raft.OffsetAndEpoch; + +public final class FileRawSnapshotReader implements RawSnapshotReader { + private final FileRecords fileRecords; + private final OffsetAndEpoch snapshotId; + + private FileRawSnapshotReader(FileRecords fileRecords, OffsetAndEpoch snapshotId) { + this.fileRecords = fileRecords; + this.snapshotId = snapshotId; + } + + @Override + public OffsetAndEpoch snapshotId() { + return snapshotId; + } + + @Override + public long sizeInBytes() { + return fileRecords.sizeInBytes(); + } + + @Override + public Iterator iterator() { + return Utils.covariantCast(fileRecords.batchIterator()); + } + + @Override + public int read(ByteBuffer buffer, long position) throws IOException { + return fileRecords.channel().read(buffer, position); + } + + @Override + public void close() throws IOException { + fileRecords.close(); + } + + /** + * Opens a snapshot for reading. + * + * @param logDir the directory for the topic partition + * @param snapshotId the end offset and epoch for the snapshotId + * @throws java.nio.file.NoSuchFileException if the snapshot doesn't exist + * @throws IOException for any IO error while opening the snapshot + */ + public static FileRawSnapshotReader open(Path logDir, OffsetAndEpoch snapshotId) throws IOException { + FileRecords fileRecords = FileRecords.open( + Snapshots.snapshotPath(logDir, snapshotId).toFile(), + false, // mutable + true, // fileAlreadyExists + 0, // initFileSize + false // preallocate + ); + + return new FileRawSnapshotReader(fileRecords, snapshotId); + } +} diff --git a/raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotWriter.java b/raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotWriter.java new file mode 100644 index 0000000000000..934a12481ccf1 --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotWriter.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.snapshot; + +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.raft.OffsetAndEpoch; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +public final class FileRawSnapshotWriter implements RawSnapshotWriter { + private final Path tempSnapshotPath; + private final FileChannel channel; + private final OffsetAndEpoch snapshotId; + private boolean frozen = false; + + private FileRawSnapshotWriter( + Path tempSnapshotPath, + FileChannel channel, + OffsetAndEpoch snapshotId + ) { + this.tempSnapshotPath = tempSnapshotPath; + this.channel = channel; + this.snapshotId = snapshotId; + } + + @Override + public OffsetAndEpoch snapshotId() { + return snapshotId; + } + + @Override + public long sizeInBytes() throws IOException { + return channel.size(); + } + + @Override + public void append(ByteBuffer buffer) throws IOException { + if (frozen) { + throw new IllegalStateException( + String.format("Append is not supported. Snapshot is already frozen: id = %s; temp path = %s", snapshotId, tempSnapshotPath) + ); + } + + Utils.writeFully(channel, buffer); + } + + @Override + public boolean isFrozen() { + return frozen; + } + + @Override + public void freeze() throws IOException { + if (frozen) { + throw new IllegalStateException( + String.format("Freeze is not supported. Snapshot is already frozen: id = %s; temp path = %s", snapshotId, tempSnapshotPath) + ); + } + + channel.close(); + frozen = true; + + // Set readonly and ignore the result + if (!tempSnapshotPath.toFile().setReadOnly()) { + throw new IOException(String.format("Unable to set file (%s) as read-only", tempSnapshotPath)); + } + + Path destination = Snapshots.moveRename(tempSnapshotPath, snapshotId); + Utils.atomicMoveWithFallback(tempSnapshotPath, destination); + } + + @Override + public void close() throws IOException { + try { + channel.close(); + } finally { + // This is a noop if freeze was called before calling close + Files.deleteIfExists(tempSnapshotPath); + } + } + + /** + * Create a snapshot writer for topic partition log dir and snapshot id. + * + * @param logDir the directory for the topic partition + * @param snapshotId the end offset and epoch for the snapshotId + * @throws IOException for any IO error while creating the snapshot + */ + public static FileRawSnapshotWriter create(Path logDir, OffsetAndEpoch snapshotId) throws IOException { + Path path = Snapshots.createTempFile(logDir, snapshotId); + + return new FileRawSnapshotWriter( + path, + FileChannel.open(path, Utils.mkSet(StandardOpenOption.WRITE, StandardOpenOption.APPEND)), + snapshotId + ); + } +} diff --git a/raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotReader.java b/raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotReader.java new file mode 100644 index 0000000000000..6b10fa89bba59 --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotReader.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.snapshot; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.raft.OffsetAndEpoch; + +/** + * Interface for reading snapshots as a sequence of records. + */ +public interface RawSnapshotReader extends Closeable, Iterable { + /** + * Returns the end offset and epoch for the snapshot. + */ + public OffsetAndEpoch snapshotId(); + + /** + * Returns the number of bytes for the snapshot. + * + * @throws IOException for any IO error while reading the size + */ + public long sizeInBytes() throws IOException; + + /** + * Reads bytes from position into the given buffer. + * + * It is not guarantee that the given buffer will be filled. + * + * @param buffer byte buffer to put the read files + * @param position the starting position in the snapshot to read + * @return the number of bytes read + * @throws IOException for any IO error while reading the snapshot + */ + public int read(ByteBuffer buffer, long position) throws IOException; +} diff --git a/raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotWriter.java b/raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotWriter.java new file mode 100644 index 0000000000000..001d2ef37c78e --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotWriter.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.snapshot; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.kafka.raft.OffsetAndEpoch; + +/** + * Interface for writing snapshot as a sequence of records. + */ +public interface RawSnapshotWriter extends Closeable { + /** + * Returns the end offset and epoch for the snapshot. + */ + public OffsetAndEpoch snapshotId(); + + /** + * Returns the number of bytes for the snapshot. + * + * @throws IOException for any IO error while reading the size + */ + public long sizeInBytes() throws IOException; + + /** + * Fully appends the buffer to the snapshot. + * + * If the method returns without an exception the given buffer was fully writing the + * snapshot. + * + * @param buffer the buffer to append + * @throws IOException for any IO error during append + */ + public void append(ByteBuffer buffer) throws IOException; + + /** + * Returns true if the snapshot has been frozen, otherwise false is returned. + * + * Modification to the snapshot are not allowed once it is frozen. + */ + public boolean isFrozen(); + + /** + * Freezes the snapshot and marking it as immutable. + * + * @throws IOException for any IO error during freezing + */ + public void freeze() throws IOException; + + /** + * Closes the snapshot writer. + * + * If close is called without first calling freeze the the snapshot is aborted. + * + * @throws IOException for any IO error during close + */ + public void close() throws IOException; +} diff --git a/raft/src/main/java/org/apache/kafka/snapshot/SnapshotWriter.java b/raft/src/main/java/org/apache/kafka/snapshot/SnapshotWriter.java new file mode 100644 index 0000000000000..0232e93ac380f --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/snapshot/SnapshotWriter.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.snapshot; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import org.apache.kafka.common.memory.MemoryPool; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.raft.OffsetAndEpoch; +import org.apache.kafka.raft.RecordSerde; +import org.apache.kafka.raft.internals.BatchAccumulator.CompletedBatch; +import org.apache.kafka.raft.internals.BatchAccumulator; + +/** + * A type for writing a snapshot fora given end offset and epoch. + * + * A snapshot writer can be used to append objects until freeze is called. When freeze is + * called the snapshot is validated and marked as immutable. After freeze is called any + * append will fail with an exception. + * + * It is assumed that the content of the snapshot represents all of the records for the + * topic partition from offset 0 up to but not including the end offset in the snapshot + * id. + * + * @see org.apache.kafka.raft.RaftClient#createSnapshot(OffsetAndEpoch) + */ +final public class SnapshotWriter implements Closeable { + final private RawSnapshotWriter snapshot; + final private BatchAccumulator accumulator; + final private Time time; + + /** + * Initializes a new instance of the class. + * + * @param snapshot the low level snapshot writer + * @param maxBatchSize the maximum size in byte for a batch + * @param memoryPool the memory pool for buffer allocation + * @param time the clock implementation + * @param compressionType the compression algorithm to use + * @param serde the record serialization and deserialization implementation + */ + public SnapshotWriter( + RawSnapshotWriter snapshot, + int maxBatchSize, + MemoryPool memoryPool, + Time time, + CompressionType compressionType, + RecordSerde serde + ) { + this.snapshot = snapshot; + this.time = time; + + this.accumulator = new BatchAccumulator<>( + snapshot.snapshotId().epoch, + 0, + Integer.MAX_VALUE, + maxBatchSize, + memoryPool, + time, + compressionType, + serde + ); + } + + /** + * Returns the end offset and epoch for the snapshot. + */ + public OffsetAndEpoch snapshotId() { + return snapshot.snapshotId(); + } + + /** + * Returns true if the snapshot has been frozen, otherwise false is returned. + * + * Modification to the snapshot are not allowed once it is frozen. + */ + public boolean isFrozen() { + return snapshot.isFrozen(); + } + + /** + * Appends a list of values to the snapshot. + * + * The list of record passed are guaranteed to get written together. + * + * @param records the list of records to append to the snapshot + * @throws IOException for any IO error while appending + * @throws IllegalStateException if append is called when isFrozen is true + */ + public void append(List records) throws IOException { + if (snapshot.isFrozen()) { + String message = String.format( + "Append not supported. Snapshot is already frozen: id = {}.", + snapshot.snapshotId() + ); + + throw new IllegalStateException(message); + } + + accumulator.append(snapshot.snapshotId().epoch, records); + + if (accumulator.needsDrain(time.milliseconds())) { + appendBatches(accumulator.drain()); + } + } + + /** + * Freezes the snapshot by flushing all pending writes and marking it as immutable. + * + * @throws IOException for any IO error during freezing + */ + public void freeze() throws IOException { + appendBatches(accumulator.drain()); + snapshot.freeze(); + accumulator.close(); + } + + /** + * Closes the snapshot writer. + * + * If close is called without first calling freeze the the snapshot is aborted. + * + * @throws IOException for any IO error during close + */ + public void close() throws IOException { + snapshot.close(); + accumulator.close(); + } + + private void appendBatches(List> batches) throws IOException { + try { + for (CompletedBatch batch : batches) { + snapshot.append(batch.data.buffer()); + } + } finally { + batches.forEach(CompletedBatch::release); + } + } +} diff --git a/raft/src/main/java/org/apache/kafka/snapshot/Snapshots.java b/raft/src/main/java/org/apache/kafka/snapshot/Snapshots.java new file mode 100644 index 0000000000000..8abf160b1a82e --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/snapshot/Snapshots.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.snapshot; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.NumberFormat; +import org.apache.kafka.raft.OffsetAndEpoch; + +final class Snapshots { + private static final String SUFFIX = ".checkpoint"; + private static final String PARTIAL_SUFFIX = String.format("%s.part", SUFFIX); + + private static final NumberFormat OFFSET_FORMATTER = NumberFormat.getInstance(); + private static final NumberFormat EPOCH_FORMATTER = NumberFormat.getInstance(); + + static { + OFFSET_FORMATTER.setMinimumIntegerDigits(20); + OFFSET_FORMATTER.setGroupingUsed(false); + + EPOCH_FORMATTER.setMinimumIntegerDigits(10); + EPOCH_FORMATTER.setGroupingUsed(false); + } + + static Path snapshotDir(Path logDir) { + return logDir; + } + + static Path snapshotPath(Path logDir, OffsetAndEpoch snapshotId) { + return snapshotDir(logDir).resolve(filenameFromSnapshotId(snapshotId) + SUFFIX); + } + + static String filenameFromSnapshotId(OffsetAndEpoch snapshotId) { + return String.format("%s-%s", OFFSET_FORMATTER.format(snapshotId.offset), EPOCH_FORMATTER.format(snapshotId.epoch)); + } + + static Path moveRename(Path source, OffsetAndEpoch snapshotId) { + return source.resolveSibling(filenameFromSnapshotId(snapshotId) + SUFFIX); + } + + static Path createTempFile(Path logDir, OffsetAndEpoch snapshotId) throws IOException { + Path dir = snapshotDir(logDir); + + // Create the snapshot directory if it doesn't exists + Files.createDirectories(dir); + + String prefix = String.format("%s-", filenameFromSnapshotId(snapshotId)); + + return Files.createTempFile(dir, prefix, PARTIAL_SUFFIX); + } +} diff --git a/raft/src/test/java/org/apache/kafka/raft/MockLog.java b/raft/src/test/java/org/apache/kafka/raft/MockLog.java index 57c519e001d7a..1e8c1b53fcb72 100644 --- a/raft/src/test/java/org/apache/kafka/raft/MockLog.java +++ b/raft/src/test/java/org/apache/kafka/raft/MockLog.java @@ -26,13 +26,19 @@ import org.apache.kafka.common.record.Records; import org.apache.kafka.common.record.SimpleRecord; import org.apache.kafka.common.record.TimestampType; +import org.apache.kafka.common.utils.ByteBufferOutputStream; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RawSnapshotWriter; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; @@ -45,6 +51,7 @@ public class MockLog implements ReplicatedLog { private final List epochStartOffsets = new ArrayList<>(); private final List log = new ArrayList<>(); + private final Map snapshots = new HashMap<>(); private final TopicPartition topicPartition; private long nextId = ID_GENERATOR.getAndIncrement(); @@ -351,6 +358,16 @@ public void initializeLeaderEpoch(int epoch) { epochStartOffsets.add(new EpochStartOffset(epoch, startOffset)); } + @Override + public RawSnapshotWriter createSnapshot(OffsetAndEpoch snapshotId) { + return new MockRawSnapshotWriter(snapshotId); + } + + @Override + public Optional readSnapshot(OffsetAndEpoch snapshotId) { + return Optional.ofNullable(snapshots.get(snapshotId)); + } + static class MockOffsetMetadata implements OffsetMetadata { final long id; @@ -472,4 +489,94 @@ private EpochStartOffset(int epoch, long startOffset) { } } + final class MockRawSnapshotWriter implements RawSnapshotWriter { + private final OffsetAndEpoch snapshotId; + private ByteBufferOutputStream data; + private boolean frozen; + + public MockRawSnapshotWriter(OffsetAndEpoch snapshotId) { + this.snapshotId = snapshotId; + this.data = new ByteBufferOutputStream(0); + this.frozen = false; + } + + @Override + public OffsetAndEpoch snapshotId() { + return snapshotId; + } + + @Override + public long sizeInBytes() { + return data.position(); + } + + @Override + public void append(ByteBuffer buffer) { + if (frozen) { + throw new RuntimeException("Snapshot is already frozen " + snapshotId); + } + + data.write(buffer); + } + + @Override + public boolean isFrozen() { + return frozen; + } + + @Override + public void freeze() { + if (frozen) { + throw new RuntimeException("Snapshot is already frozen " + snapshotId); + } + + frozen = true; + ByteBuffer buffer = data.buffer(); + buffer.flip(); + + snapshots.putIfAbsent(snapshotId, new MockRawSnapshotReader(snapshotId, buffer)); + } + + @Override + public void close() {} + } + + final static class MockRawSnapshotReader implements RawSnapshotReader { + private final OffsetAndEpoch snapshotId; + private final MemoryRecords data; + + MockRawSnapshotReader(OffsetAndEpoch snapshotId, ByteBuffer data) { + this.snapshotId = snapshotId; + this.data = MemoryRecords.readableRecords(data); + } + + @Override + public OffsetAndEpoch snapshotId() { + return snapshotId; + } + + @Override + public long sizeInBytes() { + return data.sizeInBytes(); + } + + @Override + public Iterator iterator() { + return Utils.covariantCast(data.batchIterator()); + } + + @Override + public int read(ByteBuffer buffer, long position) { + ByteBuffer copy = data.buffer(); + copy.position((int) position); + copy.limit((int) position + Math.min(copy.remaining(), buffer.remaining())); + + buffer.put(copy); + + return copy.remaining(); + } + + @Override + public void close() {} + } } diff --git a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java index 4d9db4aaf90a5..0e803e5ebef4b 100644 --- a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java +++ b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java @@ -78,7 +78,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -final class RaftClientTestContext { +public final class RaftClientTestContext { private static final StringSerde STRING_SERDE = new StringSerde(); final TopicPartition metadataPartition = Builder.METADATA_PARTITION; @@ -91,9 +91,9 @@ final class RaftClientTestContext { private final QuorumStateStore quorumStateStore; final int localId; - final KafkaRaftClient client; + public final KafkaRaftClient client; final Metrics metrics; - final MockLog log; + public final MockLog log; final MockNetworkChannel channel; final MockTime time; final MockListener listener; @@ -124,7 +124,7 @@ public static final class Builder { private int appendLingerMs = DEFAULT_APPEND_LINGER_MS; private MemoryPool memoryPool = MemoryPool.NONE; - Builder(int localId, Set voters) { + public Builder(int localId, Set voters) { this.voters = voters; this.localId = localId; } @@ -175,7 +175,7 @@ Builder withRequestTimeoutMs(int requestTimeoutMs) { return this; } - RaftClientTestContext build() throws IOException { + public RaftClientTestContext build() throws IOException { Metrics metrics = new Metrics(time); MockNetworkChannel channel = new MockNetworkChannel(); LogContext logContext = new LogContext(); diff --git a/raft/src/test/java/org/apache/kafka/snapshot/FileRawSnapshotTest.java b/raft/src/test/java/org/apache/kafka/snapshot/FileRawSnapshotTest.java new file mode 100644 index 0000000000000..e52943558f2cc --- /dev/null +++ b/raft/src/test/java/org/apache/kafka/snapshot/FileRawSnapshotTest.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.snapshot; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Iterator; +import java.util.stream.IntStream; +import org.apache.kafka.common.record.BufferSupplier.GrowableBufferSupplier; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.raft.OffsetAndEpoch; +import org.apache.kafka.test.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public final class FileRawSnapshotTest { + @Test + public void testWritingSnapshot() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3); + int bufferSize = 256; + int batches = 10; + int expectedSize = 0; + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + assertEquals(0, snapshot.sizeInBytes()); + + MemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize))); + for (int i = 0; i < batches; i++) { + snapshot.append(records.buffer()); + expectedSize += records.sizeInBytes(); + } + + assertEquals(expectedSize, snapshot.sizeInBytes()); + + snapshot.freeze(); + } + + // File should exist and the size should be the sum of all the buffers + assertTrue(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch))); + assertEquals(expectedSize, Files.size(Snapshots.snapshotPath(tempDir, offsetAndEpoch))); + } + + @Test + public void testWriteReadSnapshot() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3); + int bufferSize = 256; + int batches = 10; + + ByteBuffer expectedBuffer = ByteBuffer.wrap(randomBytes(bufferSize)); + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + MemoryRecords records = buildRecords(expectedBuffer); + for (int i = 0; i < batches; i++) { + snapshot.append(records.buffer()); + } + + snapshot.freeze(); + } + + try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) { + int countBatches = 0; + int countRecords = 0; + for (RecordBatch batch : snapshot) { + countBatches += 1; + + Iterator records = batch.streamingIterator(new GrowableBufferSupplier()); + while (records.hasNext()) { + Record record = records.next(); + + countRecords += 1; + + assertFalse(record.hasKey()); + assertTrue(record.hasValue()); + assertEquals(bufferSize, record.value().remaining()); + assertEquals(expectedBuffer, record.value()); + } + } + + assertEquals(batches, countBatches); + assertEquals(batches, countRecords); + } + } + + @Test + public void testBatchWriteReadSnapshot() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3); + int bufferSize = 256; + int batchSize = 3; + int batches = 10; + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + for (int i = 0; i < batches; i++) { + ByteBuffer[] buffers = IntStream + .range(0, batchSize) + .mapToObj(ignore -> ByteBuffer.wrap(randomBytes(bufferSize))).toArray(ByteBuffer[]::new); + + snapshot.append(buildRecords(buffers).buffer()); + } + + snapshot.freeze(); + } + + try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) { + int countBatches = 0; + int countRecords = 0; + for (RecordBatch batch : snapshot) { + countBatches += 1; + + Iterator records = batch.streamingIterator(new GrowableBufferSupplier()); + while (records.hasNext()) { + Record record = records.next(); + + countRecords += 1; + + assertFalse(record.hasKey()); + assertTrue(record.hasValue()); + assertEquals(bufferSize, record.value().remaining()); + } + } + + assertEquals(batches, countBatches); + assertEquals(batches * batchSize, countRecords); + } + } + + @Test + public void testBufferWriteReadSnapshot() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3); + int bufferSize = 256; + int batchSize = 3; + int batches = 10; + int expectedSize = 0; + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + for (int i = 0; i < batches; i++) { + ByteBuffer[] buffers = IntStream + .range(0, batchSize) + .mapToObj(ignore -> ByteBuffer.wrap(randomBytes(bufferSize))).toArray(ByteBuffer[]::new); + + MemoryRecords records = buildRecords(buffers); + snapshot.append(records.buffer()); + expectedSize += records.sizeInBytes(); + } + + assertEquals(expectedSize, snapshot.sizeInBytes()); + + snapshot.freeze(); + } + + // File should exist and the size should be the sum of all the buffers + assertTrue(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch))); + assertEquals(expectedSize, Files.size(Snapshots.snapshotPath(tempDir, offsetAndEpoch))); + + try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) { + int countBatches = 0; + int countRecords = 0; + + for (RecordBatch batch : snapshot) { + countBatches += 1; + + Iterator records = batch.streamingIterator(new GrowableBufferSupplier()); + while (records.hasNext()) { + Record record = records.next(); + + countRecords += 1; + + assertFalse(record.hasKey()); + assertTrue(record.hasValue()); + assertEquals(bufferSize, record.value().remaining()); + } + } + + assertEquals(batches, countBatches); + assertEquals(batches * batchSize, countRecords); + } + } + + @Test + public void testAbortedSnapshot() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(20L, 2); + int bufferSize = 256; + int batches = 10; + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + MemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize))); + for (int i = 0; i < batches; i++) { + snapshot.append(records.buffer()); + } + } + + // File should not exist since freeze was not called before + assertFalse(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch))); + assertEquals(0, Files.list(Snapshots.snapshotDir(tempDir)).count()); + } + + @Test + public void testAppendToFrozenSnapshot() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3); + int bufferSize = 256; + int batches = 10; + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + MemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize))); + for (int i = 0; i < batches; i++) { + snapshot.append(records.buffer()); + } + + snapshot.freeze(); + + assertThrows(RuntimeException.class, () -> snapshot.append(records.buffer())); + } + + // File should exist and the size should be greater than the sum of all the buffers + assertTrue(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch))); + assertTrue(Files.size(Snapshots.snapshotPath(tempDir, offsetAndEpoch)) > bufferSize * batches); + } + + @Test + public void testCreateSnapshotWithSameId() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(20L, 2); + int bufferSize = 256; + int batches = 1; + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + MemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize))); + for (int i = 0; i < batches; i++) { + snapshot.append(records.buffer()); + } + + snapshot.freeze(); + } + + // Create another snapshot with the same id + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + MemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize))); + for (int i = 0; i < batches; i++) { + snapshot.append(records.buffer()); + } + + snapshot.freeze(); + } + } + + private static byte[] randomBytes(int size) { + byte[] array = new byte[size]; + + TestUtils.SEEDED_RANDOM.nextBytes(array); + + return array; + } + + private static MemoryRecords buildRecords(ByteBuffer... buffers) { + return MemoryRecords.withRecords( + CompressionType.NONE, + Arrays.stream(buffers).map(buffer -> new SimpleRecord(buffer)).toArray(SimpleRecord[]::new) + ); + } +} diff --git a/raft/src/test/java/org/apache/kafka/snapshot/SnapshotWriterTest.java b/raft/src/test/java/org/apache/kafka/snapshot/SnapshotWriterTest.java new file mode 100644 index 0000000000000..3957640284e59 --- /dev/null +++ b/raft/src/test/java/org/apache/kafka/snapshot/SnapshotWriterTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.snapshot; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.Set; +import org.apache.kafka.common.record.BufferSupplier.GrowableBufferSupplier; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.raft.OffsetAndEpoch; +import org.apache.kafka.raft.RaftClientTestContext; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +final public class SnapshotWriterTest { + private final int localId = 0; + private final Set voters = Collections.singleton(localId); + + @Test + public void testWritingSnapshot() throws IOException { + OffsetAndEpoch id = new OffsetAndEpoch(10L, 3); + List> expected = buildRecords(3, 3); + RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).build(); + + try (SnapshotWriter snapshot = context.client.createSnapshot(id)) { + expected.forEach(batch -> { + assertDoesNotThrow(() -> snapshot.append(batch)); + }); + snapshot.freeze(); + } + + try (RawSnapshotReader reader = context.log.readSnapshot(id).get()) { + assertSnapshot(expected, reader); + } + } + + @Test + public void testAbortedSnapshot() throws IOException { + OffsetAndEpoch id = new OffsetAndEpoch(10L, 3); + List> expected = buildRecords(3, 3); + RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).build(); + + try (SnapshotWriter snapshot = context.client.createSnapshot(id)) { + expected.forEach(batch -> { + assertDoesNotThrow(() -> snapshot.append(batch)); + }); + } + + assertFalse(context.log.readSnapshot(id).isPresent()); + } + + @Test + public void testAppendToFrozenSnapshot() throws IOException { + OffsetAndEpoch id = new OffsetAndEpoch(10L, 3); + List> expected = buildRecords(3, 3); + RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).build(); + + try (SnapshotWriter snapshot = context.client.createSnapshot(id)) { + expected.forEach(batch -> { + assertDoesNotThrow(() -> snapshot.append(batch)); + }); + + snapshot.freeze(); + + assertThrows(RuntimeException.class, () -> snapshot.append(expected.get(0))); + } + } + + private List> buildRecords(int recordsPerBatch, int batches) { + Random random = new Random(0); + List> result = new ArrayList<>(batches); + for (int i = 0; i < batches; i++) { + List batch = new ArrayList<>(recordsPerBatch); + for (int j = 0; j < recordsPerBatch; j++) { + batch.add(String.valueOf(random.nextInt())); + } + result.add(batch); + } + + return result; + } + + private void assertSnapshot(List> batches, RawSnapshotReader reader) { + List expected = new ArrayList<>(); + batches.forEach(expected::addAll); + + List actual = new ArrayList<>(expected.size()); + reader.forEach(batch -> { + batch.streamingIterator(new GrowableBufferSupplier()).forEachRemaining(record -> { + actual.add(Utils.utf8(record.value())); + }); + }); + + assertEquals(expected, actual); + } +} From 6f27bb02da0dec63a16c0c6aa456b47ba416eb19 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Mon, 7 Dec 2020 15:39:57 -0800 Subject: [PATCH 11/14] KAFKA-10818: Skip conversion to `Struct` when serializing generated requests/responses (#7409) Generated request/response classes have code to serialize/deserialize directly to `ByteBuffer` so the intermediate conversion to `Struct` can be skipped for them. We have recently completed the transition to generated request/response classes, so we can also remove the `Struct` based fallbacks. Additional noteworthy changes: * `AbstractRequest.parseRequest` has a more efficient computation of request size that relies on the received buffer instead of the parsed `Struct`. * Use `SendBuilder` for `AbstractRequest/Response` `toSend`, made the superclass implementation final and removed the overrides that are no longer necessary. * Removed request/response constructors that assume latest version as they are unsafe outside of tests. * Removed redundant version fields in requests/responses. * Removed unnecessary work in `OffsetFetchResponse`'s constructor when version >= 2. * Made `AbstractResponse.throttleTimeMs()` abstract. * Using `toSend` in `SaslClientAuthenticator` instead of `serialize`. * Various changes in Request/Response classes to make them more consistent and to rely on the Data classes as much as possible when it comes to their state. * Remove the version argument from `AbstractResponse.toString`. * Fix `getErrorResponse` for `ProduceRequest` and `DescribeClientQuotasRequest` to use `ApiError` which processes the error message sent back to the clients. This was uncovered by an accidental fix to a `RequestResponseTest` test (it was calling `AbstractResponse.toString` instead of `AbstractResponse.toString(short)`). Rely on existing protocol tests to ensure this refactoring does not change observed behavior (aside from improved performance). Reviewers: Chia-Ping Tsai --- .../apache/kafka/clients/ClientRequest.java | 6 +- .../apache/kafka/clients/NetworkClient.java | 5 +- .../kafka/clients/admin/KafkaAdminClient.java | 3 +- .../kafka/common/protocol/SendBuilder.java | 14 +- .../common/requests/AbstractRequest.java | 173 ++++++++------- .../requests/AbstractRequestResponse.java | 4 +- .../common/requests/AbstractResponse.java | 175 ++++++++------- .../requests/AddOffsetsToTxnRequest.java | 13 +- .../requests/AddOffsetsToTxnResponse.java | 17 +- .../requests/AddPartitionsToTxnRequest.java | 13 +- .../requests/AddPartitionsToTxnResponse.java | 14 +- .../requests/AlterClientQuotasRequest.java | 36 ++-- .../requests/AlterClientQuotasResponse.java | 61 ++---- .../common/requests/AlterConfigsRequest.java | 13 +- .../common/requests/AlterConfigsResponse.java | 17 +- .../common/requests/AlterIsrRequest.java | 13 +- .../common/requests/AlterIsrResponse.java | 14 +- .../AlterPartitionReassignmentsRequest.java | 24 +-- .../AlterPartitionReassignmentsResponse.java | 19 +- .../requests/AlterReplicaLogDirsRequest.java | 22 +- .../requests/AlterReplicaLogDirsResponse.java | 23 +- .../AlterUserScramCredentialsRequest.java | 24 +-- .../AlterUserScramCredentialsResponse.java | 20 +- .../kafka/common/requests/ApiError.java | 16 -- .../common/requests/ApiVersionsRequest.java | 14 +- .../common/requests/ApiVersionsResponse.java | 66 +----- .../requests/BeginQuorumEpochRequest.java | 16 +- .../requests/BeginQuorumEpochResponse.java | 31 ++- .../requests/ControlledShutdownRequest.java | 19 +- .../requests/ControlledShutdownResponse.java | 17 +- .../common/requests/CreateAclsRequest.java | 20 +- .../common/requests/CreateAclsResponse.java | 13 +- .../CreateDelegationTokenRequest.java | 15 +- .../CreateDelegationTokenResponse.java | 15 +- .../requests/CreatePartitionsRequest.java | 13 +- .../requests/CreatePartitionsResponse.java | 15 +- .../common/requests/CreateTopicsRequest.java | 19 +- .../common/requests/CreateTopicsResponse.java | 15 +- .../common/requests/DeleteAclsRequest.java | 19 +- .../common/requests/DeleteAclsResponse.java | 21 +- .../common/requests/DeleteGroupsRequest.java | 13 +- .../common/requests/DeleteGroupsResponse.java | 18 +- .../common/requests/DeleteRecordsRequest.java | 13 +- .../requests/DeleteRecordsResponse.java | 14 +- .../common/requests/DeleteTopicsRequest.java | 23 +- .../common/requests/DeleteTopicsResponse.java | 14 +- .../common/requests/DescribeAclsRequest.java | 16 +- .../common/requests/DescribeAclsResponse.java | 40 ++-- .../requests/DescribeClientQuotasRequest.java | 28 ++- .../DescribeClientQuotasResponse.java | 90 ++++---- .../requests/DescribeConfigsRequest.java | 15 +- .../requests/DescribeConfigsResponse.java | 20 +- .../DescribeDelegationTokenRequest.java | 20 +- .../DescribeDelegationTokenResponse.java | 15 +- .../requests/DescribeGroupsRequest.java | 19 +- .../requests/DescribeGroupsResponse.java | 20 +- .../requests/DescribeLogDirsRequest.java | 15 +- .../requests/DescribeLogDirsResponse.java | 14 +- .../requests/DescribeQuorumRequest.java | 16 +- .../requests/DescribeQuorumResponse.java | 24 ++- .../DescribeUserScramCredentialsRequest.java | 21 +- .../DescribeUserScramCredentialsResponse.java | 20 +- .../common/requests/ElectLeadersRequest.java | 14 +- .../common/requests/ElectLeadersResponse.java | 40 +--- .../requests/EndQuorumEpochRequest.java | 16 +- .../requests/EndQuorumEpochResponse.java | 29 ++- .../kafka/common/requests/EndTxnRequest.java | 13 +- .../kafka/common/requests/EndTxnResponse.java | 18 +- .../common/requests/EnvelopeRequest.java | 22 +- .../common/requests/EnvelopeResponse.java | 23 +- .../ExpireDelegationTokenRequest.java | 14 +- .../ExpireDelegationTokenResponse.java | 14 +- .../kafka/common/requests/FetchRequest.java | 34 +-- .../kafka/common/requests/FetchResponse.java | 21 +- .../requests/FindCoordinatorRequest.java | 16 +- .../requests/FindCoordinatorResponse.java | 14 +- .../common/requests/HeartbeatRequest.java | 14 +- .../common/requests/HeartbeatResponse.java | 14 +- .../IncrementalAlterConfigsRequest.java | 16 +- .../IncrementalAlterConfigsResponse.java | 17 +- .../requests/InitProducerIdRequest.java | 15 +- .../requests/InitProducerIdResponse.java | 13 +- .../common/requests/JoinGroupRequest.java | 15 +- .../common/requests/JoinGroupResponse.java | 21 +- .../common/requests/LeaderAndIsrRequest.java | 15 +- .../common/requests/LeaderAndIsrResponse.java | 18 +- .../common/requests/LeaveGroupRequest.java | 19 +- .../common/requests/LeaveGroupResponse.java | 31 ++- .../common/requests/ListGroupsRequest.java | 17 +- .../common/requests/ListGroupsResponse.java | 14 +- .../common/requests/ListOffsetRequest.java | 20 +- .../common/requests/ListOffsetResponse.java | 15 +- .../ListPartitionReassignmentsRequest.java | 23 +- .../ListPartitionReassignmentsResponse.java | 19 +- .../common/requests/MetadataRequest.java | 23 +- .../common/requests/MetadataResponse.java | 116 +++------- .../common/requests/OffsetCommitRequest.java | 19 +- .../common/requests/OffsetCommitResponse.java | 20 +- .../common/requests/OffsetDeleteRequest.java | 14 +- .../common/requests/OffsetDeleteResponse.java | 18 +- .../common/requests/OffsetFetchRequest.java | 13 +- .../common/requests/OffsetFetchResponse.java | 32 +-- .../OffsetsForLeaderEpochRequest.java | 16 +- .../OffsetsForLeaderEpochResponse.java | 15 +- .../kafka/common/requests/ProduceRequest.java | 25 +-- .../common/requests/ProduceResponse.java | 18 +- .../requests/RenewDelegationTokenRequest.java | 15 +- .../RenewDelegationTokenResponse.java | 14 +- .../kafka/common/requests/RequestContext.java | 11 +- .../kafka/common/requests/RequestHeader.java | 40 ++-- .../kafka/common/requests/RequestUtils.java | 58 +++-- .../kafka/common/requests/ResponseHeader.java | 26 ++- .../requests/SaslAuthenticateRequest.java | 23 +- .../requests/SaslAuthenticateResponse.java | 21 +- .../common/requests/SaslHandshakeRequest.java | 18 +- .../requests/SaslHandshakeResponse.java | 20 +- .../common/requests/StopReplicaRequest.java | 16 +- .../common/requests/StopReplicaResponse.java | 18 +- .../common/requests/SyncGroupRequest.java | 13 +- .../common/requests/SyncGroupResponse.java | 18 +- .../requests/TxnOffsetCommitRequest.java | 14 +- .../requests/TxnOffsetCommitResponse.java | 14 +- .../requests/UpdateFeaturesRequest.java | 16 +- .../requests/UpdateFeaturesResponse.java | 20 +- .../requests/UpdateMetadataRequest.java | 14 +- .../requests/UpdateMetadataResponse.java | 19 +- .../kafka/common/requests/VoteRequest.java | 17 +- .../kafka/common/requests/VoteResponse.java | 29 ++- .../requests/WriteTxnMarkersRequest.java | 13 +- .../requests/WriteTxnMarkersResponse.java | 51 +++-- .../SaslClientAuthenticator.java | 12 +- .../SaslServerAuthenticator.java | 9 +- .../apache/kafka/clients/MetadataTest.java | 23 +- .../kafka/clients/NetworkClientTest.java | 67 ++---- .../clients/admin/KafkaAdminClientTest.java | 85 +++++--- .../clients/consumer/KafkaConsumerTest.java | 2 +- .../internals/ConsumerCoordinatorTest.java | 2 +- .../internals/ConsumerMetadataTest.java | 4 +- .../consumer/internals/FetcherTest.java | 11 +- .../clients/producer/KafkaProducerTest.java | 2 +- .../producer/internals/SenderTest.java | 10 +- .../internals/TransactionManagerTest.java | 2 +- .../common/protocol/MessageTestUtil.java | 6 +- .../AddPartitionsToTxnResponseTest.java | 13 +- .../requests/CreateAclsRequestTest.java | 18 +- .../requests/DeleteAclsRequestTest.java | 12 +- .../requests/DeleteAclsResponseTest.java | 54 +++-- .../requests/DescribeAclsRequestTest.java | 24 +-- .../requests/DescribeAclsResponseTest.java | 14 +- .../common/requests/EndTxnResponseTest.java | 13 +- .../requests/LeaderAndIsrRequestTest.java | 8 +- .../requests/LeaveGroupResponseTest.java | 60 +++--- .../requests/ListOffsetRequestTest.java | 3 +- .../requests/OffsetCommitResponseTest.java | 30 +-- .../requests/OffsetFetchResponseTest.java | 6 +- .../OffsetsForLeaderEpochRequestTest.java | 5 +- .../common/requests/ProduceRequestTest.java | 2 +- .../common/requests/ProduceResponseTest.java | 19 +- .../common/requests/RequestContextTest.java | 6 +- .../common/requests/RequestHeaderTest.java | 28 ++- .../common/requests/RequestResponseTest.java | 203 +++++++++--------- .../requests/StopReplicaRequestTest.java | 12 +- .../requests/TxnOffsetCommitResponseTest.java | 29 ++- .../requests/UpdateMetadataRequestTest.java | 6 +- .../requests/WriteTxnMarkersRequestTest.java | 2 +- .../requests/WriteTxnMarkersResponseTest.java | 8 +- .../authenticator/SaslAuthenticatorTest.java | 14 +- .../SaslServerAuthenticatorTest.java | 32 +-- .../java/org/apache/kafka/test/TestUtils.java | 56 ++++- .../kafka/connect/util/TopicAdminTest.java | 3 +- .../controller/ControllerChannelManager.scala | 2 +- ...actionMarkerRequestCompletionHandler.scala | 3 +- .../scala/kafka/network/RequestChannel.scala | 14 +- .../main/scala/kafka/server/KafkaApis.scala | 77 +++++-- .../kafka/tools/TestRaftRequestHandler.scala | 2 +- .../kafka/api/AuthorizerIntegrationTest.scala | 4 +- .../kafka/network/RequestChannelTest.scala | 4 +- .../unit/kafka/network/SocketServerTest.scala | 14 +- .../AbstractCreateTopicsRequestTest.scala | 10 - .../server/BaseClientQuotaManagerTest.scala | 2 +- .../unit/kafka/server/BaseRequestTest.scala | 9 +- .../kafka/server/EdgeCaseRequestTest.scala | 7 +- .../kafka/server/ForwardingManagerTest.scala | 7 +- .../unit/kafka/server/KafkaApisTest.scala | 27 ++- .../unit/kafka/server/RequestQuotaTest.scala | 4 +- .../server/SaslApiVersionsRequestTest.scala | 6 +- .../ThrottledChannelExpirationTest.scala | 2 +- .../jmh/common/FetchRequestBenchmark.java | 11 +- .../metadata/MetadataRequestBenchmark.java | 2 +- .../producer/ProducerRequestBenchmark.java | 6 - .../producer/ProducerResponseBenchmark.java | 9 - 191 files changed, 1642 insertions(+), 2501 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java b/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java index cf3b911495cb5..abba79567eb1e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java +++ b/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java @@ -83,14 +83,14 @@ public ApiKeys apiKey() { } public RequestHeader makeHeader(short version) { - short requestApiKey = requestBuilder.apiKey().id; + ApiKeys requestApiKey = apiKey(); return new RequestHeader( new RequestHeaderData() - .setRequestApiKey(requestApiKey) + .setRequestApiKey(requestApiKey.id) .setRequestApiVersion(version) .setClientId(clientId) .setCorrelationId(correlationId), - ApiKeys.forId(requestApiKey).requestHeaderVersion(version)); + requestApiKey.requestHeaderVersion(version)); } public AbstractRequest.Builder requestBuilder() { diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java index 7ec3b93be4b2e..99bfd85d98530 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java @@ -870,9 +870,8 @@ private void handleCompletedReceives(List responses, long now) { InFlightRequest req = inFlightRequests.completeNext(source); AbstractResponse response = parseResponse(receive.payload(), req.header); - if (throttleTimeSensor != null) { - throttleTimeSensor.record(response.throttleTimeMs()); - } + if (throttleTimeSensor != null) + throttleTimeSensor.record(response.throttleTimeMs(), now); if (log.isDebugEnabled()) { log.debug("Received {} response from node {} for request with header {}: {}", diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 9c931cd7556f7..cf0a6f6949f90 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -1182,8 +1182,7 @@ private void handleResponses(long now, List responses) { try { call.handleResponse(response.responseBody()); if (log.isTraceEnabled()) - log.trace("{} got response {}", call, - response.responseBody().toString(response.requestHeader().apiVersion())); + log.trace("{} got response {}", call, response.responseBody()); } catch (Throwable t) { if (log.isTraceEnabled()) log.trace("{} handleResponse failed with {}", call, prettyPrintException(t)); diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/SendBuilder.java b/clients/src/main/java/org/apache/kafka/common/protocol/SendBuilder.java index 19d5600da89b8..6ed4c19a5719b 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/SendBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/SendBuilder.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.MultiRecordsSend; import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; import org.apache.kafka.common.requests.ResponseHeader; import org.apache.kafka.common.utils.ByteUtils; @@ -181,7 +182,7 @@ public Send build() { public static Send buildRequestSend( String destination, RequestHeader header, - ApiMessage apiRequest + Message apiRequest ) { return buildSend( destination, @@ -195,7 +196,7 @@ public static Send buildRequestSend( public static Send buildResponseSend( String destination, ResponseHeader header, - ApiMessage apiResponse, + Message apiResponse, short apiVersion ) { return buildSend( @@ -209,16 +210,13 @@ public static Send buildResponseSend( private static Send buildSend( String destination, - ApiMessage header, + Message header, short headerVersion, - ApiMessage apiMessage, + Message apiMessage, short apiVersion ) { ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - MessageSizeAccumulator messageSize = new MessageSizeAccumulator(); - - header.addSize(messageSize, serializationCache, headerVersion); - apiMessage.addSize(messageSize, serializationCache, apiVersion); + MessageSizeAccumulator messageSize = RequestUtils.size(serializationCache, header, headerVersion, apiMessage, apiVersion); int totalSize = messageSize.totalSize(); int sizeExcludingZeroCopyFields = totalSize - messageSize.zeroCopySize(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index a0121666dfcf2..e86cd54224ad7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -17,14 +17,12 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.errors.UnsupportedVersionException; -import org.apache.kafka.common.message.FetchRequestData; -import org.apache.kafka.common.message.AlterIsrRequestData; -import org.apache.kafka.common.message.ProduceRequestData; -import org.apache.kafka.common.network.NetworkSend; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.ObjectSerializationCache; +import org.apache.kafka.common.protocol.SendBuilder; import java.nio.ByteBuffer; import java.util.Map; @@ -79,13 +77,13 @@ public T build() { } private final short version; - public final ApiKeys api; + private final ApiKeys apiKey; - public AbstractRequest(ApiKeys api, short version) { - if (!api.isVersionSupported(version)) - throw new UnsupportedVersionException("The " + api + " protocol does not support version " + version); + public AbstractRequest(ApiKeys apiKey, short version) { + if (!apiKey.isVersionSupported(version)) + throw new UnsupportedVersionException("The " + apiKey + " protocol does not support version " + version); this.version = version; - this.api = api; + this.apiKey = apiKey; } /** @@ -95,21 +93,33 @@ public short version() { return version; } - public Send toSend(String destination, RequestHeader header) { - return new NetworkSend(destination, serialize(header)); + public ApiKeys apiKey() { + return apiKey; } - /** - * Use with care, typically {@link #toSend(String, RequestHeader)} should be used instead. - */ - public ByteBuffer serialize(RequestHeader header) { - return RequestUtils.serialize(header.toStruct(), toStruct()); + public final Send toSend(String destination, RequestHeader header) { + return SendBuilder.buildRequestSend(destination, header, data()); + } + + // Visible for testing + public final ByteBuffer serializeWithHeader(RequestHeader header) { + return RequestUtils.serialize(header.data(), header.headerVersion(), data(), version); + } + + protected abstract Message data(); + + // Visible for testing + public final ByteBuffer serializeBody() { + return RequestUtils.serialize(null, (short) 0, data(), version); } - protected abstract Struct toStruct(); + // Visible for testing + final int sizeInBytes() { + return data().size(new ObjectSerializationCache(), version); + } public String toString(boolean verbose) { - return toStruct().toString(); + return data().toString(); } @Override @@ -144,126 +154,131 @@ public Map errorCounts(Throwable e) { /** * Factory method for getting a request object based on ApiKey ID and a version */ - public static AbstractRequest parseRequest(ApiKeys apiKey, short apiVersion, Struct struct) { + public static RequestAndSize parseRequest(ApiKeys apiKey, short apiVersion, ByteBuffer buffer) { + int bufferSize = buffer.remaining(); + return new RequestAndSize(doParseRequest(apiKey, apiVersion, buffer), bufferSize); + } + + private static AbstractRequest doParseRequest(ApiKeys apiKey, short apiVersion, ByteBuffer buffer) { switch (apiKey) { case PRODUCE: - return new ProduceRequest(new ProduceRequestData(struct, apiVersion), apiVersion); + return ProduceRequest.parse(buffer, apiVersion); case FETCH: - return new FetchRequest(new FetchRequestData(struct, apiVersion), apiVersion); + return FetchRequest.parse(buffer, apiVersion); case LIST_OFFSETS: - return new ListOffsetRequest(struct, apiVersion); + return ListOffsetRequest.parse(buffer, apiVersion); case METADATA: - return new MetadataRequest(struct, apiVersion); + return MetadataRequest.parse(buffer, apiVersion); case OFFSET_COMMIT: - return new OffsetCommitRequest(struct, apiVersion); + return OffsetCommitRequest.parse(buffer, apiVersion); case OFFSET_FETCH: - return new OffsetFetchRequest(struct, apiVersion); + return OffsetFetchRequest.parse(buffer, apiVersion); case FIND_COORDINATOR: - return new FindCoordinatorRequest(struct, apiVersion); + return FindCoordinatorRequest.parse(buffer, apiVersion); case JOIN_GROUP: - return new JoinGroupRequest(struct, apiVersion); + return JoinGroupRequest.parse(buffer, apiVersion); case HEARTBEAT: - return new HeartbeatRequest(struct, apiVersion); + return HeartbeatRequest.parse(buffer, apiVersion); case LEAVE_GROUP: - return new LeaveGroupRequest(struct, apiVersion); + return LeaveGroupRequest.parse(buffer, apiVersion); case SYNC_GROUP: - return new SyncGroupRequest(struct, apiVersion); + return SyncGroupRequest.parse(buffer, apiVersion); case STOP_REPLICA: - return new StopReplicaRequest(struct, apiVersion); + return StopReplicaRequest.parse(buffer, apiVersion); case CONTROLLED_SHUTDOWN: - return new ControlledShutdownRequest(struct, apiVersion); + return ControlledShutdownRequest.parse(buffer, apiVersion); case UPDATE_METADATA: - return new UpdateMetadataRequest(struct, apiVersion); + return UpdateMetadataRequest.parse(buffer, apiVersion); case LEADER_AND_ISR: - return new LeaderAndIsrRequest(struct, apiVersion); + return LeaderAndIsrRequest.parse(buffer, apiVersion); case DESCRIBE_GROUPS: - return new DescribeGroupsRequest(struct, apiVersion); + return DescribeGroupsRequest.parse(buffer, apiVersion); case LIST_GROUPS: - return new ListGroupsRequest(struct, apiVersion); + return ListGroupsRequest.parse(buffer, apiVersion); case SASL_HANDSHAKE: - return new SaslHandshakeRequest(struct, apiVersion); + return SaslHandshakeRequest.parse(buffer, apiVersion); case API_VERSIONS: - return new ApiVersionsRequest(struct, apiVersion); + return ApiVersionsRequest.parse(buffer, apiVersion); case CREATE_TOPICS: - return new CreateTopicsRequest(struct, apiVersion); + return CreateTopicsRequest.parse(buffer, apiVersion); case DELETE_TOPICS: - return new DeleteTopicsRequest(struct, apiVersion); + return DeleteTopicsRequest.parse(buffer, apiVersion); case DELETE_RECORDS: - return new DeleteRecordsRequest(struct, apiVersion); + return DeleteRecordsRequest.parse(buffer, apiVersion); case INIT_PRODUCER_ID: - return new InitProducerIdRequest(struct, apiVersion); + return InitProducerIdRequest.parse(buffer, apiVersion); case OFFSET_FOR_LEADER_EPOCH: - return new OffsetsForLeaderEpochRequest(struct, apiVersion); + return OffsetsForLeaderEpochRequest.parse(buffer, apiVersion); case ADD_PARTITIONS_TO_TXN: - return new AddPartitionsToTxnRequest(struct, apiVersion); + return AddPartitionsToTxnRequest.parse(buffer, apiVersion); case ADD_OFFSETS_TO_TXN: - return new AddOffsetsToTxnRequest(struct, apiVersion); + return AddOffsetsToTxnRequest.parse(buffer, apiVersion); case END_TXN: - return new EndTxnRequest(struct, apiVersion); + return EndTxnRequest.parse(buffer, apiVersion); case WRITE_TXN_MARKERS: - return new WriteTxnMarkersRequest(struct, apiVersion); + return WriteTxnMarkersRequest.parse(buffer, apiVersion); case TXN_OFFSET_COMMIT: - return new TxnOffsetCommitRequest(struct, apiVersion); + return TxnOffsetCommitRequest.parse(buffer, apiVersion); case DESCRIBE_ACLS: - return new DescribeAclsRequest(struct, apiVersion); + return DescribeAclsRequest.parse(buffer, apiVersion); case CREATE_ACLS: - return new CreateAclsRequest(struct, apiVersion); + return CreateAclsRequest.parse(buffer, apiVersion); case DELETE_ACLS: - return new DeleteAclsRequest(struct, apiVersion); + return DeleteAclsRequest.parse(buffer, apiVersion); case DESCRIBE_CONFIGS: - return new DescribeConfigsRequest(struct, apiVersion); + return DescribeConfigsRequest.parse(buffer, apiVersion); case ALTER_CONFIGS: - return new AlterConfigsRequest(struct, apiVersion); + return AlterConfigsRequest.parse(buffer, apiVersion); case ALTER_REPLICA_LOG_DIRS: - return new AlterReplicaLogDirsRequest(struct, apiVersion); + return AlterReplicaLogDirsRequest.parse(buffer, apiVersion); case DESCRIBE_LOG_DIRS: - return new DescribeLogDirsRequest(struct, apiVersion); + return DescribeLogDirsRequest.parse(buffer, apiVersion); case SASL_AUTHENTICATE: - return new SaslAuthenticateRequest(struct, apiVersion); + return SaslAuthenticateRequest.parse(buffer, apiVersion); case CREATE_PARTITIONS: - return new CreatePartitionsRequest(struct, apiVersion); + return CreatePartitionsRequest.parse(buffer, apiVersion); case CREATE_DELEGATION_TOKEN: - return new CreateDelegationTokenRequest(struct, apiVersion); + return CreateDelegationTokenRequest.parse(buffer, apiVersion); case RENEW_DELEGATION_TOKEN: - return new RenewDelegationTokenRequest(struct, apiVersion); + return RenewDelegationTokenRequest.parse(buffer, apiVersion); case EXPIRE_DELEGATION_TOKEN: - return new ExpireDelegationTokenRequest(struct, apiVersion); + return ExpireDelegationTokenRequest.parse(buffer, apiVersion); case DESCRIBE_DELEGATION_TOKEN: - return new DescribeDelegationTokenRequest(struct, apiVersion); + return DescribeDelegationTokenRequest.parse(buffer, apiVersion); case DELETE_GROUPS: - return new DeleteGroupsRequest(struct, apiVersion); + return DeleteGroupsRequest.parse(buffer, apiVersion); case ELECT_LEADERS: - return new ElectLeadersRequest(struct, apiVersion); + return ElectLeadersRequest.parse(buffer, apiVersion); case INCREMENTAL_ALTER_CONFIGS: - return new IncrementalAlterConfigsRequest(struct, apiVersion); + return IncrementalAlterConfigsRequest.parse(buffer, apiVersion); case ALTER_PARTITION_REASSIGNMENTS: - return new AlterPartitionReassignmentsRequest(struct, apiVersion); + return AlterPartitionReassignmentsRequest.parse(buffer, apiVersion); case LIST_PARTITION_REASSIGNMENTS: - return new ListPartitionReassignmentsRequest(struct, apiVersion); + return ListPartitionReassignmentsRequest.parse(buffer, apiVersion); case OFFSET_DELETE: - return new OffsetDeleteRequest(struct, apiVersion); + return OffsetDeleteRequest.parse(buffer, apiVersion); case DESCRIBE_CLIENT_QUOTAS: - return new DescribeClientQuotasRequest(struct, apiVersion); + return DescribeClientQuotasRequest.parse(buffer, apiVersion); case ALTER_CLIENT_QUOTAS: - return new AlterClientQuotasRequest(struct, apiVersion); + return AlterClientQuotasRequest.parse(buffer, apiVersion); case DESCRIBE_USER_SCRAM_CREDENTIALS: - return new DescribeUserScramCredentialsRequest(struct, apiVersion); + return DescribeUserScramCredentialsRequest.parse(buffer, apiVersion); case ALTER_USER_SCRAM_CREDENTIALS: - return new AlterUserScramCredentialsRequest(struct, apiVersion); + return AlterUserScramCredentialsRequest.parse(buffer, apiVersion); case VOTE: - return new VoteRequest(struct, apiVersion); + return VoteRequest.parse(buffer, apiVersion); case BEGIN_QUORUM_EPOCH: - return new BeginQuorumEpochRequest(struct, apiVersion); + return BeginQuorumEpochRequest.parse(buffer, apiVersion); case END_QUORUM_EPOCH: - return new EndQuorumEpochRequest(struct, apiVersion); + return EndQuorumEpochRequest.parse(buffer, apiVersion); case DESCRIBE_QUORUM: - return new DescribeQuorumRequest(struct, apiVersion); + return DescribeQuorumRequest.parse(buffer, apiVersion); case ALTER_ISR: - return new AlterIsrRequest(new AlterIsrRequestData(struct, apiVersion), apiVersion); + return AlterIsrRequest.parse(buffer, apiVersion); case UPDATE_FEATURES: - return new UpdateFeaturesRequest(struct, apiVersion); + return UpdateFeaturesRequest.parse(buffer, apiVersion); case ENVELOPE: - return new EnvelopeRequest(struct, apiVersion); + return EnvelopeRequest.parse(buffer, apiVersion); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseRequest`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java index 39698a1d20b74..5d655bd0075dd 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java @@ -16,6 +16,4 @@ */ package org.apache.kafka.common.requests; -public interface AbstractRequestResponse { - -} +public interface AbstractRequestResponse { } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index 11566073374cf..747698166359c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -16,38 +16,49 @@ */ package org.apache.kafka.common.requests; -import org.apache.kafka.common.message.EnvelopeResponseData; -import org.apache.kafka.common.message.FetchResponseData; -import org.apache.kafka.common.message.AlterIsrResponseData; -import org.apache.kafka.common.message.ProduceResponseData; -import org.apache.kafka.common.network.NetworkSend; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.SendBuilder; import java.nio.ByteBuffer; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; public abstract class AbstractResponse implements AbstractRequestResponse { public static final int DEFAULT_THROTTLE_TIME = 0; - protected Send toSend(String destination, ResponseHeader header, short apiVersion) { - return new NetworkSend(destination, RequestUtils.serialize(header.toStruct(), toStruct(apiVersion))); + private final ApiKeys apiKey; + + protected AbstractResponse(ApiKeys apiKey) { + this.apiKey = apiKey; + } + + public final Send toSend(String destination, ResponseHeader header, short version) { + return SendBuilder.buildResponseSend(destination, header, data(), version); } /** * Visible for testing, typically {@link #toSend(String, ResponseHeader, short)} should be used instead. */ - public ByteBuffer serialize(ApiKeys apiKey, short version, int correlationId) { - ResponseHeader header = - new ResponseHeader(correlationId, apiKey.responseHeaderVersion(version)); - return RequestUtils.serialize(header.toStruct(), toStruct(version)); + public final ByteBuffer serializeWithHeader(short version, int correlationId) { + return serializeWithHeader(new ResponseHeader(correlationId, apiKey.responseHeaderVersion(version)), version); + } + + final ByteBuffer serializeWithHeader(ResponseHeader header, short version) { + Objects.requireNonNull(header, "header should not be null"); + return RequestUtils.serialize(header.data(), header.headerVersion(), data(), version); + } + + // Visible for testing + final ByteBuffer serializeBody(short version) { + return RequestUtils.serialize(null, (short) 0, data(), version); } /** @@ -84,17 +95,18 @@ protected void updateErrorCounts(Map errorCounts, Errors error) errorCounts.put(error, count + 1); } - protected abstract Struct toStruct(short version); + protected abstract Message data(); /** * Parse a response from the provided buffer. The buffer is expected to hold both * the {@link ResponseHeader} as well as the response payload. */ - public static AbstractResponse parseResponse(ByteBuffer byteBuffer, RequestHeader requestHeader) { + public static AbstractResponse parseResponse(ByteBuffer buffer, RequestHeader requestHeader) { ApiKeys apiKey = requestHeader.apiKey(); short apiVersion = requestHeader.apiVersion(); - ResponseHeader responseHeader = ResponseHeader.parse(byteBuffer, apiKey.responseHeaderVersion(apiVersion)); + ResponseHeader responseHeader = ResponseHeader.parse(buffer, apiKey.responseHeaderVersion(apiVersion)); + if (requestHeader.correlationId() != responseHeader.correlationId()) { throw new CorrelationIdMismatchException("Correlation id for response (" + responseHeader.correlationId() + ") does not match request (" @@ -102,130 +114,129 @@ public static AbstractResponse parseResponse(ByteBuffer byteBuffer, RequestHeade requestHeader.correlationId(), responseHeader.correlationId()); } - Struct struct = apiKey.parseResponse(apiVersion, byteBuffer); - return AbstractResponse.parseResponse(apiKey, struct, apiVersion); + return AbstractResponse.parseResponse(apiKey, buffer, apiVersion); } - public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, short version) { + public static AbstractResponse parseResponse(ApiKeys apiKey, ByteBuffer responseBuffer, short version) { switch (apiKey) { case PRODUCE: - return new ProduceResponse(new ProduceResponseData(struct, version)); + return ProduceResponse.parse(responseBuffer, version); case FETCH: - return new FetchResponse<>(new FetchResponseData(struct, version)); + return FetchResponse.parse(responseBuffer, version); case LIST_OFFSETS: - return new ListOffsetResponse(struct, version); + return ListOffsetResponse.parse(responseBuffer, version); case METADATA: - return new MetadataResponse(struct, version); + return MetadataResponse.parse(responseBuffer, version); case OFFSET_COMMIT: - return new OffsetCommitResponse(struct, version); + return OffsetCommitResponse.parse(responseBuffer, version); case OFFSET_FETCH: - return new OffsetFetchResponse(struct, version); + return OffsetFetchResponse.parse(responseBuffer, version); case FIND_COORDINATOR: - return new FindCoordinatorResponse(struct, version); + return FindCoordinatorResponse.parse(responseBuffer, version); case JOIN_GROUP: - return new JoinGroupResponse(struct, version); + return JoinGroupResponse.parse(responseBuffer, version); case HEARTBEAT: - return new HeartbeatResponse(struct, version); + return HeartbeatResponse.parse(responseBuffer, version); case LEAVE_GROUP: - return new LeaveGroupResponse(struct, version); + return LeaveGroupResponse.parse(responseBuffer, version); case SYNC_GROUP: - return new SyncGroupResponse(struct, version); + return SyncGroupResponse.parse(responseBuffer, version); case STOP_REPLICA: - return new StopReplicaResponse(struct, version); + return StopReplicaResponse.parse(responseBuffer, version); case CONTROLLED_SHUTDOWN: - return new ControlledShutdownResponse(struct, version); + return ControlledShutdownResponse.parse(responseBuffer, version); case UPDATE_METADATA: - return new UpdateMetadataResponse(struct, version); + return UpdateMetadataResponse.parse(responseBuffer, version); case LEADER_AND_ISR: - return new LeaderAndIsrResponse(struct, version); + return LeaderAndIsrResponse.parse(responseBuffer, version); case DESCRIBE_GROUPS: - return new DescribeGroupsResponse(struct, version); + return DescribeGroupsResponse.parse(responseBuffer, version); case LIST_GROUPS: - return new ListGroupsResponse(struct, version); + return ListGroupsResponse.parse(responseBuffer, version); case SASL_HANDSHAKE: - return new SaslHandshakeResponse(struct, version); + return SaslHandshakeResponse.parse(responseBuffer, version); case API_VERSIONS: - return ApiVersionsResponse.fromStruct(struct, version); + return ApiVersionsResponse.parse(responseBuffer, version); case CREATE_TOPICS: - return new CreateTopicsResponse(struct, version); + return CreateTopicsResponse.parse(responseBuffer, version); case DELETE_TOPICS: - return new DeleteTopicsResponse(struct, version); + return DeleteTopicsResponse.parse(responseBuffer, version); case DELETE_RECORDS: - return new DeleteRecordsResponse(struct, version); + return DeleteRecordsResponse.parse(responseBuffer, version); case INIT_PRODUCER_ID: - return new InitProducerIdResponse(struct, version); + return InitProducerIdResponse.parse(responseBuffer, version); case OFFSET_FOR_LEADER_EPOCH: - return new OffsetsForLeaderEpochResponse(struct, version); + return OffsetsForLeaderEpochResponse.parse(responseBuffer, version); case ADD_PARTITIONS_TO_TXN: - return new AddPartitionsToTxnResponse(struct, version); + return AddPartitionsToTxnResponse.parse(responseBuffer, version); case ADD_OFFSETS_TO_TXN: - return new AddOffsetsToTxnResponse(struct, version); + return AddOffsetsToTxnResponse.parse(responseBuffer, version); case END_TXN: - return new EndTxnResponse(struct, version); + return EndTxnResponse.parse(responseBuffer, version); case WRITE_TXN_MARKERS: - return new WriteTxnMarkersResponse(struct, version); + return WriteTxnMarkersResponse.parse(responseBuffer, version); case TXN_OFFSET_COMMIT: - return new TxnOffsetCommitResponse(struct, version); + return TxnOffsetCommitResponse.parse(responseBuffer, version); case DESCRIBE_ACLS: - return new DescribeAclsResponse(struct, version); + return DescribeAclsResponse.parse(responseBuffer, version); case CREATE_ACLS: - return new CreateAclsResponse(struct, version); + return CreateAclsResponse.parse(responseBuffer, version); case DELETE_ACLS: - return new DeleteAclsResponse(struct, version); + return DeleteAclsResponse.parse(responseBuffer, version); case DESCRIBE_CONFIGS: - return new DescribeConfigsResponse(struct, version); + return DescribeConfigsResponse.parse(responseBuffer, version); case ALTER_CONFIGS: - return new AlterConfigsResponse(struct, version); + return AlterConfigsResponse.parse(responseBuffer, version); case ALTER_REPLICA_LOG_DIRS: - return new AlterReplicaLogDirsResponse(struct, version); + return AlterReplicaLogDirsResponse.parse(responseBuffer, version); case DESCRIBE_LOG_DIRS: - return new DescribeLogDirsResponse(struct, version); + return DescribeLogDirsResponse.parse(responseBuffer, version); case SASL_AUTHENTICATE: - return new SaslAuthenticateResponse(struct, version); + return SaslAuthenticateResponse.parse(responseBuffer, version); case CREATE_PARTITIONS: - return new CreatePartitionsResponse(struct, version); + return CreatePartitionsResponse.parse(responseBuffer, version); case CREATE_DELEGATION_TOKEN: - return new CreateDelegationTokenResponse(struct, version); + return CreateDelegationTokenResponse.parse(responseBuffer, version); case RENEW_DELEGATION_TOKEN: - return new RenewDelegationTokenResponse(struct, version); + return RenewDelegationTokenResponse.parse(responseBuffer, version); case EXPIRE_DELEGATION_TOKEN: - return new ExpireDelegationTokenResponse(struct, version); + return ExpireDelegationTokenResponse.parse(responseBuffer, version); case DESCRIBE_DELEGATION_TOKEN: - return new DescribeDelegationTokenResponse(struct, version); + return DescribeDelegationTokenResponse.parse(responseBuffer, version); case DELETE_GROUPS: - return new DeleteGroupsResponse(struct, version); + return DeleteGroupsResponse.parse(responseBuffer, version); case ELECT_LEADERS: - return new ElectLeadersResponse(struct, version); + return ElectLeadersResponse.parse(responseBuffer, version); case INCREMENTAL_ALTER_CONFIGS: - return new IncrementalAlterConfigsResponse(struct, version); + return IncrementalAlterConfigsResponse.parse(responseBuffer, version); case ALTER_PARTITION_REASSIGNMENTS: - return new AlterPartitionReassignmentsResponse(struct, version); + return AlterPartitionReassignmentsResponse.parse(responseBuffer, version); case LIST_PARTITION_REASSIGNMENTS: - return new ListPartitionReassignmentsResponse(struct, version); + return ListPartitionReassignmentsResponse.parse(responseBuffer, version); case OFFSET_DELETE: - return new OffsetDeleteResponse(struct, version); + return OffsetDeleteResponse.parse(responseBuffer, version); case DESCRIBE_CLIENT_QUOTAS: - return new DescribeClientQuotasResponse(struct, version); + return DescribeClientQuotasResponse.parse(responseBuffer, version); case ALTER_CLIENT_QUOTAS: - return new AlterClientQuotasResponse(struct, version); + return AlterClientQuotasResponse.parse(responseBuffer, version); case DESCRIBE_USER_SCRAM_CREDENTIALS: - return new DescribeUserScramCredentialsResponse(struct, version); + return DescribeUserScramCredentialsResponse.parse(responseBuffer, version); case ALTER_USER_SCRAM_CREDENTIALS: - return new AlterUserScramCredentialsResponse(struct, version); + return AlterUserScramCredentialsResponse.parse(responseBuffer, version); case VOTE: - return new VoteResponse(struct, version); + return VoteResponse.parse(responseBuffer, version); case BEGIN_QUORUM_EPOCH: - return new BeginQuorumEpochResponse(struct, version); + return BeginQuorumEpochResponse.parse(responseBuffer, version); case END_QUORUM_EPOCH: - return new EndQuorumEpochResponse(struct, version); + return EndQuorumEpochResponse.parse(responseBuffer, version); case DESCRIBE_QUORUM: - return new DescribeQuorumResponse(struct, version); + return DescribeQuorumResponse.parse(responseBuffer, version); case ALTER_ISR: - return new AlterIsrResponse(new AlterIsrResponseData(struct, version)); + return AlterIsrResponse.parse(responseBuffer, version); case UPDATE_FEATURES: - return new UpdateFeaturesResponse(struct, version); + return UpdateFeaturesResponse.parse(responseBuffer, version); case ENVELOPE: - return new EnvelopeResponse(new EnvelopeResponseData(struct, version)); + return EnvelopeResponse.parse(responseBuffer, version); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseResponse`, the " + "code should be updated to do so.", apiKey)); @@ -241,11 +252,13 @@ public boolean shouldClientThrottle(short version) { return false; } - public int throttleTimeMs() { - return DEFAULT_THROTTLE_TIME; + public ApiKeys apiKey() { + return apiKey; } - public String toString(short version) { - return toStruct(version).toString(); + public abstract int throttleTimeMs(); + + public String toString() { + return data().toString(); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java index 3b1c746c5b5c9..4ad56aa94268f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.AddOffsetsToTxnRequestData; import org.apache.kafka.common.message.AddOffsetsToTxnResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -52,14 +52,9 @@ public AddOffsetsToTxnRequest(AddOffsetsToTxnRequestData data, short version) { this.data = data; } - public AddOffsetsToTxnRequest(Struct struct, short version) { - super(ApiKeys.ADD_OFFSETS_TO_TXN, version); - this.data = new AddOffsetsToTxnRequestData(struct, version); - } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected AddOffsetsToTxnRequestData data() { + return data; } @Override @@ -70,6 +65,6 @@ public AddOffsetsToTxnResponse getErrorResponse(int throttleTimeMs, Throwable e) } public static AddOffsetsToTxnRequest parse(ByteBuffer buffer, short version) { - return new AddOffsetsToTxnRequest(ApiKeys.ADD_OFFSETS_TO_TXN.parseRequest(version, buffer), version); + return new AddOffsetsToTxnRequest(new AddOffsetsToTxnRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java index 99864f4bffd4a..6908e51960609 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.AddOffsetsToTxnResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -42,30 +42,27 @@ public class AddOffsetsToTxnResponse extends AbstractResponse { public AddOffsetsToTxnResponseData data; public AddOffsetsToTxnResponse(AddOffsetsToTxnResponseData data) { + super(ApiKeys.ADD_OFFSETS_TO_TXN); this.data = data; } - public AddOffsetsToTxnResponse(Struct struct, short version) { - this.data = new AddOffsetsToTxnResponseData(struct, version); - } - @Override public Map errorCounts() { return errorCounts(Errors.forCode(data.errorCode())); } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public int throttleTimeMs() { + return data.throttleTimeMs(); } @Override - public int throttleTimeMs() { - return data.throttleTimeMs(); + protected AddOffsetsToTxnResponseData data() { + return data; } public static AddOffsetsToTxnResponse parse(ByteBuffer buffer, short version) { - return new AddOffsetsToTxnResponse(ApiKeys.ADD_OFFSETS_TO_TXN.parseResponse(version, buffer), version); + return new AddOffsetsToTxnResponse(new AddOffsetsToTxnResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java index 13be2127cdd4d..57645789ca000 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopic; import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopicCollection; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -103,11 +103,6 @@ public AddPartitionsToTxnRequest(final AddPartitionsToTxnRequestData data, short this.data = data; } - public AddPartitionsToTxnRequest(Struct struct, short version) { - super(ApiKeys.ADD_PARTITIONS_TO_TXN, version); - this.data = new AddPartitionsToTxnRequestData(struct, version); - } - public List partitions() { if (cachedPartitions != null) { return cachedPartitions; @@ -117,8 +112,8 @@ public List partitions() { } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected AddPartitionsToTxnRequestData data() { + return data; } @Override @@ -131,6 +126,6 @@ public AddPartitionsToTxnResponse getErrorResponse(int throttleTimeMs, Throwable } public static AddPartitionsToTxnRequest parse(ByteBuffer buffer, short version) { - return new AddPartitionsToTxnRequest(ApiKeys.ADD_PARTITIONS_TO_TXN.parseRequest(version, buffer), version); + return new AddPartitionsToTxnRequest(new AddPartitionsToTxnRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java index cc79ce8ca4200..c6da3af0a9319 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java @@ -23,8 +23,8 @@ import org.apache.kafka.common.message.AddPartitionsToTxnResponseData.AddPartitionsToTxnTopicResult; import org.apache.kafka.common.message.AddPartitionsToTxnResponseData.AddPartitionsToTxnTopicResultCollection; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -50,11 +50,13 @@ public class AddPartitionsToTxnResponse extends AbstractResponse { private Map cachedErrorsMap = null; - public AddPartitionsToTxnResponse(Struct struct, short version) { - this.data = new AddPartitionsToTxnResponseData(struct, version); + public AddPartitionsToTxnResponse(AddPartitionsToTxnResponseData data) { + super(ApiKeys.ADD_PARTITIONS_TO_TXN); + this.data = data; } public AddPartitionsToTxnResponse(int throttleTimeMs, Map errors) { + super(ApiKeys.ADD_PARTITIONS_TO_TXN); Map resultMap = new HashMap<>(); @@ -115,12 +117,12 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected AddPartitionsToTxnResponseData data() { + return data; } public static AddPartitionsToTxnResponse parse(ByteBuffer buffer, short version) { - return new AddPartitionsToTxnResponse(ApiKeys.ADD_PARTITIONS_TO_TXN.parseResponse(version, buffer), version); + return new AddPartitionsToTxnResponse(new AddPartitionsToTxnResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java index 6d4c2a1f357c3..7178475123633 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java @@ -20,11 +20,13 @@ import org.apache.kafka.common.message.AlterClientQuotasRequestData.EntityData; import org.apache.kafka.common.message.AlterClientQuotasRequestData.EntryData; import org.apache.kafka.common.message.AlterClientQuotasRequestData.OpData; +import org.apache.kafka.common.message.AlterClientQuotasResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.quota.ClientQuotaAlteration; import org.apache.kafka.common.quota.ClientQuotaEntity; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -85,12 +87,7 @@ public AlterClientQuotasRequest(AlterClientQuotasRequestData data, short version this.data = data; } - public AlterClientQuotasRequest(Struct struct, short version) { - super(ApiKeys.ALTER_CLIENT_QUOTAS, version); - this.data = new AlterClientQuotasRequestData(struct, version); - } - - public Collection entries() { + public List entries() { List entries = new ArrayList<>(data.entries().size()); for (EntryData entryData : data.entries()) { Map entity = new HashMap<>(entryData.entity().size()); @@ -113,21 +110,30 @@ public boolean validateOnly() { return data.validateOnly(); } + @Override + protected AlterClientQuotasRequestData data() { + return data; + } + @Override public AlterClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { - ArrayList entities = new ArrayList<>(data.entries().size()); + List responseEntries = new ArrayList<>(); for (EntryData entryData : data.entries()) { - Map entity = new HashMap<>(entryData.entity().size()); + List responseEntities = new ArrayList<>(); for (EntityData entityData : entryData.entity()) { - entity.put(entityData.entityType(), entityData.entityName()); + responseEntities.add(new AlterClientQuotasResponseData.EntityData() + .setEntityType(entityData.entityType()) + .setEntityName(entityData.entityName())); } - entities.add(new ClientQuotaEntity(entity)); + responseEntries.add(new AlterClientQuotasResponseData.EntryData().setEntity(responseEntities)); } - return new AlterClientQuotasResponse(entities, throttleTimeMs, e); + AlterClientQuotasResponseData responseData = new AlterClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setEntries(responseEntries); + return new AlterClientQuotasResponse(responseData); } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + public static AlterClientQuotasRequest parse(ByteBuffer buffer, short version) { + return new AlterClientQuotasRequest(new AlterClientQuotasRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java index 3e95671a6f939..bce548822d895 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java @@ -21,13 +21,12 @@ import org.apache.kafka.common.message.AlterClientQuotasResponseData.EntityData; import org.apache.kafka.common.message.AlterClientQuotasResponseData.EntryData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.quota.ClientQuotaEntity; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -36,39 +35,9 @@ public class AlterClientQuotasResponse extends AbstractResponse { private final AlterClientQuotasResponseData data; - public AlterClientQuotasResponse(Map result, int throttleTimeMs) { - List entries = new ArrayList<>(result.size()); - for (Map.Entry entry : result.entrySet()) { - ApiError e = entry.getValue(); - entries.add(new EntryData() - .setErrorCode(e.error().code()) - .setErrorMessage(e.message()) - .setEntity(toEntityData(entry.getKey()))); - } - - this.data = new AlterClientQuotasResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setEntries(entries); - } - - public AlterClientQuotasResponse(Collection entities, int throttleTimeMs, Throwable e) { - ApiError apiError = ApiError.fromThrowable(e); - - List entries = new ArrayList<>(entities.size()); - for (ClientQuotaEntity entity : entities) { - entries.add(new EntryData() - .setErrorCode(apiError.error().code()) - .setErrorMessage(apiError.message()) - .setEntity(toEntityData(entity))); - } - - this.data = new AlterClientQuotasResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setEntries(entries); - } - - public AlterClientQuotasResponse(Struct struct, short version) { - this.data = new AlterClientQuotasResponseData(struct, version); + public AlterClientQuotasResponse(AlterClientQuotasResponseData data) { + super(ApiKeys.ALTER_CLIENT_QUOTAS); + this.data = data; } public void complete(Map> futures) { @@ -108,8 +77,8 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected AlterClientQuotasResponseData data() { + return data; } private static List toEntityData(ClientQuotaEntity entity) { @@ -123,6 +92,22 @@ private static List toEntityData(ClientQuotaEntity entity) { } public static AlterClientQuotasResponse parse(ByteBuffer buffer, short version) { - return new AlterClientQuotasResponse(ApiKeys.ALTER_CLIENT_QUOTAS.parseResponse(version, buffer), version); + return new AlterClientQuotasResponse(new AlterClientQuotasResponseData(new ByteBufferAccessor(buffer), version)); } + + public static AlterClientQuotasResponse fromQuotaEntities(Map result, int throttleTimeMs) { + List entries = new ArrayList<>(result.size()); + for (Map.Entry entry : result.entrySet()) { + ApiError e = entry.getValue(); + entries.add(new EntryData() + .setErrorCode(e.error().code()) + .setErrorMessage(e.message()) + .setEntity(toEntityData(entry.getKey()))); + } + + return new AlterClientQuotasResponse(new AlterClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setEntries(entries)); + } + } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java index 1ee3330a71ee5..f30e8b9c1ffe9 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java @@ -21,7 +21,7 @@ import org.apache.kafka.common.message.AlterConfigsRequestData; import org.apache.kafka.common.message.AlterConfigsResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; import java.util.Collection; @@ -97,11 +97,6 @@ public AlterConfigsRequest(AlterConfigsRequestData data, short version) { this.data = data; } - public AlterConfigsRequest(Struct struct, short version) { - super(ApiKeys.ALTER_CONFIGS, version); - this.data = new AlterConfigsRequestData(struct, version); - } - public Map configs() { return data.resources().stream().collect(Collectors.toMap( resource -> new ConfigResource( @@ -117,8 +112,8 @@ public boolean validateOnly() { } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected AlterConfigsRequestData data() { + return data; } @Override @@ -138,6 +133,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static AlterConfigsRequest parse(ByteBuffer buffer, short version) { - return new AlterConfigsRequest(ApiKeys.ALTER_CONFIGS.parseRequest(version, buffer), version); + return new AlterConfigsRequest(new AlterConfigsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java index 7122a132e5393..1115f06ee80a9 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.message.AlterConfigsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -32,17 +32,10 @@ public class AlterConfigsResponse extends AbstractResponse { private final AlterConfigsResponseData data; public AlterConfigsResponse(AlterConfigsResponseData data) { + super(ApiKeys.ALTER_CONFIGS); this.data = data; } - public AlterConfigsResponse(Struct struct, short version) { - this.data = new AlterConfigsResponseData(struct, version); - } - - public AlterConfigsResponseData data() { - return data; - } - public Map errors() { return data.responses().stream().collect(Collectors.toMap( response -> new ConfigResource( @@ -63,12 +56,12 @@ public int throttleTimeMs() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public AlterConfigsResponseData data() { + return data; } public static AlterConfigsResponse parse(ByteBuffer buffer, short version) { - return new AlterConfigsResponse(ApiKeys.ALTER_CONFIGS.parseResponse(version, buffer), version); + return new AlterConfigsResponse(new AlterConfigsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrRequest.java index 61662721f19b6..7ce86aedf432e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrRequest.java @@ -20,8 +20,10 @@ import org.apache.kafka.common.message.AlterIsrRequestData; import org.apache.kafka.common.message.AlterIsrResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; + +import java.nio.ByteBuffer; public class AlterIsrRequest extends AbstractRequest { @@ -36,11 +38,6 @@ public AlterIsrRequestData data() { return data; } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - /** * Get an error response for a request with specified throttle time in the response if applicable */ @@ -51,6 +48,10 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { .setErrorCode(Errors.forException(e).code())); } + public static AlterIsrRequest parse(ByteBuffer buffer, short version) { + return new AlterIsrRequest(new AlterIsrRequestData(new ByteBufferAccessor(buffer), version), version); + } + public static class Builder extends AbstractRequest.Builder { private final AlterIsrRequestData data; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrResponse.java index b475bfaecf705..433ba66bdadd3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrResponse.java @@ -18,9 +18,11 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.AlterIsrResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; @@ -29,6 +31,7 @@ public class AlterIsrResponse extends AbstractResponse { private final AlterIsrResponseData data; public AlterIsrResponse(AlterIsrResponseData data) { + super(ApiKeys.ALTER_ISR); this.data = data; } @@ -46,13 +49,12 @@ public Map errorCounts() { return counts; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); } + + public static AlterIsrResponse parse(ByteBuffer buffer, short version) { + return new AlterIsrResponse(new AlterIsrResponseData(new ByteBufferAccessor(buffer), version)); + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java index 7b2f848f614f9..2d289cc1497e1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java @@ -23,7 +23,7 @@ import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse; import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -52,39 +52,21 @@ public String toString() { } private final AlterPartitionReassignmentsRequestData data; - private final short version; private AlterPartitionReassignmentsRequest(AlterPartitionReassignmentsRequestData data, short version) { super(ApiKeys.ALTER_PARTITION_REASSIGNMENTS, version); this.data = data; - this.version = version; - } - - AlterPartitionReassignmentsRequest(Struct struct, short version) { - super(ApiKeys.ALTER_PARTITION_REASSIGNMENTS, version); - this.data = new AlterPartitionReassignmentsRequestData(struct, version); - this.version = version; } public static AlterPartitionReassignmentsRequest parse(ByteBuffer buffer, short version) { - return new AlterPartitionReassignmentsRequest( - ApiKeys.ALTER_PARTITION_REASSIGNMENTS.parseRequest(version, buffer), - version - ); + return new AlterPartitionReassignmentsRequest(new AlterPartitionReassignmentsRequestData( + new ByteBufferAccessor(buffer), version), version); } public AlterPartitionReassignmentsRequestData data() { return data; } - /** - * Visible for testing. - */ - @Override - public Struct toStruct() { - return data.toStruct(version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { ApiError apiError = ApiError.fromThrowable(e); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java index 495258655e9bb..6aea8b1edcc87 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -30,20 +30,14 @@ public class AlterPartitionReassignmentsResponse extends AbstractResponse { private final AlterPartitionReassignmentsResponseData data; - public AlterPartitionReassignmentsResponse(Struct struct) { - this(struct, ApiKeys.ALTER_PARTITION_REASSIGNMENTS.latestVersion()); - } - public AlterPartitionReassignmentsResponse(AlterPartitionReassignmentsResponseData data) { + super(ApiKeys.ALTER_PARTITION_REASSIGNMENTS); this.data = data; } - AlterPartitionReassignmentsResponse(Struct struct, short version) { - this.data = new AlterPartitionReassignmentsResponseData(struct, version); - } - public static AlterPartitionReassignmentsResponse parse(ByteBuffer buffer, short version) { - return new AlterPartitionReassignmentsResponse(ApiKeys.ALTER_PARTITION_REASSIGNMENTS.responseSchema(version).read(buffer), version); + return new AlterPartitionReassignmentsResponse( + new AlterPartitionReassignmentsResponseData(new ByteBufferAccessor(buffer), version)); } public AlterPartitionReassignmentsResponseData data() { @@ -71,9 +65,4 @@ public Map errorCounts() { )); return counts; } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java index 8cde2c0a1bde2..5eba039fa556b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java @@ -17,18 +17,20 @@ package org.apache.kafka.common.requests; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Errors; + import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.AlterReplicaLogDirsRequestData; import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData; import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData.AlterReplicaLogDirTopicResult; -import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class AlterReplicaLogDirsRequest extends AbstractRequest { @@ -53,22 +55,16 @@ public String toString() { } } - public AlterReplicaLogDirsRequest(Struct struct, short version) { - super(ApiKeys.ALTER_REPLICA_LOG_DIRS, version); - this.data = new AlterReplicaLogDirsRequestData(struct, version); - } - public AlterReplicaLogDirsRequest(AlterReplicaLogDirsRequestData data, short version) { super(ApiKeys.ALTER_REPLICA_LOG_DIRS, version); this.data = data; } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected AlterReplicaLogDirsRequestData data() { + return data; } - @Override public AlterReplicaLogDirsResponse getErrorResponse(int throttleTimeMs, Throwable e) { AlterReplicaLogDirsResponseData data = new AlterReplicaLogDirsResponseData(); data.setResults(this.data.dirs().stream().flatMap(alterDir -> @@ -93,6 +89,6 @@ public Map partitionDirs() { } public static AlterReplicaLogDirsRequest parse(ByteBuffer buffer, short version) { - return new AlterReplicaLogDirsRequest(ApiKeys.ALTER_REPLICA_LOG_DIRS.parseRequest(version, buffer), version); + return new AlterReplicaLogDirsRequest(new AlterReplicaLogDirsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java index bdc1ed0003b9e..afa658d1e150f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java @@ -17,14 +17,14 @@ package org.apache.kafka.common.requests; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; - import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; /** * Possible error codes: @@ -38,23 +38,16 @@ public class AlterReplicaLogDirsResponse extends AbstractResponse { private final AlterReplicaLogDirsResponseData data; - public AlterReplicaLogDirsResponse(Struct struct, short version) { - this.data = new AlterReplicaLogDirsResponseData(struct, version); - } - public AlterReplicaLogDirsResponse(AlterReplicaLogDirsResponseData data) { + super(ApiKeys.ALTER_REPLICA_LOG_DIRS); this.data = data; } + @Override public AlterReplicaLogDirsResponseData data() { return data; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -70,7 +63,7 @@ public Map errorCounts() { } public static AlterReplicaLogDirsResponse parse(ByteBuffer buffer, short version) { - return new AlterReplicaLogDirsResponse(ApiKeys.ALTER_REPLICA_LOG_DIRS.responseSchema(version).read(buffer), version); + return new AlterReplicaLogDirsResponse(new AlterReplicaLogDirsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsRequest.java index 8d3a18846021c..c319ec344525a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsRequest.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.message.AlterUserScramCredentialsRequestData; import org.apache.kafka.common.message.AlterUserScramCredentialsResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; import java.util.List; @@ -48,39 +48,21 @@ public String toString() { } } - private AlterUserScramCredentialsRequestData data; - private final short version; + private final AlterUserScramCredentialsRequestData data; private AlterUserScramCredentialsRequest(AlterUserScramCredentialsRequestData data, short version) { super(ApiKeys.ALTER_USER_SCRAM_CREDENTIALS, version); this.data = data; - this.version = version; - } - - AlterUserScramCredentialsRequest(Struct struct, short version) { - super(ApiKeys.ALTER_USER_SCRAM_CREDENTIALS, version); - this.data = new AlterUserScramCredentialsRequestData(struct, version); - this.version = version; } public static AlterUserScramCredentialsRequest parse(ByteBuffer buffer, short version) { - return new AlterUserScramCredentialsRequest( - ApiKeys.ALTER_USER_SCRAM_CREDENTIALS.parseRequest(version, buffer), version - ); + return new AlterUserScramCredentialsRequest(new AlterUserScramCredentialsRequestData(new ByteBufferAccessor(buffer), version), version); } public AlterUserScramCredentialsRequestData data() { return data; } - /** - * Visible for testing. - */ - @Override - public Struct toStruct() { - return data.toStruct(version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { ApiError apiError = ApiError.fromThrowable(e); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsResponse.java index 88ff920d3f132..2fa4937242742 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.AlterUserScramCredentialsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -28,22 +28,11 @@ public class AlterUserScramCredentialsResponse extends AbstractResponse { private final AlterUserScramCredentialsResponseData data; - public AlterUserScramCredentialsResponse(Struct struct) { - this(struct, ApiKeys.ALTER_USER_SCRAM_CREDENTIALS.latestVersion()); - } - public AlterUserScramCredentialsResponse(AlterUserScramCredentialsResponseData responseData) { + super(ApiKeys.ALTER_USER_SCRAM_CREDENTIALS); this.data = responseData; } - AlterUserScramCredentialsResponse(Struct struct, short version) { - this.data = new AlterUserScramCredentialsResponseData(struct, version); - } - - public static AlterUserScramCredentialsResponse parse(ByteBuffer buffer, short version) { - return new AlterUserScramCredentialsResponse(ApiKeys.ALTER_USER_SCRAM_CREDENTIALS.responseSchema(version).read(buffer), version); - } - public AlterUserScramCredentialsResponseData data() { return data; } @@ -63,8 +52,7 @@ public Map errorCounts() { return errorCounts(data.results().stream().map(r -> Errors.forCode(r.errorCode()))); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public static AlterUserScramCredentialsResponse parse(ByteBuffer buffer, short version) { + return new AlterUserScramCredentialsResponse(new AlterUserScramCredentialsResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiError.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiError.java index a790bc8b161c6..38712adbd55f8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiError.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiError.java @@ -19,13 +19,9 @@ import org.apache.kafka.common.errors.ApiException; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.util.Objects; -import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE; -import static org.apache.kafka.common.protocol.CommonFields.ERROR_MESSAGE; - /** * Encapsulates an error code (via the Errors enum) and an optional message. Generally, the optional message is only * defined if it adds information over the default message associated with the error code. @@ -47,12 +43,6 @@ public static ApiError fromThrowable(Throwable t) { return new ApiError(error, message); } - public ApiError(Struct struct) { - error = Errors.forCode(struct.get(ERROR_CODE)); - // In some cases, the error message field was introduced in newer version - message = struct.getOrElse(ERROR_MESSAGE, null); - } - public ApiError(Errors error) { this(error, error.message()); } @@ -67,12 +57,6 @@ public ApiError(short code, String message) { this.message = message; } - public void write(Struct struct) { - struct.set(ERROR_CODE, error.code()); - if (error != Errors.NONE) - struct.setIfExists(ERROR_MESSAGE, message); - } - public boolean is(Errors error) { return this.error == error; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java index 87ffceaa26e1c..946a9d9da7bed 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java @@ -22,8 +22,8 @@ import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.AppInfoParser; import java.nio.ByteBuffer; @@ -60,7 +60,7 @@ public String toString() { private final Short unsupportedRequestVersion; - public final ApiVersionsRequestData data; + private final ApiVersionsRequestData data; public ApiVersionsRequest(ApiVersionsRequestData data, short version) { this(data, version, null); @@ -78,10 +78,6 @@ public ApiVersionsRequest(ApiVersionsRequestData data, short version, Short unsu this.unsupportedRequestVersion = unsupportedRequestVersion; } - public ApiVersionsRequest(Struct struct, short version) { - this(new ApiVersionsRequestData(struct, version), version); - } - public boolean hasUnsupportedRequestVersion() { return unsupportedRequestVersion != null; } @@ -96,8 +92,8 @@ public boolean isValid() { } @Override - protected Struct toStruct() { - return data.toStruct(version()); + public ApiVersionsRequestData data() { + return data; } @Override @@ -124,7 +120,7 @@ public ApiVersionsResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static ApiVersionsRequest parse(ByteBuffer buffer, short version) { - return new ApiVersionsRequest(ApiKeys.API_VERSIONS.parseRequest(version, buffer), version); + return new ApiVersionsRequest(new ApiVersionsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java index 1e4ad17b47dc0..eaf8113fcd0b7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java @@ -29,8 +29,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.SchemaException; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import java.nio.ByteBuffer; @@ -45,38 +43,21 @@ public class ApiVersionsResponse extends AbstractResponse { public static final long UNKNOWN_FINALIZED_FEATURES_EPOCH = -1L; - public static final ApiVersionsResponse DEFAULT_API_VERSIONS_RESPONSE = - createApiVersionsResponse( - DEFAULT_THROTTLE_TIME, - RecordBatch.CURRENT_MAGIC_VALUE, - Features.emptySupportedFeatures(), - Features.emptyFinalizedFeatures(), - UNKNOWN_FINALIZED_FEATURES_EPOCH - ); + public static final ApiVersionsResponse DEFAULT_API_VERSIONS_RESPONSE = createApiVersionsResponse( + DEFAULT_THROTTLE_TIME, RecordBatch.CURRENT_MAGIC_VALUE); public final ApiVersionsResponseData data; public ApiVersionsResponse(ApiVersionsResponseData data) { + super(ApiKeys.API_VERSIONS); this.data = data; } - public ApiVersionsResponse(Struct struct) { - this(new ApiVersionsResponseData(struct, (short) (ApiVersionsResponseData.SCHEMAS.length - 1))); - } - - public ApiVersionsResponse(Struct struct, short version) { - this(new ApiVersionsResponseData(struct, version)); - } - + @Override public ApiVersionsResponseData data() { return data; } - @Override - protected Struct toStruct(short version) { - return this.data.toStruct(version); - } - public ApiVersionsResponseKey apiVersion(short apiKey) { return data.apiKeys().find(apiKey); } @@ -100,48 +81,23 @@ public static ApiVersionsResponse parse(ByteBuffer buffer, short version) { // Fallback to version 0 for ApiVersions response. If a client sends an ApiVersionsRequest // using a version higher than that supported by the broker, a version 0 response is sent // to the client indicating UNSUPPORTED_VERSION. When the client receives the response, it - // falls back while parsing it into a Struct which means that the version received by this - // method is not necessary the real one. It may be version 0 as well. + // falls back while parsing it which means that the version received by this + // method is not necessarily the real one. It may be version 0 as well. int prev = buffer.position(); try { - return new ApiVersionsResponse( - new ApiVersionsResponseData(new ByteBufferAccessor(buffer), version)); + return new ApiVersionsResponse(new ApiVersionsResponseData(new ByteBufferAccessor(buffer), version)); } catch (RuntimeException e) { buffer.position(prev); if (version != 0) - return new ApiVersionsResponse( - new ApiVersionsResponseData(new ByteBufferAccessor(buffer), (short) 0)); + return new ApiVersionsResponse(new ApiVersionsResponseData(new ByteBufferAccessor(buffer), (short) 0)); else throw e; } } - public static ApiVersionsResponse fromStruct(Struct struct, short version) { - // Fallback to version 0 for ApiVersions response. If a client sends an ApiVersionsRequest - // using a version higher than that supported by the broker, a version 0 response is sent - // to the client indicating UNSUPPORTED_VERSION. When the client receives the response, it - // falls back while parsing it into a Struct which means that the version received by this - // method is not necessary the real one. It may be version 0 as well. - try { - return new ApiVersionsResponse(struct, version); - } catch (SchemaException e) { - if (version != 0) - return new ApiVersionsResponse(struct, (short) 0); - else - throw e; - } - } - - public static ApiVersionsResponse createApiVersionsResponse( - final int throttleTimeMs, - final byte minMagic) { - return createApiVersionsResponse( - throttleTimeMs, - minMagic, - Features.emptySupportedFeatures(), - Features.emptyFinalizedFeatures(), - UNKNOWN_FINALIZED_FEATURES_EPOCH - ); + public static ApiVersionsResponse createApiVersionsResponse(final int throttleTimeMs, final byte minMagic) { + return createApiVersionsResponse(throttleTimeMs, minMagic, Features.emptySupportedFeatures(), + Features.emptyFinalizedFeatures(), UNKNOWN_FINALIZED_FEATURES_EPOCH); } private static ApiVersionsResponse createApiVersionsResponse( diff --git a/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochRequest.java index a3e492b72e233..c83e29dd3d52a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochRequest.java @@ -20,9 +20,10 @@ import org.apache.kafka.common.message.BeginQuorumEpochRequestData; import org.apache.kafka.common.message.BeginQuorumEpochResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import java.nio.ByteBuffer; import java.util.Collections; public class BeginQuorumEpochRequest extends AbstractRequest { @@ -52,14 +53,9 @@ private BeginQuorumEpochRequest(BeginQuorumEpochRequestData data, short version) this.data = data; } - public BeginQuorumEpochRequest(Struct struct, short version) { - super(ApiKeys.BEGIN_QUORUM_EPOCH, version); - this.data = new BeginQuorumEpochRequestData(struct, version); - } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected BeginQuorumEpochRequestData data() { + return data; } @Override @@ -68,6 +64,10 @@ public BeginQuorumEpochResponse getErrorResponse(int throttleTimeMs, Throwable e .setErrorCode(Errors.forException(e).code())); } + public static BeginQuorumEpochRequest parse(ByteBuffer buffer, short version) { + return new BeginQuorumEpochRequest(new BeginQuorumEpochRequestData(new ByteBufferAccessor(buffer), version), version); + } + public static BeginQuorumEpochRequestData singletonRequest(TopicPartition topicPartition, int leaderEpoch, int leaderId) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochResponse.java index 24eedb2bd82c6..c3e80eccaa82c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.BeginQuorumEpochResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -45,18 +45,10 @@ public class BeginQuorumEpochResponse extends AbstractResponse { public final BeginQuorumEpochResponseData data; public BeginQuorumEpochResponse(BeginQuorumEpochResponseData data) { + super(ApiKeys.BEGIN_QUORUM_EPOCH); this.data = data; } - public BeginQuorumEpochResponse(Struct struct, short version) { - this.data = new BeginQuorumEpochResponseData(struct, version); - } - - public BeginQuorumEpochResponse(Struct struct) { - short latestVersion = (short) (BeginQuorumEpochResponseData.SCHEMAS.length - 1); - this.data = new BeginQuorumEpochResponseData(struct, latestVersion); - } - public static BeginQuorumEpochResponseData singletonResponse( Errors topLevelError, TopicPartition topicPartition, @@ -78,11 +70,6 @@ public static BeginQuorumEpochResponseData singletonResponse( ); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public Map errorCounts() { Map errors = new HashMap<>(); @@ -98,8 +85,18 @@ public Map errorCounts() { return errors; } + @Override + protected BeginQuorumEpochResponseData data() { + return data; + } + + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static BeginQuorumEpochResponse parse(ByteBuffer buffer, short version) { - return new BeginQuorumEpochResponse(ApiKeys.BEGIN_QUORUM_EPOCH.responseSchema(version).read(buffer), version); + return new BeginQuorumEpochResponse(new BeginQuorumEpochResponseData(new ByteBufferAccessor(buffer), version)); } -} \ No newline at end of file +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java index 71238452cf782..f3c063a7a060d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.ControlledShutdownRequestData; import org.apache.kafka.common.message.ControlledShutdownResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -47,18 +47,10 @@ public String toString() { } private final ControlledShutdownRequestData data; - private final short version; private ControlledShutdownRequest(ControlledShutdownRequestData data, short version) { super(ApiKeys.CONTROLLED_SHUTDOWN, version); this.data = data; - this.version = version; - } - - public ControlledShutdownRequest(Struct struct, short version) { - super(ApiKeys.CONTROLLED_SHUTDOWN, version); - this.data = new ControlledShutdownRequestData(struct, version); - this.version = version; } @Override @@ -69,13 +61,8 @@ public ControlledShutdownResponse getErrorResponse(int throttleTimeMs, Throwable } public static ControlledShutdownRequest parse(ByteBuffer buffer, short version) { - return new ControlledShutdownRequest( - ApiKeys.CONTROLLED_SHUTDOWN.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version); + return new ControlledShutdownRequest(new ControlledShutdownRequestData(new ByteBufferAccessor(buffer), version), + version); } public ControlledShutdownRequestData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java index 53742c21a49a6..1add8000364e0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.message.ControlledShutdownResponseData; import org.apache.kafka.common.message.ControlledShutdownResponseData.RemainingPartition; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -40,13 +40,10 @@ public class ControlledShutdownResponse extends AbstractResponse { private final ControlledShutdownResponseData data; public ControlledShutdownResponse(ControlledShutdownResponseData data) { + super(ApiKeys.CONTROLLED_SHUTDOWN); this.data = data; } - public ControlledShutdownResponse(Struct struct, short version) { - this.data = new ControlledShutdownResponseData(struct, version); - } - public Errors error() { return Errors.forCode(data.errorCode()); } @@ -56,13 +53,13 @@ public Map errorCounts() { return errorCounts(error()); } - public static ControlledShutdownResponse parse(ByteBuffer buffer, short version) { - return new ControlledShutdownResponse(ApiKeys.CONTROLLED_SHUTDOWN.parseResponse(version, buffer), version); + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public static ControlledShutdownResponse parse(ByteBuffer buffer, short version) { + return new ControlledShutdownResponse(new ControlledShutdownResponseData(new ByteBufferAccessor(buffer), version)); } public ControlledShutdownResponseData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java index 3eb88a9b9144b..2ce651565b745 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java @@ -21,15 +21,15 @@ import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; +import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.CreateAclsRequestData; import org.apache.kafka.common.message.CreateAclsRequestData.AclCreation; import org.apache.kafka.common.message.CreateAclsResponseData; import org.apache.kafka.common.message.CreateAclsResponseData.AclCreationResult; -import org.apache.kafka.common.resource.ResourcePattern; -import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; import java.nio.ByteBuffer; @@ -48,7 +48,7 @@ public Builder(CreateAclsRequestData data) { @Override public CreateAclsRequest build(short version) { - return new CreateAclsRequest(version, data); + return new CreateAclsRequest(data, version); } @Override @@ -59,23 +59,19 @@ public String toString() { private final CreateAclsRequestData data; - CreateAclsRequest(short version, CreateAclsRequestData data) { + CreateAclsRequest(CreateAclsRequestData data, short version) { super(ApiKeys.CREATE_ACLS, version); validate(data); this.data = data; } - public CreateAclsRequest(Struct struct, short version) { - this(version, new CreateAclsRequestData(struct, version)); - } - public List aclCreations() { return data.creations(); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected CreateAclsRequestData data() { + return data; } @Override @@ -88,7 +84,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable throwable } public static CreateAclsRequest parse(ByteBuffer buffer, short version) { - return new CreateAclsRequest(ApiKeys.CREATE_ACLS.parseRequest(version, buffer), version); + return new CreateAclsRequest(new CreateAclsRequestData(new ByteBufferAccessor(buffer), version), version); } private void validate(CreateAclsRequestData data) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java index 5afce9eba217e..d0149b9a2a0d7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.CreateAclsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.List; @@ -29,16 +29,13 @@ public class CreateAclsResponse extends AbstractResponse { private final CreateAclsResponseData data; public CreateAclsResponse(CreateAclsResponseData data) { + super(ApiKeys.CREATE_ACLS); this.data = data; } - public CreateAclsResponse(Struct struct, short version) { - this.data = new CreateAclsResponseData(struct, version); - } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected CreateAclsResponseData data() { + return data; } @Override @@ -56,7 +53,7 @@ public Map errorCounts() { } public static CreateAclsResponse parse(ByteBuffer buffer, short version) { - return new CreateAclsResponse(ApiKeys.CREATE_ACLS.responseSchema(version).read(buffer), version); + return new CreateAclsResponse(new CreateAclsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java index 61b404a7e0623..e1d4cfaab80bd 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.CreateDelegationTokenRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import java.nio.ByteBuffer; @@ -33,18 +33,9 @@ private CreateDelegationTokenRequest(CreateDelegationTokenRequestData data, shor this.data = data; } - public CreateDelegationTokenRequest(Struct struct, short version) { - super(ApiKeys.CREATE_DELEGATION_TOKEN, version); - this.data = new CreateDelegationTokenRequestData(struct, version); - } - public static CreateDelegationTokenRequest parse(ByteBuffer buffer, short version) { - return new CreateDelegationTokenRequest(ApiKeys.CREATE_DELEGATION_TOKEN.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new CreateDelegationTokenRequest(new CreateDelegationTokenRequestData(new ByteBufferAccessor(buffer), version), + version); } public CreateDelegationTokenRequestData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java index 9796d68a82b99..9d39b6b641345 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.CreateDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import java.nio.ByteBuffer; @@ -30,15 +30,13 @@ public class CreateDelegationTokenResponse extends AbstractResponse { private final CreateDelegationTokenResponseData data; public CreateDelegationTokenResponse(CreateDelegationTokenResponseData data) { + super(ApiKeys.CREATE_DELEGATION_TOKEN); this.data = data; } - public CreateDelegationTokenResponse(Struct struct, short version) { - this.data = new CreateDelegationTokenResponseData(struct, version); - } - public static CreateDelegationTokenResponse parse(ByteBuffer buffer, short version) { - return new CreateDelegationTokenResponse(ApiKeys.CREATE_DELEGATION_TOKEN.responseSchema(version).read(buffer), version); + return new CreateDelegationTokenResponse( + new CreateDelegationTokenResponseData(new ByteBufferAccessor(buffer), version)); } public static CreateDelegationTokenResponse prepareResponse(int throttleTimeMs, @@ -75,11 +73,6 @@ public Map errorCounts() { return errorCounts(error()); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java index 77d4175c339de..8b0b9f81cc92c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java @@ -22,7 +22,7 @@ import org.apache.kafka.common.message.CreatePartitionsResponseData; import org.apache.kafka.common.message.CreatePartitionsResponseData.CreatePartitionsTopicResult; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; @@ -55,15 +55,6 @@ public String toString() { this.data = data; } - public CreatePartitionsRequest(Struct struct, short apiVersion) { - this(new CreatePartitionsRequestData(struct, apiVersion), apiVersion); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - public CreatePartitionsRequestData data() { return data; } @@ -85,6 +76,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static CreatePartitionsRequest parse(ByteBuffer buffer, short version) { - return new CreatePartitionsRequest(ApiKeys.CREATE_PARTITIONS.parseRequest(version, buffer), version); + return new CreatePartitionsRequest(new CreatePartitionsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsResponse.java index d658b0e4c230d..e0af04b07ae6a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsResponse.java @@ -19,35 +19,26 @@ import org.apache.kafka.common.message.CreatePartitionsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; - public class CreatePartitionsResponse extends AbstractResponse { private final CreatePartitionsResponseData data; public CreatePartitionsResponse(CreatePartitionsResponseData data) { + super(ApiKeys.CREATE_PARTITIONS); this.data = data; } - public CreatePartitionsResponse(Struct struct, short version) { - this.data = new CreatePartitionsResponseData(struct, version); - } - public CreatePartitionsResponseData data() { return data; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public Map errorCounts() { Map counts = new HashMap<>(); @@ -58,7 +49,7 @@ public Map errorCounts() { } public static CreatePartitionsResponse parse(ByteBuffer buffer, short version) { - return new CreatePartitionsResponse(ApiKeys.CREATE_PARTITIONS.parseResponse(version, buffer), version); + return new CreatePartitionsResponse(new CreatePartitionsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java index dd26e5642632e..7ec2c9adde3c4 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java @@ -25,7 +25,7 @@ import org.apache.kafka.common.message.CreateTopicsResponseData; import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; public class CreateTopicsRequest extends AbstractRequest { public static class Builder extends AbstractRequest.Builder { @@ -72,16 +72,11 @@ public String toString() { public static final int NO_NUM_PARTITIONS = -1; public static final short NO_REPLICATION_FACTOR = -1; - private CreateTopicsRequest(CreateTopicsRequestData data, short version) { + public CreateTopicsRequest(CreateTopicsRequestData data, short version) { super(ApiKeys.CREATE_TOPICS, version); this.data = data; } - public CreateTopicsRequest(Struct struct, short version) { - super(ApiKeys.CREATE_TOPICS, version); - this.data = new CreateTopicsRequestData(struct, version); - } - public CreateTopicsRequestData data() { return data; } @@ -103,14 +98,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static CreateTopicsRequest parse(ByteBuffer buffer, short version) { - return new CreateTopicsRequest(ApiKeys.CREATE_TOPICS.parseRequest(version, buffer), version); - } - - /** - * Visible for testing. - */ - @Override - public Struct toStruct() { - return data.toStruct(version()); + return new CreateTopicsRequest(new CreateTopicsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java index 7b64684d4dd4a..c15da20dfc79a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.CreateTopicsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -46,22 +46,14 @@ public class CreateTopicsResponse extends AbstractResponse { private final CreateTopicsResponseData data; public CreateTopicsResponse(CreateTopicsResponseData data) { + super(ApiKeys.CREATE_TOPICS); this.data = data; } - public CreateTopicsResponse(Struct struct, short version) { - this.data = new CreateTopicsResponseData(struct, version); - } - public CreateTopicsResponseData data() { return data; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -77,8 +69,7 @@ public Map errorCounts() { } public static CreateTopicsResponse parse(ByteBuffer buffer, short version) { - return new CreateTopicsResponse( - ApiKeys.CREATE_TOPICS.responseSchema(version).read(buffer), version); + return new CreateTopicsResponse(new CreateTopicsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java index b020b2b0ae3d3..6face08f5675d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java @@ -28,7 +28,7 @@ import org.apache.kafka.common.message.DeleteAclsResponseData; import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsFilterResult; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; @@ -49,7 +49,7 @@ public Builder(DeleteAclsRequestData data) { @Override public DeleteAclsRequest build(short version) { - return new DeleteAclsRequest(version, data); + return new DeleteAclsRequest(data, version); } @Override @@ -61,7 +61,7 @@ public String toString() { private final DeleteAclsRequestData data; - private DeleteAclsRequest(short version, DeleteAclsRequestData data) { + private DeleteAclsRequest(DeleteAclsRequestData data, short version) { super(ApiKeys.DELETE_ACLS, version); this.data = data; normalizeAndValidate(); @@ -95,18 +95,13 @@ else if (patternType != PatternType.LITERAL) } } - public DeleteAclsRequest(Struct struct, short version) { - super(ApiKeys.DELETE_ACLS, version); - this.data = new DeleteAclsRequestData(struct, version); - } - public List filters() { return data.filters().stream().map(DeleteAclsRequest::aclBindingFilter).collect(Collectors.toList()); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected DeleteAclsRequestData data() { + return data; } @Override @@ -118,11 +113,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable throwable .setErrorMessage(apiError.message())); return new DeleteAclsResponse(new DeleteAclsResponseData() .setThrottleTimeMs(throttleTimeMs) - .setFilterResults(filterResults)); + .setFilterResults(filterResults), version()); } public static DeleteAclsRequest parse(ByteBuffer buffer, short version) { - return new DeleteAclsRequest(DELETE_ACLS.parseRequest(version, buffer), version); + return new DeleteAclsRequest(new DeleteAclsRequestData(new ByteBufferAccessor(buffer), version), version); } public static DeleteAclsFilter deleteAclsFilter(AclBindingFilter filter) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java index 33d72a1df77fb..3ff8a9834f888 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java @@ -18,17 +18,17 @@ import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.DeleteAclsResponseData; import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsFilterResult; import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsMatchingAcl; -import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; -import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; import org.apache.kafka.server.authorizer.AclDeleteResult; import org.slf4j.Logger; @@ -44,18 +44,15 @@ public class DeleteAclsResponse extends AbstractResponse { private final DeleteAclsResponseData data; - public DeleteAclsResponse(DeleteAclsResponseData data) { + public DeleteAclsResponse(DeleteAclsResponseData data, short version) { + super(ApiKeys.DELETE_ACLS); this.data = data; - } - - public DeleteAclsResponse(Struct struct, short version) { - data = new DeleteAclsResponseData(struct, version); + validate(version); } @Override - protected Struct toStruct(short version) { - validate(version); - return data.toStruct(version); + protected DeleteAclsResponseData data() { + return data; } @Override @@ -73,7 +70,7 @@ public Map errorCounts() { } public static DeleteAclsResponse parse(ByteBuffer buffer, short version) { - return new DeleteAclsResponse(ApiKeys.DELETE_ACLS.parseResponse(version, buffer), version); + return new DeleteAclsResponse(new DeleteAclsResponseData(new ByteBufferAccessor(buffer), version), version); } public String toString() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java index 2ad947c5904be..d09a4d4c02d77 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResult; import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResultCollection; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -53,11 +53,6 @@ public DeleteGroupsRequest(DeleteGroupsRequestData data, short version) { this.data = data; } - public DeleteGroupsRequest(Struct struct, short version) { - super(ApiKeys.DELETE_GROUPS, version); - this.data = new DeleteGroupsRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); @@ -76,11 +71,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static DeleteGroupsRequest parse(ByteBuffer buffer, short version) { - return new DeleteGroupsRequest(ApiKeys.DELETE_GROUPS.parseRequest(version, buffer), version); + return new DeleteGroupsRequest(new DeleteGroupsRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected DeleteGroupsRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java index ad171c8f9d03b..a8e8d482de1d1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.DeleteGroupsResponseData; import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -42,21 +42,13 @@ public class DeleteGroupsResponse extends AbstractResponse { public final DeleteGroupsResponseData data; public DeleteGroupsResponse(DeleteGroupsResponseData data) { + super(ApiKeys.DELETE_GROUPS); this.data = data; } - public DeleteGroupsResponse(Struct struct) { - short latestVersion = (short) (DeleteGroupsResponseData.SCHEMAS.length - 1); - this.data = new DeleteGroupsResponseData(struct, latestVersion); - } - - public DeleteGroupsResponse(Struct struct, short version) { - this.data = new DeleteGroupsResponseData(struct, version); - } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected DeleteGroupsResponseData data() { + return data; } public Map errors() { @@ -85,7 +77,7 @@ public Map errorCounts() { } public static DeleteGroupsResponse parse(ByteBuffer buffer, short version) { - return new DeleteGroupsResponse(ApiKeys.DELETE_GROUPS.parseResponse(version, buffer), version); + return new DeleteGroupsResponse(new DeleteGroupsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java index 1939923c554e4..a4f62e18f675a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java @@ -22,8 +22,8 @@ import org.apache.kafka.common.message.DeleteRecordsResponseData; import org.apache.kafka.common.message.DeleteRecordsResponseData.DeleteRecordsTopicResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -57,16 +57,7 @@ private DeleteRecordsRequest(DeleteRecordsRequestData data, short version) { this.data = data; } - public DeleteRecordsRequest(Struct struct, short version) { - super(ApiKeys.DELETE_RECORDS, version); - this.data = new DeleteRecordsRequestData(struct, version); - } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - public DeleteRecordsRequestData data() { return data; } @@ -89,6 +80,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static DeleteRecordsRequest parse(ByteBuffer buffer, short version) { - return new DeleteRecordsRequest(ApiKeys.DELETE_RECORDS.parseRequest(version, buffer), version); + return new DeleteRecordsRequest(new DeleteRecordsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java index 968a35bd32c59..ef34102f8a8fd 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.DeleteRecordsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -42,18 +42,10 @@ public class DeleteRecordsResponse extends AbstractResponse { */ public DeleteRecordsResponse(DeleteRecordsResponseData data) { + super(ApiKeys.DELETE_RECORDS); this.data = data; } - public DeleteRecordsResponse(Struct struct, short version) { - this.data = new DeleteRecordsResponseData(struct, version); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - public DeleteRecordsResponseData data() { return data; } @@ -75,7 +67,7 @@ public Map errorCounts() { } public static DeleteRecordsResponse parse(ByteBuffer buffer, short version) { - return new DeleteRecordsResponse(ApiKeys.DELETE_RECORDS.parseResponse(version, buffer), version); + return new DeleteRecordsResponse(new DeleteRecordsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java index 440acfd2b838a..5ab64184aeab0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java @@ -20,15 +20,12 @@ import org.apache.kafka.common.message.DeleteTopicsResponseData; import org.apache.kafka.common.message.DeleteTopicsResponseData.DeletableTopicResult; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; public class DeleteTopicsRequest extends AbstractRequest { - private DeleteTopicsRequestData data; - private final short version; - public static class Builder extends AbstractRequest.Builder { private DeleteTopicsRequestData data; @@ -48,21 +45,11 @@ public String toString() { } } + private DeleteTopicsRequestData data; + private DeleteTopicsRequest(DeleteTopicsRequestData data, short version) { super(ApiKeys.DELETE_TOPICS, version); this.data = data; - this.version = version; - } - - public DeleteTopicsRequest(Struct struct, short version) { - super(ApiKeys.DELETE_TOPICS, version); - this.data = new DeleteTopicsRequestData(struct, version); - this.version = version; - } - - @Override - protected Struct toStruct() { - return data.toStruct(version); } public DeleteTopicsRequestData data() { @@ -72,7 +59,7 @@ public DeleteTopicsRequestData data() { @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { DeleteTopicsResponseData response = new DeleteTopicsResponseData(); - if (version >= 1) { + if (version() >= 1) { response.setThrottleTimeMs(throttleTimeMs); } ApiError apiError = ApiError.fromThrowable(e); @@ -85,7 +72,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static DeleteTopicsRequest parse(ByteBuffer buffer, short version) { - return new DeleteTopicsRequest(ApiKeys.DELETE_TOPICS.parseRequest(version, buffer), version); + return new DeleteTopicsRequest(new DeleteTopicsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java index bad23f6a2bd0b..3584ddfbfc497 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.DeleteTopicsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -41,18 +41,10 @@ public class DeleteTopicsResponse extends AbstractResponse { private DeleteTopicsResponseData data; public DeleteTopicsResponse(DeleteTopicsResponseData data) { + super(ApiKeys.DELETE_TOPICS); this.data = data; } - public DeleteTopicsResponse(Struct struct, short version) { - this.data = new DeleteTopicsResponseData(struct, version); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -72,7 +64,7 @@ public Map errorCounts() { } public static DeleteTopicsResponse parse(ByteBuffer buffer, short version) { - return new DeleteTopicsResponse(ApiKeys.DELETE_TOPICS.parseResponse(version, buffer), version); + return new DeleteTopicsResponse(new DeleteTopicsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java index 98059cfddbb8d..e6f9c38fd598b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java @@ -24,7 +24,7 @@ import org.apache.kafka.common.message.DescribeAclsRequestData; import org.apache.kafka.common.message.DescribeAclsResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; @@ -89,20 +89,10 @@ else if (patternType != PatternType.LITERAL) } } - public DescribeAclsRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_ACLS, version); - this.data = new DescribeAclsRequestData(struct, version); - } - public DescribeAclsRequestData data() { return data; } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable throwable) { ApiError error = ApiError.fromThrowable(throwable); @@ -110,11 +100,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable throwable .setThrottleTimeMs(throttleTimeMs) .setErrorCode(error.error().code()) .setErrorMessage(error.message()); - return new DescribeAclsResponse(response); + return new DescribeAclsResponse(response, version()); } public static DescribeAclsRequest parse(ByteBuffer buffer, short version) { - return new DescribeAclsRequest(ApiKeys.DESCRIBE_ACLS.parseRequest(version, buffer), version); + return new DescribeAclsRequest(new DescribeAclsRequestData(new ByteBufferAccessor(buffer), version), version); } public AclBindingFilter filter() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java index 3a41cb3de85b1..4308c9ea45bf7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java @@ -17,6 +17,15 @@ package org.apache.kafka.common.requests; +import org.apache.kafka.common.acl.AccessControlEntry; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; +import org.apache.kafka.common.errors.UnsupportedVersionException; +import org.apache.kafka.common.protocol.Errors; + import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; @@ -24,40 +33,37 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.kafka.common.acl.AccessControlEntry; -import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; -import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.DescribeAclsResponseData; import org.apache.kafka.common.message.DescribeAclsResponseData.AclDescription; import org.apache.kafka.common.message.DescribeAclsResponseData.DescribeAclsResource; -import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; -import org.apache.kafka.common.resource.PatternType; -import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; public class DescribeAclsResponse extends AbstractResponse { private final DescribeAclsResponseData data; - public DescribeAclsResponse(DescribeAclsResponseData data) { + public DescribeAclsResponse(DescribeAclsResponseData data, short version) { + super(ApiKeys.DESCRIBE_ACLS); this.data = data; + validate(Optional.of(version)); } - public DescribeAclsResponse(Struct struct, short version) { - this.data = new DescribeAclsResponseData(struct, version); + // Skips version validation, visible for testing + DescribeAclsResponse(DescribeAclsResponseData data) { + super(ApiKeys.DESCRIBE_ACLS); + this.data = data; + validate(Optional.empty()); } @Override - protected Struct toStruct(short version) { - validate(version); - return data.toStruct(version); + protected DescribeAclsResponseData data() { + return data; } @Override @@ -79,7 +85,7 @@ public List acls() { } public static DescribeAclsResponse parse(ByteBuffer buffer, short version) { - return new DescribeAclsResponse(ApiKeys.DESCRIBE_ACLS.responseSchema(version).read(buffer), version); + return new DescribeAclsResponse(new DescribeAclsResponseData(new ByteBufferAccessor(buffer), version), version); } @Override @@ -87,8 +93,8 @@ public boolean shouldClientThrottle(short version) { return version >= 1; } - private void validate(short version) { - if (version == 0) { + private void validate(Optional version) { + if (version.isPresent() && version.get() == 0) { final boolean unsupported = acls().stream() .anyMatch(acl -> acl.patternType() != PatternType.LITERAL.code()); if (unsupported) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java index 593cc6a9aace0..1f167b0ee3c2b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java @@ -18,11 +18,13 @@ import org.apache.kafka.common.message.DescribeClientQuotasRequestData; import org.apache.kafka.common.message.DescribeClientQuotasRequestData.ComponentData; +import org.apache.kafka.common.message.DescribeClientQuotasResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.quota.ClientQuotaFilter; import org.apache.kafka.common.quota.ClientQuotaFilterComponent; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -77,11 +79,6 @@ public DescribeClientQuotasRequest(DescribeClientQuotasRequestData data, short v this.data = data; } - public DescribeClientQuotasRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_CLIENT_QUOTAS, version); - this.data = new DescribeClientQuotasRequestData(struct, version); - } - public ClientQuotaFilter filter() { List components = new ArrayList<>(data.components().size()); for (ComponentData componentData : data.components()) { @@ -109,12 +106,23 @@ public ClientQuotaFilter filter() { } @Override - public DescribeClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { - return new DescribeClientQuotasResponse(throttleTimeMs, e); + protected DescribeClientQuotasRequestData data() { + return data; } @Override - protected Struct toStruct() { - return data.toStruct(version()); + public DescribeClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { + ApiError error = ApiError.fromThrowable(e); + return new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setErrorCode(error.error().code()) + .setErrorMessage(error.message()) + .setEntries(null)); + } + + public static DescribeClientQuotasRequest parse(ByteBuffer buffer, short version) { + return new DescribeClientQuotasRequest(new DescribeClientQuotasRequestData(new ByteBufferAccessor(buffer), version), + version); } + } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java index bda3673c27f2c..94fca64221d0f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java @@ -22,8 +22,8 @@ import org.apache.kafka.common.message.DescribeClientQuotasResponseData.EntryData; import org.apache.kafka.common.message.DescribeClientQuotasResponseData.ValueData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.quota.ClientQuotaEntity; import java.nio.ByteBuffer; @@ -36,48 +36,9 @@ public class DescribeClientQuotasResponse extends AbstractResponse { private final DescribeClientQuotasResponseData data; - public DescribeClientQuotasResponse(Map> entities, int throttleTimeMs) { - List entries = new ArrayList<>(entities.size()); - for (Map.Entry> entry : entities.entrySet()) { - ClientQuotaEntity quotaEntity = entry.getKey(); - List entityData = new ArrayList<>(quotaEntity.entries().size()); - for (Map.Entry entityEntry : quotaEntity.entries().entrySet()) { - entityData.add(new EntityData() - .setEntityType(entityEntry.getKey()) - .setEntityName(entityEntry.getValue())); - } - - Map quotaValues = entry.getValue(); - List valueData = new ArrayList<>(quotaValues.size()); - for (Map.Entry valuesEntry : entry.getValue().entrySet()) { - valueData.add(new ValueData() - .setKey(valuesEntry.getKey()) - .setValue(valuesEntry.getValue())); - } - - entries.add(new EntryData() - .setEntity(entityData) - .setValues(valueData)); - } - - this.data = new DescribeClientQuotasResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setErrorCode((short) 0) - .setErrorMessage(null) - .setEntries(entries); - } - - public DescribeClientQuotasResponse(int throttleTimeMs, Throwable e) { - ApiError apiError = ApiError.fromThrowable(e); - this.data = new DescribeClientQuotasResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setErrorCode(apiError.error().code()) - .setErrorMessage(apiError.message()) - .setEntries(null); - } - - public DescribeClientQuotasResponse(Struct struct, short version) { - this.data = new DescribeClientQuotasResponseData(struct, version); + public DescribeClientQuotasResponse(DescribeClientQuotasResponseData data) { + super(ApiKeys.DESCRIBE_CLIENT_QUOTAS); + this.data = data; } public void complete(KafkaFutureImpl>> future) { @@ -110,16 +71,49 @@ public int throttleTimeMs() { } @Override - public Map errorCounts() { - return errorCounts(Errors.forCode(data.errorCode())); + protected DescribeClientQuotasResponseData data() { + return data; } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public Map errorCounts() { + return errorCounts(Errors.forCode(data.errorCode())); } public static DescribeClientQuotasResponse parse(ByteBuffer buffer, short version) { - return new DescribeClientQuotasResponse(ApiKeys.DESCRIBE_CLIENT_QUOTAS.parseResponse(version, buffer), version); + return new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData(new ByteBufferAccessor(buffer), version)); + } + + public static DescribeClientQuotasResponse fromQuotaEntities(Map> entities, + int throttleTimeMs) { + List entries = new ArrayList<>(entities.size()); + for (Map.Entry> entry : entities.entrySet()) { + ClientQuotaEntity quotaEntity = entry.getKey(); + List entityData = new ArrayList<>(quotaEntity.entries().size()); + for (Map.Entry entityEntry : quotaEntity.entries().entrySet()) { + entityData.add(new EntityData() + .setEntityType(entityEntry.getKey()) + .setEntityName(entityEntry.getValue())); + } + + Map quotaValues = entry.getValue(); + List valueData = new ArrayList<>(quotaValues.size()); + for (Map.Entry valuesEntry : entry.getValue().entrySet()) { + valueData.add(new ValueData() + .setKey(valuesEntry.getKey()) + .setValue(valuesEntry.getValue())); + } + + entries.add(new EntryData() + .setEntity(entityData) + .setValues(valueData)); + } + + return new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setErrorCode((short) 0) + .setErrorMessage(null) + .setEntries(entries)); } + } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java index f17896e66e0ca..d612ca80949a0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.DescribeConfigsRequestData; import org.apache.kafka.common.message.DescribeConfigsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.stream.Collectors; @@ -48,20 +48,11 @@ public DescribeConfigsRequest(DescribeConfigsRequestData data, short version) { this.data = data; } - public DescribeConfigsRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_CONFIGS, version); - this.data = new DescribeConfigsRequestData(struct, version); - } - + @Override public DescribeConfigsRequestData data() { return data; } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - @Override public DescribeConfigsResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); @@ -77,6 +68,6 @@ public DescribeConfigsResponse getErrorResponse(int throttleTimeMs, Throwable e) } public static DescribeConfigsRequest parse(ByteBuffer buffer, short version) { - return new DescribeConfigsRequest(ApiKeys.DESCRIBE_CONFIGS.parseRequest(version, buffer), version); + return new DescribeConfigsRequest(new DescribeConfigsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java index 3767226395e81..aa7a713e8ab4e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java @@ -19,10 +19,9 @@ import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.message.DescribeConfigsResponseData; -import org.apache.kafka.common.message.DescribeConfigsResponseData.DescribeConfigsResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collection; @@ -219,13 +218,16 @@ public Map re private final DescribeConfigsResponseData data; public DescribeConfigsResponse(DescribeConfigsResponseData data) { + super(ApiKeys.DESCRIBE_CONFIGS); this.data = data; } - public DescribeConfigsResponse(Struct struct, short version) { - this.data = new DescribeConfigsResponseData(struct, version); + // This constructor should only be used after deserialization, it has special handling for version 0 + private DescribeConfigsResponse(DescribeConfigsResponseData data, short version) { + super(ApiKeys.DESCRIBE_CONFIGS); + this.data = data; if (version == 0) { - for (DescribeConfigsResult result : data.results()) { + for (DescribeConfigsResponseData.DescribeConfigsResult result : data.results()) { for (DescribeConfigsResponseData.DescribeConfigsResourceResult config : result.configs()) { if (config.isDefault()) { config.setConfigSource(ConfigSource.DEFAULT_CONFIG.id); @@ -243,6 +245,7 @@ public DescribeConfigsResponse(Struct struct, short version) { } } + @Override public DescribeConfigsResponseData data() { return data; } @@ -261,13 +264,8 @@ public Map errorCounts() { return errorCounts; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - public static DescribeConfigsResponse parse(ByteBuffer buffer, short version) { - return new DescribeConfigsResponse(ApiKeys.DESCRIBE_CONFIGS.parseResponse(version, buffer), version); + return new DescribeConfigsResponse(new DescribeConfigsResponseData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java index 7db3209d621fb..3e67140d375d0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java @@ -18,17 +18,16 @@ import org.apache.kafka.common.message.DescribeDelegationTokenRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; +import java.nio.ByteBuffer; import java.util.List; import java.util.stream.Collectors; public class DescribeDelegationTokenRequest extends AbstractRequest { - private final DescribeDelegationTokenRequestData data; - public static class Builder extends AbstractRequest.Builder { private final DescribeDelegationTokenRequestData data; @@ -54,21 +53,13 @@ public String toString() { } } - public DescribeDelegationTokenRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_DELEGATION_TOKEN, version); - this.data = new DescribeDelegationTokenRequestData(struct, version); - } + private final DescribeDelegationTokenRequestData data; public DescribeDelegationTokenRequest(DescribeDelegationTokenRequestData data, short version) { super(ApiKeys.DESCRIBE_DELEGATION_TOKEN, version); this.data = data; } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - public DescribeDelegationTokenRequestData data() { return data; } @@ -81,4 +72,9 @@ public boolean ownersListEmpty() { public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new DescribeDelegationTokenResponse(throttleTimeMs, Errors.forException(e)); } + + public static DescribeDelegationTokenRequest parse(ByteBuffer buffer, short version) { + return new DescribeDelegationTokenRequest(new DescribeDelegationTokenRequestData( + new ByteBufferAccessor(buffer), version), version); + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java index 91e7db0fccb1e..f6579a06b72da 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.message.DescribeDelegationTokenResponseData.DescribedDelegationToken; import org.apache.kafka.common.message.DescribeDelegationTokenResponseData.DescribedDelegationTokenRenewer; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.token.delegation.DelegationToken; import org.apache.kafka.common.security.token.delegation.TokenInformation; @@ -37,6 +37,7 @@ public class DescribeDelegationTokenResponse extends AbstractResponse { private final DescribeDelegationTokenResponseData data; public DescribeDelegationTokenResponse(int throttleTimeMs, Errors error, List tokens) { + super(ApiKeys.DESCRIBE_DELEGATION_TOKEN); List describedDelegationTokenList = tokens .stream() .map(dt -> new DescribedDelegationToken() @@ -63,12 +64,14 @@ public DescribeDelegationTokenResponse(int throttleTimeMs, Errors error) { this(throttleTimeMs, error, new ArrayList<>()); } - public DescribeDelegationTokenResponse(Struct struct, short version) { - this.data = new DescribeDelegationTokenResponseData(struct, version); + public DescribeDelegationTokenResponse(DescribeDelegationTokenResponseData data) { + super(ApiKeys.DESCRIBE_DELEGATION_TOKEN); + this.data = data; } public static DescribeDelegationTokenResponse parse(ByteBuffer buffer, short version) { - return new DescribeDelegationTokenResponse(ApiKeys.DESCRIBE_DELEGATION_TOKEN.responseSchema(version).read(buffer), version); + return new DescribeDelegationTokenResponse(new DescribeDelegationTokenResponseData( + new ByteBufferAccessor(buffer), version)); } @Override @@ -77,8 +80,8 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected DescribeDelegationTokenResponseData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java index 8f003acad5b97..db0490dc1b498 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.DescribeGroupsRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -46,32 +46,19 @@ public String toString() { } private final DescribeGroupsRequestData data; - private final short version; private DescribeGroupsRequest(DescribeGroupsRequestData data, short version) { super(ApiKeys.DESCRIBE_GROUPS, version); this.data = data; - this.version = version; - } - - public DescribeGroupsRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_GROUPS, version); - this.data = new DescribeGroupsRequestData(struct, version); - this.version = version; } public DescribeGroupsRequestData data() { return data; } - @Override - protected Struct toStruct() { - return data.toStruct(version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { - if (version == 0) { + if (version() == 0) { return DescribeGroupsResponse.fromError(DEFAULT_THROTTLE_TIME, Errors.forException(e), data.groups()); } else { return DescribeGroupsResponse.fromError(throttleTimeMs, Errors.forException(e), data.groups()); @@ -79,6 +66,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static DescribeGroupsRequest parse(ByteBuffer buffer, short version) { - return new DescribeGroupsRequest(ApiKeys.DESCRIBE_GROUPS.parseRequest(version, buffer), version); + return new DescribeGroupsRequest(new DescribeGroupsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java index 5836c7f3cd0cc..67c43d87ea092 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroup; import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroupMember; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.Utils; import java.nio.ByteBuffer; @@ -47,13 +47,10 @@ public class DescribeGroupsResponse extends AbstractResponse { private DescribeGroupsResponseData data; public DescribeGroupsResponse(DescribeGroupsResponseData data) { + super(ApiKeys.DESCRIBE_GROUPS); this.data = data; } - public DescribeGroupsResponse(Struct struct, short version) { - this.data = new DescribeGroupsResponseData(struct, version); - } - public static DescribedGroupMember groupMember( final String memberId, final String groupInstanceId, @@ -112,11 +109,6 @@ public DescribeGroupsResponseData data() { return data; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -129,9 +121,8 @@ public int throttleTimeMs() { @Override public Map errorCounts() { Map errorCounts = new HashMap<>(); - data.groups().forEach(describedGroup -> { - updateErrorCounts(errorCounts, Errors.forCode(describedGroup.errorCode())); - }); + data.groups().forEach(describedGroup -> + updateErrorCounts(errorCounts, Errors.forCode(describedGroup.errorCode()))); return errorCounts; } @@ -149,8 +140,7 @@ public static DescribeGroupsResponse fromError(int throttleTimeMs, Errors error, } public static DescribeGroupsResponse parse(ByteBuffer buffer, short version) { - return new DescribeGroupsResponse( - ApiKeys.DESCRIBE_GROUPS.responseSchema(version).read(buffer), version); + return new DescribeGroupsResponse(new DescribeGroupsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java index a58ef1fedb532..05ca4f0d59391 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java @@ -20,7 +20,7 @@ import org.apache.kafka.common.message.DescribeLogDirsRequestData; import org.apache.kafka.common.message.DescribeLogDirsResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; @@ -47,25 +47,16 @@ public String toString() { } } - public DescribeLogDirsRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_LOG_DIRS, version); - this.data = new DescribeLogDirsRequestData(struct, version); - } - public DescribeLogDirsRequest(DescribeLogDirsRequestData data, short version) { super(ApiKeys.DESCRIBE_LOG_DIRS, version); this.data = data; } + @Override public DescribeLogDirsRequestData data() { return data; } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new DescribeLogDirsResponse(new DescribeLogDirsResponseData().setThrottleTimeMs(throttleTimeMs)); @@ -76,6 +67,6 @@ public boolean isAllTopicPartitions() { } public static DescribeLogDirsRequest parse(ByteBuffer buffer, short version) { - return new DescribeLogDirsRequest(ApiKeys.DESCRIBE_LOG_DIRS.parseRequest(version, buffer), version); + return new DescribeLogDirsRequest(new DescribeLogDirsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java index e26fc554dccc1..61149253d5982 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.DescribeLogDirsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -34,11 +34,8 @@ public class DescribeLogDirsResponse extends AbstractResponse { private final DescribeLogDirsResponseData data; - public DescribeLogDirsResponse(Struct struct, short version) { - this.data = new DescribeLogDirsResponseData(struct, version); - } - public DescribeLogDirsResponse(DescribeLogDirsResponseData data) { + super(ApiKeys.DESCRIBE_LOG_DIRS); this.data = data; } @@ -46,11 +43,6 @@ public DescribeLogDirsResponseData data() { return data; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -66,7 +58,7 @@ public Map errorCounts() { } public static DescribeLogDirsResponse parse(ByteBuffer buffer, short version) { - return new DescribeLogDirsResponse(ApiKeys.DESCRIBE_LOG_DIRS.responseSchema(version).read(buffer), version); + return new DescribeLogDirsResponse(new DescribeLogDirsResponseData(new ByteBufferAccessor(buffer), version)); } // Note this class is part of the public API, reachable from Admin.describeLogDirs() diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumRequest.java index b2f5fd1d4a8a7..c318c214b6adf 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumRequest.java @@ -20,9 +20,10 @@ import org.apache.kafka.common.message.DescribeQuorumRequestData; import org.apache.kafka.common.message.DescribeQuorumResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -55,13 +56,8 @@ private DescribeQuorumRequest(DescribeQuorumRequestData data, short version) { this.data = data; } - public DescribeQuorumRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_QUORUM, version); - this.data = new DescribeQuorumRequestData(struct, version); - } - - public DescribeQuorumRequest(DescribeQuorumRequestData data) { - this(data, (short) (DescribeQuorumRequestData.SCHEMAS.length - 1)); + public static DescribeQuorumRequest parse(ByteBuffer buffer, short version) { + return new DescribeQuorumRequest(new DescribeQuorumRequestData(new ByteBufferAccessor(buffer), version), version); } public static DescribeQuorumRequestData singletonRequest(TopicPartition topicPartition) { @@ -76,8 +72,8 @@ public static DescribeQuorumRequestData singletonRequest(TopicPartition topicPar } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected DescribeQuorumRequestData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumResponse.java index ad0ce8aee406c..cb1a5ec277c08 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.message.DescribeQuorumResponseData; import org.apache.kafka.common.message.DescribeQuorumResponseData.ReplicaState; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -44,18 +44,10 @@ public class DescribeQuorumResponse extends AbstractResponse { public final DescribeQuorumResponseData data; public DescribeQuorumResponse(DescribeQuorumResponseData data) { + super(ApiKeys.DESCRIBE_QUORUM); this.data = data; } - public DescribeQuorumResponse(Struct struct, short version) { - this.data = new DescribeQuorumResponseData(struct, version); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public Map errorCounts() { Map errors = new HashMap<>(); @@ -70,6 +62,16 @@ public Map errorCounts() { return errors; } + @Override + protected DescribeQuorumResponseData data() { + return data; + } + + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static DescribeQuorumResponseData singletonResponse(TopicPartition topicPartition, int leaderId, int leaderEpoch, @@ -89,6 +91,6 @@ public static DescribeQuorumResponseData singletonResponse(TopicPartition topicP } public static DescribeQuorumResponse parse(ByteBuffer buffer, short version) { - return new DescribeQuorumResponse(ApiKeys.DESCRIBE_QUORUM.responseSchema(version).read(buffer), version); + return new DescribeQuorumResponse(new DescribeQuorumResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsRequest.java index e0d6bbac3b074..8b28d6b18628b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsRequest.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.message.DescribeUserScramCredentialsRequestData; import org.apache.kafka.common.message.DescribeUserScramCredentialsResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; @@ -53,30 +53,15 @@ private DescribeUserScramCredentialsRequest(DescribeUserScramCredentialsRequestD this.version = version; } - DescribeUserScramCredentialsRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS, version); - this.data = new DescribeUserScramCredentialsRequestData(struct, version); - this.version = version; - } - public static DescribeUserScramCredentialsRequest parse(ByteBuffer buffer, short version) { - return new DescribeUserScramCredentialsRequest( - ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS.parseRequest(version, buffer), version - ); + return new DescribeUserScramCredentialsRequest(new DescribeUserScramCredentialsRequestData( + new ByteBufferAccessor(buffer), version), version); } public DescribeUserScramCredentialsRequestData data() { return data; } - /** - * Visible for testing. - */ - @Override - public Struct toStruct() { - return data.toStruct(version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { ApiError apiError = ApiError.fromThrowable(e); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsResponse.java index 3043e023af4e1..a736c2c6876e5 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.DescribeUserScramCredentialsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -28,22 +28,11 @@ public class DescribeUserScramCredentialsResponse extends AbstractResponse { private final DescribeUserScramCredentialsResponseData data; - public DescribeUserScramCredentialsResponse(Struct struct) { - this(struct, ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS.latestVersion()); - } - public DescribeUserScramCredentialsResponse(DescribeUserScramCredentialsResponseData responseData) { + super(ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS); this.data = responseData; } - DescribeUserScramCredentialsResponse(Struct struct, short version) { - this.data = new DescribeUserScramCredentialsResponseData(struct, version); - } - - public static DescribeUserScramCredentialsResponse parse(ByteBuffer buffer, short version) { - return new DescribeUserScramCredentialsResponse(ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS.responseSchema(version).read(buffer), version); - } - public DescribeUserScramCredentialsResponseData data() { return data; } @@ -63,8 +52,7 @@ public Map errorCounts() { return errorCounts(data.results().stream().map(r -> Errors.forCode(r.errorCode()))); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public static DescribeUserScramCredentialsResponse parse(ByteBuffer buffer, short version) { + return new DescribeUserScramCredentialsResponse(new DescribeUserScramCredentialsResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java index 025733e39c0bc..89600a9072465 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java @@ -30,8 +30,8 @@ import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult; import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.MessageUtil; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.CollectionUtils; public class ElectLeadersRequest extends AbstractRequest { @@ -90,11 +90,6 @@ private ElectLeadersRequest(ElectLeadersRequestData data, short version) { this.data = data; } - public ElectLeadersRequest(Struct struct, short version) { - super(ApiKeys.ELECT_LEADERS, version); - this.data = new ElectLeadersRequestData(struct, version); - } - public ElectLeadersRequestData data() { return data; } @@ -124,11 +119,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static ElectLeadersRequest parse(ByteBuffer buffer, short version) { - return new ElectLeadersRequest(ApiKeys.ELECT_LEADERS.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new ElectLeadersRequest(new ElectLeadersRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java index 787ed4e939149..65eb09751b12d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java @@ -25,30 +25,18 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.ElectLeadersResponseData; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class ElectLeadersResponse extends AbstractResponse { - private final short version; private final ElectLeadersResponseData data; - public ElectLeadersResponse(Struct struct) { - this(struct, ApiKeys.ELECT_LEADERS.latestVersion()); - } - - public ElectLeadersResponse(Struct struct, short version) { - this.version = version; - this.data = new ElectLeadersResponseData(struct, version); - } - - public ElectLeadersResponse( - int throttleTimeMs, - short errorCode, - List electionResults) { - this(throttleTimeMs, errorCode, electionResults, ApiKeys.ELECT_LEADERS.latestVersion()); + public ElectLeadersResponse(ElectLeadersResponseData data) { + super(ApiKeys.ELECT_LEADERS); + this.data = data; } public ElectLeadersResponse( @@ -56,16 +44,11 @@ public ElectLeadersResponse( short errorCode, List electionResults, short version) { - - this.version = version; + super(ApiKeys.ELECT_LEADERS); this.data = new ElectLeadersResponseData(); - data.setThrottleTimeMs(throttleTimeMs); - - if (version >= 1) { + if (version >= 1) data.setErrorCode(errorCode); - } - data.setReplicaElectionResults(electionResults); } @@ -73,15 +56,6 @@ public ElectLeadersResponseData data() { return data; } - public short version() { - return version; - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -100,7 +74,7 @@ public Map errorCounts() { } public static ElectLeadersResponse parse(ByteBuffer buffer, short version) { - return new ElectLeadersResponse(ApiKeys.ELECT_LEADERS.responseSchema(version).read(buffer), version); + return new ElectLeadersResponse(new ElectLeadersResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochRequest.java index 017897f5aa227..b9f87d73fbd24 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochRequest.java @@ -20,9 +20,10 @@ import org.apache.kafka.common.message.EndQuorumEpochRequestData; import org.apache.kafka.common.message.EndQuorumEpochResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -53,14 +54,9 @@ private EndQuorumEpochRequest(EndQuorumEpochRequestData data, short version) { this.data = data; } - public EndQuorumEpochRequest(Struct struct, short version) { - super(ApiKeys.END_QUORUM_EPOCH, version); - this.data = new EndQuorumEpochRequestData(struct, version); - } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected EndQuorumEpochRequestData data() { + return data; } @Override @@ -69,6 +65,10 @@ public EndQuorumEpochResponse getErrorResponse(int throttleTimeMs, Throwable e) .setErrorCode(Errors.forException(e).code())); } + public static EndQuorumEpochRequest parse(ByteBuffer buffer, short version) { + return new EndQuorumEpochRequest(new EndQuorumEpochRequestData(new ByteBufferAccessor(buffer), version), version); + } + public static EndQuorumEpochRequestData singletonRequest(TopicPartition topicPartition, int leaderEpoch, int leaderId, diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java index b6243365c54a6..9b446663ea540 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.EndQuorumEpochResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -45,23 +45,10 @@ public class EndQuorumEpochResponse extends AbstractResponse { public final EndQuorumEpochResponseData data; public EndQuorumEpochResponse(EndQuorumEpochResponseData data) { + super(ApiKeys.END_QUORUM_EPOCH); this.data = data; } - public EndQuorumEpochResponse(Struct struct, short version) { - this.data = new EndQuorumEpochResponseData(struct, version); - } - - public EndQuorumEpochResponse(Struct struct) { - short latestVersion = (short) (EndQuorumEpochResponseData.SCHEMAS.length - 1); - this.data = new EndQuorumEpochResponseData(struct, latestVersion); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public Map errorCounts() { Map errors = new HashMap<>(); @@ -76,6 +63,16 @@ public Map errorCounts() { return errors; } + @Override + protected EndQuorumEpochResponseData data() { + return data; + } + + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static EndQuorumEpochResponseData singletonResponse( Errors topLevelError, TopicPartition topicPartition, @@ -98,6 +95,6 @@ public static EndQuorumEpochResponseData singletonResponse( } public static EndQuorumEpochResponse parse(ByteBuffer buffer, short version) { - return new EndQuorumEpochResponse(ApiKeys.END_QUORUM_EPOCH.responseSchema(version).read(buffer), version); + return new EndQuorumEpochResponse(new EndQuorumEpochResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java index 95a08275f2797..fdfafd005ca66 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.EndTxnRequestData; import org.apache.kafka.common.message.EndTxnResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -52,11 +52,6 @@ private EndTxnRequest(EndTxnRequestData data, short version) { this.data = data; } - public EndTxnRequest(Struct struct, short version) { - super(ApiKeys.END_TXN, version); - this.data = new EndTxnRequestData(struct, version); - } - public TransactionResult result() { if (data.committed()) return TransactionResult.COMMIT; @@ -65,8 +60,8 @@ public TransactionResult result() { } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected EndTxnRequestData data() { + return data; } @Override @@ -78,6 +73,6 @@ public EndTxnResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static EndTxnRequest parse(ByteBuffer buffer, short version) { - return new EndTxnRequest(ApiKeys.END_TXN.parseRequest(version, buffer), version); + return new EndTxnRequest(new EndTxnRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java index 562b7a35a26b6..b782cefd50717 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.EndTxnResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -41,18 +41,10 @@ public class EndTxnResponse extends AbstractResponse { public final EndTxnResponseData data; public EndTxnResponse(EndTxnResponseData data) { + super(ApiKeys.END_TXN); this.data = data; } - public EndTxnResponse(Struct struct) { - this(struct, (short) (EndTxnResponseData.SCHEMAS.length - 1)); - } - - - public EndTxnResponse(Struct struct, short version) { - this.data = new EndTxnResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -69,12 +61,12 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected EndTxnResponseData data() { + return data; } public static EndTxnResponse parse(ByteBuffer buffer, short version) { - return new EndTxnResponse(ApiKeys.END_TXN.parseResponse(version, buffer), version); + return new EndTxnResponse(new EndTxnResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeRequest.java index 58f2d245c530d..4be5259458b6d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeRequest.java @@ -18,11 +18,9 @@ import org.apache.kafka.common.message.EnvelopeRequestData; import org.apache.kafka.common.message.EnvelopeResponseData; -import org.apache.kafka.common.network.Send; -import org.apache.kafka.common.protocol.SendBuilder; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -60,11 +58,6 @@ public EnvelopeRequest(EnvelopeRequestData data, short version) { this.data = data; } - public EnvelopeRequest(Struct struct, short version) { - super(ApiKeys.ENVELOPE, version); - this.data = new EnvelopeRequestData(struct, version); - } - public ByteBuffer requestData() { return data.requestData(); } @@ -77,11 +70,6 @@ public byte[] requestPrincipal() { return data.requestPrincipal(); } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new EnvelopeResponse(new EnvelopeResponseData() @@ -89,16 +77,10 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static EnvelopeRequest parse(ByteBuffer buffer, short version) { - return new EnvelopeRequest(ApiKeys.ENVELOPE.parseRequest(version, buffer), version); + return new EnvelopeRequest(new EnvelopeRequestData(new ByteBufferAccessor(buffer), version), version); } public EnvelopeRequestData data() { return data; } - - @Override - public Send toSend(String destination, RequestHeader header) { - return SendBuilder.buildRequestSend(destination, header, this.data); - } - } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java index d95325b2214c3..c7c8d24e4a338 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java @@ -17,10 +17,9 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.EnvelopeResponseData; -import org.apache.kafka.common.network.Send; -import org.apache.kafka.common.protocol.SendBuilder; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -29,8 +28,8 @@ public class EnvelopeResponse extends AbstractResponse { private final EnvelopeResponseData data; - public EnvelopeResponse(ByteBuffer responseData, - Errors error) { + public EnvelopeResponse(ByteBuffer responseData, Errors error) { + super(ApiKeys.ENVELOPE); this.data = new EnvelopeResponseData() .setResponseData(responseData) .setErrorCode(error.code()); @@ -41,6 +40,7 @@ public EnvelopeResponse(Errors error) { } public EnvelopeResponse(EnvelopeResponseData data) { + super(ApiKeys.ENVELOPE); this.data = data; } @@ -53,11 +53,6 @@ public Map errorCounts() { return errorCounts(error()); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - public Errors error() { return Errors.forCode(data.errorCode()); } @@ -67,8 +62,12 @@ public EnvelopeResponseData data() { } @Override - protected Send toSend(String destination, ResponseHeader header, short apiVersion) { - return SendBuilder.buildResponseSend(destination, header, this.data, apiVersion); + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + + public static EnvelopeResponse parse(ByteBuffer buffer, short version) { + return new EnvelopeResponse(new EnvelopeResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java index ca6d2d67d1f79..ac03944ab029e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.ExpireDelegationTokenRequestData; import org.apache.kafka.common.message.ExpireDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class ExpireDelegationTokenRequest extends AbstractRequest { @@ -33,18 +33,14 @@ private ExpireDelegationTokenRequest(ExpireDelegationTokenRequestData data, shor this.data = data; } - public ExpireDelegationTokenRequest(Struct struct, short version) { - super(ApiKeys.EXPIRE_DELEGATION_TOKEN, version); - this.data = new ExpireDelegationTokenRequestData(struct, version); - } - public static ExpireDelegationTokenRequest parse(ByteBuffer buffer, short version) { - return new ExpireDelegationTokenRequest(ApiKeys.EXPIRE_DELEGATION_TOKEN.parseRequest(version, buffer), version); + return new ExpireDelegationTokenRequest( + new ExpireDelegationTokenRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected ExpireDelegationTokenRequestData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java index 75b6c5328da5f..451c736ffb7e6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java @@ -21,23 +21,21 @@ import org.apache.kafka.common.message.ExpireDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class ExpireDelegationTokenResponse extends AbstractResponse { private final ExpireDelegationTokenResponseData data; public ExpireDelegationTokenResponse(ExpireDelegationTokenResponseData data) { + super(ApiKeys.EXPIRE_DELEGATION_TOKEN); this.data = data; } - public ExpireDelegationTokenResponse(Struct struct, short version) { - this.data = new ExpireDelegationTokenResponseData(struct, version); - } - public static ExpireDelegationTokenResponse parse(ByteBuffer buffer, short version) { - return new ExpireDelegationTokenResponse(ApiKeys.EXPIRE_DELEGATION_TOKEN.responseSchema(version).read(buffer), version); + return new ExpireDelegationTokenResponse(new ExpireDelegationTokenResponseData(new ByteBufferAccessor(buffer), + version)); } public Errors error() { @@ -54,8 +52,8 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected ExpireDelegationTokenResponseData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java index 6c88a37e30b9c..b62c844f63b5a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java @@ -19,12 +19,8 @@ import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.FetchRequestData; -import org.apache.kafka.common.message.RequestHeaderData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.ObjectSerializationCache; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.utils.Utils; @@ -55,10 +51,6 @@ public class FetchRequest extends AbstractRequest { private final List toForget; private final FetchMetadata metadata; - public FetchRequestData data() { - return data; - } - public static final class PartitionData { public final long fetchOffset; public final long logStartOffset; @@ -378,34 +370,12 @@ public String rackId() { return data.rackId(); } - @Override - public ByteBuffer serialize(RequestHeader header) { - // Unlike the custom FetchResponse#toSend, we don't include the buffer size here. This buffer is passed - // to a NetworkSend which adds the length value in the eventual serialization - - ObjectSerializationCache cache = new ObjectSerializationCache(); - RequestHeaderData requestHeaderData = header.data(); - - int headerSize = requestHeaderData.size(cache, header.headerVersion()); - int bodySize = data.size(cache, header.apiVersion()); - - ByteBuffer buffer = ByteBuffer.allocate(headerSize + bodySize); - ByteBufferAccessor writer = new ByteBufferAccessor(buffer); - - requestHeaderData.write(writer, cache, header.headerVersion()); - data.write(writer, cache, header.apiVersion()); - - buffer.rewind(); - return buffer; - } - - // For testing public static FetchRequest parse(ByteBuffer buffer, short version) { return new FetchRequest(new FetchRequestData(ApiKeys.FETCH.parseRequest(version, buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + public FetchRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java index db3b12a5258d8..d65a55e6c2b29 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java @@ -18,12 +18,10 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.FetchResponseData; -import org.apache.kafka.common.network.Send; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.ObjectSerializationCache; -import org.apache.kafka.common.protocol.SendBuilder; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.BaseRecords; import org.apache.kafka.common.record.MemoryRecords; @@ -273,25 +271,17 @@ public FetchResponse(Errors error, LinkedHashMap> responseData, int throttleTimeMs, int sessionId) { + super(ApiKeys.FETCH); this.data = toMessage(throttleTimeMs, error, responseData.entrySet().iterator(), sessionId); this.responseDataMap = responseData; } public FetchResponse(FetchResponseData fetchResponseData) { + super(ApiKeys.FETCH); this.data = fetchResponseData; this.responseDataMap = toResponseDataMap(fetchResponseData); } - @Override - public Struct toStruct(short version) { - return data.toStruct(version); - } - - @Override - public Send toSend(String dest, ResponseHeader responseHeader, short apiVersion) { - return SendBuilder.buildResponseSend(dest, responseHeader, this.data, apiVersion); - } - public Errors error() { return Errors.forCode(data.errorCode()); } @@ -320,10 +310,7 @@ public Map errorCounts() { } public static FetchResponse parse(ByteBuffer buffer, short version) { - FetchResponseData fetchResponseData = new FetchResponseData(); - ByteBufferAccessor reader = new ByteBufferAccessor(buffer); - fetchResponseData.read(reader, version); - return new FetchResponse<>(fetchResponseData); + return new FetchResponse<>(new FetchResponseData(new ByteBufferAccessor(buffer), version)); } @SuppressWarnings("unchecked") diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java index 0e728439169d4..b6fdbfc1e63a5 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java @@ -21,12 +21,11 @@ import org.apache.kafka.common.message.FindCoordinatorRequestData; import org.apache.kafka.common.message.FindCoordinatorResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; - public class FindCoordinatorRequest extends AbstractRequest { public static class Builder extends AbstractRequest.Builder { @@ -63,11 +62,6 @@ private FindCoordinatorRequest(FindCoordinatorRequestData data, short version) { this.data = data; } - public FindCoordinatorRequest(Struct struct, short version) { - super(ApiKeys.FIND_COORDINATOR, version); - this.data = new FindCoordinatorRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { FindCoordinatorResponseData response = new FindCoordinatorResponseData(); @@ -79,12 +73,8 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static FindCoordinatorRequest parse(ByteBuffer buffer, short version) { - return new FindCoordinatorRequest(ApiKeys.FIND_COORDINATOR.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new FindCoordinatorRequest(new FindCoordinatorRequestData(new ByteBufferAccessor(buffer), version), + version); } public FindCoordinatorRequestData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java index 2c6819e1f1421..917b087fbab35 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.message.FindCoordinatorResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -41,13 +41,10 @@ public class FindCoordinatorResponse extends AbstractResponse { private final FindCoordinatorResponseData data; public FindCoordinatorResponse(FindCoordinatorResponseData data) { + super(ApiKeys.FIND_COORDINATOR); this.data = data; } - public FindCoordinatorResponse(Struct struct, short version) { - this.data = new FindCoordinatorResponseData(struct, version); - } - public FindCoordinatorResponseData data() { return data; } @@ -74,13 +71,8 @@ public Map errorCounts() { return errorCounts(error()); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - public static FindCoordinatorResponse parse(ByteBuffer buffer, short version) { - return new FindCoordinatorResponse(ApiKeys.FIND_COORDINATOR.responseSchema(version).read(buffer), version); + return new FindCoordinatorResponse(new FindCoordinatorResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java index e5cc6c8860a58..754566db7a425 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java @@ -20,12 +20,11 @@ import org.apache.kafka.common.message.HeartbeatRequestData; import org.apache.kafka.common.message.HeartbeatResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; - public class HeartbeatRequest extends AbstractRequest { public static class Builder extends AbstractRequest.Builder { @@ -58,11 +57,6 @@ private HeartbeatRequest(HeartbeatRequestData data, short version) { this.data = data; } - public HeartbeatRequest(Struct struct, short version) { - super(ApiKeys.HEARTBEAT, version); - this.data = new HeartbeatRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { HeartbeatResponseData responseData = new HeartbeatResponseData(). @@ -74,11 +68,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static HeartbeatRequest parse(ByteBuffer buffer, short version) { - return new HeartbeatRequest(ApiKeys.HEARTBEAT.parseRequest(version, buffer), version); + return new HeartbeatRequest(new HeartbeatRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected HeartbeatRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java index e135f30089101..c8fdae2507c6b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java @@ -18,13 +18,12 @@ import org.apache.kafka.common.message.HeartbeatResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; - public class HeartbeatResponse extends AbstractResponse { /** @@ -40,13 +39,10 @@ public class HeartbeatResponse extends AbstractResponse { private final HeartbeatResponseData data; public HeartbeatResponse(HeartbeatResponseData data) { + super(ApiKeys.HEARTBEAT); this.data = data; } - public HeartbeatResponse(Struct struct, short version) { - this.data = new HeartbeatResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -62,12 +58,12 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected HeartbeatResponseData data() { + return data; } public static HeartbeatResponse parse(ByteBuffer buffer, short version) { - return new HeartbeatResponse(ApiKeys.HEARTBEAT.parseResponse(version, buffer), version); + return new HeartbeatResponse(new HeartbeatResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsRequest.java index a01351feeba57..210e18b37accb 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsRequest.java @@ -24,7 +24,7 @@ import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData; import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; import java.util.Collection; @@ -86,25 +86,15 @@ private IncrementalAlterConfigsRequest(IncrementalAlterConfigsRequestData data, this.version = version; } - IncrementalAlterConfigsRequest(final Struct struct, final short version) { - super(ApiKeys.INCREMENTAL_ALTER_CONFIGS, version); - this.data = new IncrementalAlterConfigsRequestData(struct, version); - this.version = version; - } - public static IncrementalAlterConfigsRequest parse(ByteBuffer buffer, short version) { - return new IncrementalAlterConfigsRequest(ApiKeys.INCREMENTAL_ALTER_CONFIGS.parseRequest(version, buffer), version); + return new IncrementalAlterConfigsRequest(new IncrementalAlterConfigsRequestData( + new ByteBufferAccessor(buffer), version), version); } public IncrementalAlterConfigsRequestData data() { return data; } - @Override - protected Struct toStruct() { - return data.toStruct(version); - } - @Override public AbstractResponse getErrorResponse(final int throttleTimeMs, final Throwable e) { IncrementalAlterConfigsResponseData response = new IncrementalAlterConfigsResponseData(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java index 32913ebe5df11..99427c7ad9f3e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData; import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -34,6 +34,7 @@ public class IncrementalAlterConfigsResponse extends AbstractResponse { public IncrementalAlterConfigsResponse(final int requestThrottleMs, final Map results) { + super(ApiKeys.INCREMENTAL_ALTER_CONFIGS); final List newResults = new ArrayList<>(results.size()); results.forEach( (resource, error) -> newResults.add( @@ -61,13 +62,10 @@ public static Map fromResponseData(final IncrementalAl private final IncrementalAlterConfigsResponseData data; public IncrementalAlterConfigsResponse(IncrementalAlterConfigsResponseData data) { + super(ApiKeys.INCREMENTAL_ALTER_CONFIGS); this.data = data; } - public IncrementalAlterConfigsResponse(final Struct struct, final short version) { - this.data = new IncrementalAlterConfigsResponseData(struct, version); - } - public IncrementalAlterConfigsResponseData data() { return data; } @@ -81,11 +79,6 @@ public Map errorCounts() { return counts; } - @Override - protected Struct toStruct(final short version) { - return data.toStruct(version); - } - @Override public boolean shouldClientThrottle(short version) { return version >= 0; @@ -97,7 +90,7 @@ public int throttleTimeMs() { } public static IncrementalAlterConfigsResponse parse(ByteBuffer buffer, short version) { - return new IncrementalAlterConfigsResponse( - ApiKeys.INCREMENTAL_ALTER_CONFIGS.responseSchema(version).read(buffer), version); + return new IncrementalAlterConfigsResponse(new IncrementalAlterConfigsResponseData( + new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java index 8351c2159836a..08e75cb536c63 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.InitProducerIdRequestData; import org.apache.kafka.common.message.InitProducerIdResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import java.nio.ByteBuffer; @@ -58,12 +58,6 @@ private InitProducerIdRequest(InitProducerIdRequestData data, short version) { this.data = data; } - public InitProducerIdRequest(Struct struct, short version) { - super(ApiKeys.INIT_PRODUCER_ID, version); - this.data = new InitProducerIdRequestData(struct, version); - } - - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { InitProducerIdResponseData response = new InitProducerIdResponseData() @@ -75,12 +69,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static InitProducerIdRequest parse(ByteBuffer buffer, short version) { - return new InitProducerIdRequest(ApiKeys.INIT_PRODUCER_ID.parseRequest(version, buffer), version); + return new InitProducerIdRequest(new InitProducerIdRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected InitProducerIdRequestData data() { + return data; } - } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java index 822a4b6ce7cd6..6066c13ef5ab6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.InitProducerIdResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -39,13 +39,10 @@ public class InitProducerIdResponse extends AbstractResponse { public final InitProducerIdResponseData data; public InitProducerIdResponse(InitProducerIdResponseData data) { + super(ApiKeys.INIT_PRODUCER_ID); this.data = data; } - public InitProducerIdResponse(Struct struct, short version) { - this.data = new InitProducerIdResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -57,12 +54,12 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected InitProducerIdResponseData data() { + return data; } public static InitProducerIdResponse parse(ByteBuffer buffer, short version) { - return new InitProducerIdResponse(ApiKeys.INIT_PRODUCER_ID.parseResponse(version, buffer), version); + return new InitProducerIdResponse(new InitProducerIdResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java index 61486abc1be74..7be16b6410c3b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.JoinGroupRequestData; import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -99,12 +99,6 @@ public JoinGroupRequest(JoinGroupRequestData data, short version) { maybeOverrideRebalanceTimeout(version); } - public JoinGroupRequest(Struct struct, short version) { - super(ApiKeys.JOIN_GROUP, version); - this.data = new JoinGroupRequestData(struct, version); - maybeOverrideRebalanceTimeout(version); - } - private void maybeOverrideRebalanceTimeout(short version) { if (version == 0) { // Version 0 has no rebalance timeout, so we use the session timeout @@ -137,11 +131,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static JoinGroupRequest parse(ByteBuffer buffer, short version) { - return new JoinGroupRequest(ApiKeys.JOIN_GROUP.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new JoinGroupRequest(new JoinGroupRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java index 8e585d660aa76..8472175edab4a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -29,18 +29,10 @@ public class JoinGroupResponse extends AbstractResponse { private final JoinGroupResponseData data; public JoinGroupResponse(JoinGroupResponseData data) { + super(ApiKeys.JOIN_GROUP); this.data = data; } - public JoinGroupResponse(Struct struct) { - short latestVersion = (short) (JoinGroupResponseData.SCHEMAS.length - 1); - this.data = new JoinGroupResponseData(struct, latestVersion); - } - - public JoinGroupResponse(Struct struct, short version) { - this.data = new JoinGroupResponseData(struct, version); - } - public JoinGroupResponseData data() { return data; } @@ -63,13 +55,8 @@ public Map errorCounts() { return errorCounts(Errors.forCode(data.errorCode())); } - public static JoinGroupResponse parse(ByteBuffer buffer, short versionId) { - return new JoinGroupResponse(ApiKeys.JOIN_GROUP.parseResponse(versionId, buffer), versionId); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public static JoinGroupResponse parse(ByteBuffer buffer, short version) { + return new JoinGroupResponse(new JoinGroupResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java index 2e6b9560baeea..9927dda1de6e6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java @@ -24,8 +24,8 @@ import org.apache.kafka.common.message.LeaderAndIsrResponseData; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.FlattenedIterator; import org.apache.kafka.common.utils.Utils; @@ -123,15 +123,6 @@ private void normalize() { } } - public LeaderAndIsrRequest(Struct struct, short version) { - this(new LeaderAndIsrRequestData(struct, version), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - @Override public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) { LeaderAndIsrResponseData responseData = new LeaderAndIsrResponseData(); @@ -175,11 +166,11 @@ public List liveLeaders() { return Collections.unmodifiableList(data.liveLeaders()); } - public LeaderAndIsrRequestData data() { + protected LeaderAndIsrRequestData data() { return data; } public static LeaderAndIsrRequest parse(ByteBuffer buffer, short version) { - return new LeaderAndIsrRequest(ApiKeys.LEADER_AND_ISR.parseRequest(version, buffer), version); + return new LeaderAndIsrRequest(new LeaderAndIsrRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java index d2ded841224bd..1e1069edea3a1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.LeaderAndIsrResponseData; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -38,13 +38,10 @@ public class LeaderAndIsrResponse extends AbstractResponse { private final LeaderAndIsrResponseData data; public LeaderAndIsrResponse(LeaderAndIsrResponseData data) { + super(ApiKeys.LEADER_AND_ISR); this.data = data; } - public LeaderAndIsrResponse(Struct struct, short version) { - this.data = new LeaderAndIsrResponseData(struct, version); - } - public List partitions() { return data.partitionErrors(); } @@ -65,13 +62,18 @@ public Map errorCounts() { return errors; } + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static LeaderAndIsrResponse parse(ByteBuffer buffer, short version) { - return new LeaderAndIsrResponse(ApiKeys.LEADER_AND_ISR.parseResponse(version, buffer), version); + return new LeaderAndIsrResponse(new LeaderAndIsrResponseData(new ByteBufferAccessor(buffer), version)); } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected LeaderAndIsrResponseData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java index 4cd72a6cf44ca..ad7547c648ee7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java @@ -21,9 +21,9 @@ import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.MessageUtil; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -81,18 +81,10 @@ public String toString() { } } private final LeaveGroupRequestData data; - private final short version; private LeaveGroupRequest(LeaveGroupRequestData data, short version) { super(ApiKeys.LEAVE_GROUP, version); this.data = data; - this.version = version; - } - - public LeaveGroupRequest(Struct struct, short version) { - super(ApiKeys.LEAVE_GROUP, version); - this.data = new LeaveGroupRequestData(struct, version); - this.version = version; } public LeaveGroupRequestData data() { @@ -101,7 +93,7 @@ public LeaveGroupRequestData data() { public List members() { // Before version 3, leave group request is still in single mode - return version <= 2 ? Collections.singletonList( + return version() <= 2 ? Collections.singletonList( new MemberIdentity() .setMemberId(data.memberId())) : data.members(); } @@ -118,11 +110,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static LeaveGroupRequest parse(ByteBuffer buffer, short version) { - return new LeaveGroupRequest(ApiKeys.LEAVE_GROUP.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version); + return new LeaveGroupRequest(new LeaveGroupRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java index 717730f25d04b..3bc70674f8a75 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -51,6 +51,7 @@ public class LeaveGroupResponse extends AbstractResponse { public final LeaveGroupResponseData data; public LeaveGroupResponse(LeaveGroupResponseData data) { + super(ApiKeys.LEAVE_GROUP); this.data = data; } @@ -58,6 +59,7 @@ public LeaveGroupResponse(List memberResponses, Errors topLevelError, final int throttleTimeMs, final short version) { + super(ApiKeys.LEAVE_GROUP); if (version <= 2) { // Populate member level error. final short errorCode = getError(topLevelError, memberResponses).code(); @@ -75,15 +77,6 @@ public LeaveGroupResponse(List memberResponses, } } - public LeaveGroupResponse(Struct struct) { - short latestVersion = (short) (LeaveGroupResponseData.SCHEMAS.length - 1); - this.data = new LeaveGroupResponseData(struct, latestVersion); - } - - public LeaveGroupResponse(Struct struct, short version) { - this.data = new LeaveGroupResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -129,17 +122,12 @@ public Map errorCounts() { } @Override - public String toString() { - return data.toString(); - } - - @Override - public Struct toStruct(short version) { - return data.toStruct(version); + protected LeaveGroupResponseData data() { + return data; } - public static LeaveGroupResponse parse(ByteBuffer buffer, short versionId) { - return new LeaveGroupResponse(ApiKeys.LEAVE_GROUP.parseResponse(versionId, buffer), versionId); + public static LeaveGroupResponse parse(ByteBuffer buffer, short version) { + return new LeaveGroupResponse(new LeaveGroupResponseData(new ByteBufferAccessor(buffer), version)); } @Override @@ -157,4 +145,9 @@ public boolean equals(Object other) { public int hashCode() { return Objects.hashCode(data); } + + @Override + public String toString() { + return data.toString(); + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java index ce3938530b615..ab7ec61f6c343 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.message.ListGroupsRequestData; import org.apache.kafka.common.message.ListGroupsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -66,15 +66,6 @@ public ListGroupsRequest(ListGroupsRequestData data, short version) { this.data = data; } - public ListGroupsRequest(Struct struct, short version) { - super(ApiKeys.LIST_GROUPS, version); - this.data = new ListGroupsRequestData(struct, version); - } - - public ListGroupsRequestData data() { - return data; - } - @Override public ListGroupsResponse getErrorResponse(int throttleTimeMs, Throwable e) { ListGroupsResponseData listGroupsResponseData = new ListGroupsResponseData(). @@ -87,11 +78,11 @@ public ListGroupsResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static ListGroupsRequest parse(ByteBuffer buffer, short version) { - return new ListGroupsRequest(ApiKeys.LIST_GROUPS.parseRequest(version, buffer), version); + return new ListGroupsRequest(new ListGroupsRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + public ListGroupsRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java index f3b3bb663595e..336368395b76f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.ListGroupsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -29,13 +29,10 @@ public class ListGroupsResponse extends AbstractResponse { private final ListGroupsResponseData data; public ListGroupsResponse(ListGroupsResponseData data) { + super(ApiKeys.LIST_GROUPS); this.data = data; } - public ListGroupsResponse(Struct struct, short version) { - this.data = new ListGroupsResponseData(struct, version); - } - public ListGroupsResponseData data() { return data; } @@ -50,13 +47,8 @@ public Map errorCounts() { return errorCounts(Errors.forCode(data.errorCode())); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - public static ListGroupsResponse parse(ByteBuffer buffer, short version) { - return new ListGroupsResponse(ApiKeys.LIST_GROUPS.responseSchema(version).read(buffer), version); + return new ListGroupsResponse(new ListGroupsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java index 07f2998a4b4a5..9ebb06bdd6511 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java @@ -34,8 +34,8 @@ import org.apache.kafka.common.message.ListOffsetResponseData.ListOffsetPartitionResponse; import org.apache.kafka.common.message.ListOffsetResponseData.ListOffsetTopicResponse; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class ListOffsetRequest extends AbstractRequest { public static final long EARLIEST_TIMESTAMP = -2L; @@ -80,7 +80,7 @@ public Builder setTargetTimes(List topics) { @Override public ListOffsetRequest build(short version) { - return new ListOffsetRequest(version, data); + return new ListOffsetRequest(data, version); } @Override @@ -92,15 +92,9 @@ public String toString() { /** * Private constructor with a specified version. */ - private ListOffsetRequest(short version, ListOffsetRequestData data) { + private ListOffsetRequest(ListOffsetRequestData data, short version) { super(ApiKeys.LIST_OFFSETS, version); this.data = data; - this.duplicatePartitions = Collections.emptySet(); - } - - public ListOffsetRequest(Struct struct, short version) { - super(ApiKeys.LIST_OFFSETS, version); - data = new ListOffsetRequestData(struct, version); duplicatePartitions = new HashSet<>(); Set partitions = new HashSet<>(); for (ListOffsetTopic topic : data.topics()) { @@ -143,6 +137,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new ListOffsetResponse(responseData); } + @Override public ListOffsetRequestData data() { return data; } @@ -164,12 +159,7 @@ public Set duplicatePartitions() { } public static ListOffsetRequest parse(ByteBuffer buffer, short version) { - return new ListOffsetRequest(ApiKeys.LIST_OFFSETS.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new ListOffsetRequest(new ListOffsetRequestData(new ByteBufferAccessor(buffer), version), version); } public static List toListOffsetTopics(Map timestampsToSearch) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java index dd941d742ab90..ca57382d9c0ca 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java @@ -27,8 +27,8 @@ import org.apache.kafka.common.message.ListOffsetResponseData.ListOffsetPartitionResponse; import org.apache.kafka.common.message.ListOffsetResponseData.ListOffsetTopicResponse; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; /** @@ -55,18 +55,16 @@ public class ListOffsetResponse extends AbstractResponse { private final ListOffsetResponseData data; public ListOffsetResponse(ListOffsetResponseData data) { + super(ApiKeys.LIST_OFFSETS); this.data = data; } - public ListOffsetResponse(Struct struct, short version) { - data = new ListOffsetResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); } + @Override public ListOffsetResponseData data() { return data; } @@ -87,12 +85,7 @@ public Map errorCounts() { } public static ListOffsetResponse parse(ByteBuffer buffer, short version) { - return new ListOffsetResponse(ApiKeys.LIST_OFFSETS.parseResponse(version, buffer), version); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + return new ListOffsetResponse(new ListOffsetResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java index 0aa5e557556ea..86cfc6632b7a0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java @@ -21,7 +21,7 @@ import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingPartitionReassignment; import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -52,38 +52,21 @@ public String toString() { } private ListPartitionReassignmentsRequestData data; - private final short version; private ListPartitionReassignmentsRequest(ListPartitionReassignmentsRequestData data, short version) { super(ApiKeys.LIST_PARTITION_REASSIGNMENTS, version); this.data = data; - this.version = version; - } - - ListPartitionReassignmentsRequest(Struct struct, short version) { - super(ApiKeys.LIST_PARTITION_REASSIGNMENTS, version); - this.data = new ListPartitionReassignmentsRequestData(struct, version); - this.version = version; } public static ListPartitionReassignmentsRequest parse(ByteBuffer buffer, short version) { - return new ListPartitionReassignmentsRequest( - ApiKeys.LIST_PARTITION_REASSIGNMENTS.parseRequest(version, buffer), version - ); + return new ListPartitionReassignmentsRequest(new ListPartitionReassignmentsRequestData( + new ByteBufferAccessor(buffer), version), version); } public ListPartitionReassignmentsRequestData data() { return data; } - /** - * Visible for testing. - */ - @Override - public Struct toStruct() { - return data.toStruct(version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { ApiError apiError = ApiError.fromThrowable(e); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java index dd4e6989d606b..3449f1cabe9ba 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -28,20 +28,14 @@ public class ListPartitionReassignmentsResponse extends AbstractResponse { private final ListPartitionReassignmentsResponseData data; - public ListPartitionReassignmentsResponse(Struct struct) { - this(struct, ApiKeys.LIST_PARTITION_REASSIGNMENTS.latestVersion()); - } - public ListPartitionReassignmentsResponse(ListPartitionReassignmentsResponseData responseData) { + super(ApiKeys.LIST_PARTITION_REASSIGNMENTS); this.data = responseData; } - ListPartitionReassignmentsResponse(Struct struct, short version) { - this.data = new ListPartitionReassignmentsResponseData(struct, version); - } - public static ListPartitionReassignmentsResponse parse(ByteBuffer buffer, short version) { - return new ListPartitionReassignmentsResponse(ApiKeys.LIST_PARTITION_REASSIGNMENTS.responseSchema(version).read(buffer), version); + return new ListPartitionReassignmentsResponse(new ListPartitionReassignmentsResponseData( + new ByteBufferAccessor(buffer), version)); } public ListPartitionReassignmentsResponseData data() { @@ -62,9 +56,4 @@ public int throttleTimeMs() { public Map errorCounts() { return errorCounts(Errors.forCode(data.errorCode())); } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java index c2533e4f3779c..5aceb2291ff45 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; import org.apache.kafka.common.message.MetadataResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collection; @@ -102,18 +102,10 @@ public String toString() { } private final MetadataRequestData data; - private final short version; public MetadataRequest(MetadataRequestData data, short version) { super(ApiKeys.METADATA, version); this.data = data; - this.version = version; - } - - public MetadataRequest(Struct struct, short version) { - super(ApiKeys.METADATA, version); - this.data = new MetadataRequestData(struct, version); - this.version = version; } public MetadataRequestData data() { @@ -125,7 +117,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); MetadataResponseData responseData = new MetadataResponseData(); if (topics() != null) { - for (String topic :topics()) + for (String topic : topics()) responseData.topics().add(new MetadataResponseData.MetadataResponseTopic() .setName(topic) .setErrorCode(error.code()) @@ -134,12 +126,12 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } responseData.setThrottleTimeMs(throttleTimeMs); - return new MetadataResponse(responseData); + return new MetadataResponse(responseData, true); } public boolean isAllTopics() { return (data.topics() == null) || - (data.topics().isEmpty() && version == 0); //In version 0, an empty topic list indicates + (data.topics().isEmpty() && version() == 0); //In version 0, an empty topic list indicates // "request metadata for all topics." } @@ -158,7 +150,7 @@ public boolean allowAutoTopicCreation() { } public static MetadataRequest parse(ByteBuffer buffer, short version) { - return new MetadataRequest(ApiKeys.METADATA.parseRequest(version, buffer), version); + return new MetadataRequest(new MetadataRequestData(new ByteBufferAccessor(buffer), version), version); } public static List convertToMetadataRequestTopic(final Collection topics) { @@ -166,9 +158,4 @@ public static List convertToMetadataRequestTopic(final Col .setName(topic)) .collect(Collectors.toList()); } - - @Override - protected Struct toStruct() { - return data.toStruct(version); - } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java index 2014a05c7b0f9..ba06439bad1ec 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java @@ -25,9 +25,8 @@ import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; -import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.utils.Utils; import java.nio.ByteBuffer; @@ -64,28 +63,19 @@ public class MetadataResponse extends AbstractResponse { private volatile Holder holder; private final boolean hasReliableLeaderEpochs; - public MetadataResponse(MetadataResponseData data) { - this(data, true); + public MetadataResponse(MetadataResponseData data, short version) { + this(data, hasReliableLeaderEpochs(version)); } - public MetadataResponse(Struct struct, short version) { - // Prior to Kafka version 2.4 (which coincides with Metadata version 9), the broker - // does not propagate leader epoch information accurately while a reassignment is in - // progress. Relying on a stale epoch can lead to FENCED_LEADER_EPOCH errors which - // can prevent consumption throughout the course of a reassignment. It is safer in - // this case to revert to the behavior in previous protocol versions which checks - // leader status only. - this(new MetadataResponseData(struct, version), version >= 9); - } - - private MetadataResponse(MetadataResponseData data, boolean hasReliableLeaderEpochs) { + MetadataResponse(MetadataResponseData data, boolean hasReliableLeaderEpochs) { + super(ApiKeys.METADATA); this.data = data; this.hasReliableLeaderEpochs = hasReliableLeaderEpochs; } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected MetadataResponseData data() { + return data; } @Override @@ -243,8 +233,19 @@ public boolean hasReliableLeaderEpochs() { return hasReliableLeaderEpochs; } + // Prior to Kafka version 2.4 (which coincides with Metadata version 9), the broker + // does not propagate leader epoch information accurately while a reassignment is in + // progress. Relying on a stale epoch can lead to FENCED_LEADER_EPOCH errors which + // can prevent consumption throughout the course of a reassignment. It is safer in + // this case to revert to the behavior in previous protocol versions which checks + // leader status only. + private static boolean hasReliableLeaderEpochs(short version) { + return version >= 9; + } + public static MetadataResponse parse(ByteBuffer buffer, short version) { - return new MetadataResponse(ApiKeys.METADATA.responseSchema(version).read(buffer), version); + return new MetadataResponse(new MetadataResponseData(new ByteBufferAccessor(buffer), version), + hasReliableLeaderEpochs(version)); } public static class TopicMetadata { @@ -429,79 +430,24 @@ private Collection createTopicMetadata(MetadataResponseData data) } - public static MetadataResponse prepareResponse(int throttleTimeMs, Collection brokers, String clusterId, - int controllerId, List topicMetadataList, - int clusterAuthorizedOperations, - short responseVersion) { - MetadataResponseData responseData = new MetadataResponseData(); - responseData.setThrottleTimeMs(throttleTimeMs); - brokers.forEach(broker -> - responseData.brokers().add(new MetadataResponseBroker() - .setNodeId(broker.id()) - .setHost(broker.host()) - .setPort(broker.port()) - .setRack(broker.rack())) - ); - - responseData.setClusterId(clusterId); - responseData.setControllerId(controllerId); - responseData.setClusterAuthorizedOperations(clusterAuthorizedOperations); - - topicMetadataList.forEach(topicMetadata -> { - MetadataResponseTopic metadataResponseTopic = new MetadataResponseTopic(); - metadataResponseTopic - .setErrorCode(topicMetadata.error.code()) - .setName(topicMetadata.topic) - .setIsInternal(topicMetadata.isInternal) - .setTopicAuthorizedOperations(topicMetadata.authorizedOperations); - - for (PartitionMetadata partitionMetadata : topicMetadata.partitionMetadata) { - metadataResponseTopic.partitions().add(new MetadataResponsePartition() - .setErrorCode(partitionMetadata.error.code()) - .setPartitionIndex(partitionMetadata.partition()) - .setLeaderId(partitionMetadata.leaderId.orElse(NO_LEADER_ID)) - .setLeaderEpoch(partitionMetadata.leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)) - .setReplicaNodes(partitionMetadata.replicaIds) - .setIsrNodes(partitionMetadata.inSyncReplicaIds) - .setOfflineReplicas(partitionMetadata.offlineReplicaIds)); - } - responseData.topics().add(metadataResponseTopic); - }); - return new MetadataResponse(responseData.toStruct(responseVersion), responseVersion); - } - - public static MetadataResponse prepareResponse(int throttleTimeMs, + public static MetadataResponse prepareResponse(short version, + int throttleTimeMs, Collection brokers, String clusterId, int controllerId, - List topicMetadataList, - short responseVersion) { - return prepareResponse(throttleTimeMs, brokers, clusterId, controllerId, topicMetadataList, - MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED, responseVersion); - } - - public static MetadataResponse prepareResponse(Collection brokers, - String clusterId, - int controllerId, - List topicMetadata, - short responseVersion) { - return prepareResponse(AbstractResponse.DEFAULT_THROTTLE_TIME, brokers, clusterId, controllerId, - topicMetadata, responseVersion); - } - - public static MetadataResponse prepareResponse(Collection brokers, - String clusterId, - int controllerId, - List topicMetadata) { - return prepareResponse(AbstractResponse.DEFAULT_THROTTLE_TIME, brokers, clusterId, controllerId, - topicMetadata, ApiKeys.METADATA.latestVersion()); + List topics, + int clusterAuthorizedOperations) { + return prepareResponse(hasReliableLeaderEpochs(version), throttleTimeMs, brokers, clusterId, controllerId, + topics, clusterAuthorizedOperations); } - public static MetadataResponse prepareResponse(int throttleTimeMs, - List topicMetadataList, + // Visible for testing + public static MetadataResponse prepareResponse(boolean hasReliableEpoch, + int throttleTimeMs, Collection brokers, String clusterId, int controllerId, + List topics, int clusterAuthorizedOperations) { MetadataResponseData responseData = new MetadataResponseData(); responseData.setThrottleTimeMs(throttleTimeMs); @@ -517,8 +463,8 @@ public static MetadataResponse prepareResponse(int throttleTimeMs, responseData.setControllerId(controllerId); responseData.setClusterAuthorizedOperations(clusterAuthorizedOperations); - topicMetadataList.forEach(topicMetadata -> responseData.topics().add(topicMetadata)); - return new MetadataResponse(responseData); + topics.forEach(topicMetadata -> responseData.topics().add(topicMetadata)); + return new MetadataResponse(responseData, hasReliableEpoch); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java index ba6d182af6950..6542ed425746a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java @@ -24,8 +24,8 @@ import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition; import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -68,19 +68,9 @@ public String toString() { } } - private final short version; - public OffsetCommitRequest(OffsetCommitRequestData data, short version) { super(ApiKeys.OFFSET_COMMIT, version); this.data = data; - this.version = version; - } - - - public OffsetCommitRequest(Struct struct, short version) { - super(ApiKeys.OFFSET_COMMIT, version); - this.data = new OffsetCommitRequestData(struct, version); - this.version = version; } public OffsetCommitRequestData data() { @@ -128,11 +118,6 @@ public OffsetCommitResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static OffsetCommitRequest parse(ByteBuffer buffer, short version) { - return new OffsetCommitRequest(ApiKeys.OFFSET_COMMIT.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version); + return new OffsetCommitRequest(new OffsetCommitRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java index bacc9ad30fe02..b2851a68deb84 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition; import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -50,10 +50,12 @@ public class OffsetCommitResponse extends AbstractResponse { private final OffsetCommitResponseData data; public OffsetCommitResponse(OffsetCommitResponseData data) { + super(ApiKeys.OFFSET_COMMIT); this.data = data; } public OffsetCommitResponse(int requestThrottleMs, Map responseData) { + super(ApiKeys.OFFSET_COMMIT); Map responseTopicDataMap = new HashMap<>(); @@ -79,15 +81,6 @@ public OffsetCommitResponse(Map responseData) { this(DEFAULT_THROTTLE_TIME, responseData); } - public OffsetCommitResponse(Struct struct) { - short latestVersion = (short) (OffsetCommitResponseData.SCHEMAS.length - 1); - this.data = new OffsetCommitResponseData(struct, latestVersion); - } - - public OffsetCommitResponse(Struct struct, short version) { - this.data = new OffsetCommitResponseData(struct, version); - } - public OffsetCommitResponseData data() { return data; } @@ -100,12 +93,7 @@ public Map errorCounts() { } public static OffsetCommitResponse parse(ByteBuffer buffer, short version) { - return new OffsetCommitResponse(ApiKeys.OFFSET_COMMIT.parseResponse(version, buffer), version); - } - - @Override - public Struct toStruct(short version) { - return data.toStruct(version); + return new OffsetCommitResponse(new OffsetCommitResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java index 293efc2f6f2db..2f9d0203a9d7a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.OffsetDeleteRequestData; import org.apache.kafka.common.message.OffsetDeleteResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -53,11 +53,6 @@ public OffsetDeleteRequest(OffsetDeleteRequestData data, short version) { this.data = data; } - public OffsetDeleteRequest(Struct struct, short version) { - super(ApiKeys.OFFSET_DELETE, version); - this.data = new OffsetDeleteRequestData(struct, version); - } - public AbstractResponse getErrorResponse(int throttleTimeMs, Errors error) { return new OffsetDeleteResponse( new OffsetDeleteResponseData() @@ -72,12 +67,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static OffsetDeleteRequest parse(ByteBuffer buffer, short version) { - return new OffsetDeleteRequest(ApiKeys.OFFSET_DELETE.parseRequest(version, buffer), - version); + return new OffsetDeleteRequest(new OffsetDeleteRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected OffsetDeleteRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java index d52ab4dd49472..ceb932fb21d1e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -47,21 +47,13 @@ public class OffsetDeleteResponse extends AbstractResponse { public final OffsetDeleteResponseData data; public OffsetDeleteResponse(OffsetDeleteResponseData data) { + super(ApiKeys.OFFSET_DELETE); this.data = data; } - public OffsetDeleteResponse(Struct struct) { - short latestVersion = (short) (OffsetDeleteResponseData.SCHEMAS.length - 1); - this.data = new OffsetDeleteResponseData(struct, latestVersion); - } - - public OffsetDeleteResponse(Struct struct, short version) { - this.data = new OffsetDeleteResponseData(struct, version); - } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected OffsetDeleteResponseData data() { + return data; } @Override @@ -77,7 +69,7 @@ public Map errorCounts() { } public static OffsetDeleteResponse parse(ByteBuffer buffer, short version) { - return new OffsetDeleteResponse(ApiKeys.OFFSET_DELETE.parseResponse(version, buffer)); + return new OffsetDeleteResponse(new OffsetDeleteResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java index 382ca4ad93614..fd08c725399cb 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.OffsetFetchRequestData; import org.apache.kafka.common.message.OffsetFetchRequestData.OffsetFetchRequestTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -133,11 +133,6 @@ private OffsetFetchRequest(OffsetFetchRequestData data, short version) { this.data = data; } - public OffsetFetchRequest(Struct struct, short version) { - super(ApiKeys.OFFSET_FETCH, version); - this.data = new OffsetFetchRequestData(struct, version); - } - public OffsetFetchResponse getErrorResponse(Errors error) { return getErrorResponse(AbstractResponse.DEFAULT_THROTTLE_TIME, error); } @@ -172,7 +167,7 @@ public OffsetFetchResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static OffsetFetchRequest parse(ByteBuffer buffer, short version) { - return new OffsetFetchRequest(ApiKeys.OFFSET_FETCH.parseRequest(version, buffer), version); + return new OffsetFetchRequest(new OffsetFetchRequestData(new ByteBufferAccessor(buffer), version), version); } public boolean isAllPartitions() { @@ -180,7 +175,7 @@ public boolean isAllPartitions() { } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected OffsetFetchRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java index 8b550bef7aed5..3e30968715f8d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponsePartition; import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -129,6 +129,7 @@ public OffsetFetchResponse(Errors error, Map resp * @param responseData Fetched offset information grouped by topic-partition */ public OffsetFetchResponse(int throttleTimeMs, Errors error, Map responseData) { + super(ApiKeys.OFFSET_FETCH); Map offsetFetchResponseTopicMap = new HashMap<>(); for (Map.Entry entry : responseData.entrySet()) { String topicName = entry.getKey().topic(); @@ -153,25 +154,26 @@ public OffsetFetchResponse(int throttleTimeMs, Errors error, Map= 2 ? Errors.forCode(data.errorCode()) : topLevelError(data); + } - Errors topLevelError = Errors.NONE; + private static Errors topLevelError(OffsetFetchResponseData data) { for (OffsetFetchResponseTopic topic : data.topics()) { for (OffsetFetchResponsePartition partition : topic.partitions()) { Errors partitionError = Errors.forCode(partition.errorCode()); if (partitionError != Errors.NONE && !PARTITION_ERRORS.contains(partitionError)) { - topLevelError = partitionError; - break; + return partitionError; } } } - - // for version 2 and later use the top-level error code (in ERROR_CODE_KEY_NAME) from the response. - // for older versions there is no top-level error in the response and all errors are partition errors, - // so if there is a group or coordinator error at the partition level use that as the top-level error. - // this way clients can depend on the top-level error regardless of the offset fetch version. - this.error = version >= 2 ? Errors.forCode(data.errorCode()) : topLevelError; + return Errors.NONE; } @Override @@ -213,12 +215,12 @@ public Map responseData() { } public static OffsetFetchResponse parse(ByteBuffer buffer, short version) { - return new OffsetFetchResponse(ApiKeys.OFFSET_FETCH.parseResponse(version, buffer), version); + return new OffsetFetchResponse(new OffsetFetchResponseData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected OffsetFetchResponseData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java index 6d377b257be3e..727c7087c439a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java @@ -26,8 +26,8 @@ import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset; import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.OffsetForLeaderTopicResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import java.nio.ByteBuffer; @@ -38,6 +38,7 @@ import static org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.UNDEFINED_EPOCH_OFFSET; public class OffsetsForLeaderEpochRequest extends AbstractRequest { + /** * Sentinel replica_id value to indicate a regular consumer rather than another broker */ @@ -109,11 +110,7 @@ public OffsetsForLeaderEpochRequest(OffsetForLeaderEpochRequestData data, short this.data = data; } - public OffsetsForLeaderEpochRequest(Struct struct, short version) { - super(ApiKeys.OFFSET_FOR_LEADER_EPOCH, version); - this.data = new OffsetForLeaderEpochRequestData(struct, version); - } - + @Override public OffsetForLeaderEpochRequestData data() { return data; } @@ -123,12 +120,7 @@ public int replicaId() { } public static OffsetsForLeaderEpochRequest parse(ByteBuffer buffer, short version) { - return new OffsetsForLeaderEpochRequest(ApiKeys.OFFSET_FOR_LEADER_EPOCH.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new OffsetsForLeaderEpochRequest(new OffsetForLeaderEpochRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java index d737051724f17..893d5a2af20a0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -46,13 +46,11 @@ public class OffsetsForLeaderEpochResponse extends AbstractResponse { private final OffsetForLeaderEpochResponseData data; public OffsetsForLeaderEpochResponse(OffsetForLeaderEpochResponseData data) { + super(ApiKeys.OFFSET_FOR_LEADER_EPOCH); this.data = data; } - public OffsetsForLeaderEpochResponse(Struct struct, short version) { - data = new OffsetForLeaderEpochResponseData(struct, version); - } - + @Override public OffsetForLeaderEpochResponseData data() { return data; } @@ -71,12 +69,7 @@ public int throttleTimeMs() { } public static OffsetsForLeaderEpochResponse parse(ByteBuffer buffer, short version) { - return new OffsetsForLeaderEpochResponse(ApiKeys.OFFSET_FOR_LEADER_EPOCH.responseSchema(version).read(buffer), version); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + return new OffsetsForLeaderEpochResponse(new OffsetForLeaderEpochResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java index bf1a38f9dfab2..758631a1d87aa 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java @@ -21,12 +21,9 @@ import org.apache.kafka.common.errors.UnsupportedCompressionTypeException; import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.message.ProduceResponseData; -import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.SendBuilder; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.BaseRecords; import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.RecordBatch; @@ -130,11 +127,6 @@ public ProduceRequest(ProduceRequestData produceRequestData, short version) { this.transactionalId = data.transactionalId(); } - @Override - public Send toSend(String destination, RequestHeader header) { - return SendBuilder.buildRequestSend(destination, header, dataOrException()); - } - // visible for testing Map partitionSizes() { if (partitionSizes == null) { @@ -158,7 +150,8 @@ Map partitionSizes() { /** * @return data or IllegalStateException if the data is removed (to prevent unnecessary memory retention). */ - public ProduceRequestData dataOrException() { + @Override + public ProduceRequestData data() { // Store it in a local variable to protect against concurrent updates ProduceRequestData tmp = data; if (tmp == null) @@ -166,14 +159,6 @@ public ProduceRequestData dataOrException() { return tmp; } - /** - * Visible for testing. - */ - @Override - public Struct toStruct() { - return dataOrException().toStruct(version()); - } - @Override public String toString(boolean verbose) { // Use the same format as `Struct.toString()` @@ -194,7 +179,7 @@ public String toString(boolean verbose) { public ProduceResponse getErrorResponse(int throttleTimeMs, Throwable e) { /* In case the producer doesn't actually want any response */ if (acks == 0) return null; - Errors error = Errors.forException(e); + ApiError apiError = ApiError.fromThrowable(e); ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(throttleTimeMs); partitionSizes().forEach((tp, ignored) -> { ProduceResponseData.TopicProduceResponse tpr = data.responses().find(tp.topic()); @@ -208,8 +193,8 @@ public ProduceResponse getErrorResponse(int throttleTimeMs, Throwable e) { .setBaseOffset(INVALID_OFFSET) .setLogAppendTimeMs(RecordBatch.NO_TIMESTAMP) .setLogStartOffset(INVALID_OFFSET) - .setErrorMessage(e.getMessage()) - .setErrorCode(error.code())); + .setErrorMessage(apiError.message()) + .setErrorCode(apiError.error().code())); }); return new ProduceResponse(data); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java index d2b72408078d9..2860576d53972 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java @@ -18,11 +18,9 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.ProduceResponseData; -import org.apache.kafka.common.network.Send; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.SendBuilder; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import java.nio.ByteBuffer; @@ -60,6 +58,7 @@ public class ProduceResponse extends AbstractResponse { private final ProduceResponseData data; public ProduceResponse(ProduceResponseData produceResponseData) { + super(ApiKeys.PRODUCE); this.data = produceResponseData; } @@ -82,11 +81,6 @@ public ProduceResponse(Map responses, int thr this(toData(responses, throttleTimeMs)); } - @Override - protected Send toSend(String destination, ResponseHeader header, short apiVersion) { - return SendBuilder.buildResponseSend(destination, header, this.data, apiVersion); - } - private static ProduceResponseData toData(Map responses, int throttleTimeMs) { ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(throttleTimeMs); responses.forEach((tp, response) -> { @@ -113,14 +107,6 @@ private static ProduceResponseData toData(Map return data; } - /** - * Visible for testing. - */ - @Override - public Struct toStruct(short version) { - return data.toStruct(version); - } - public ProduceResponseData data() { return this.data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java index 2c83f06adda62..2c364aef3dff7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.RenewDelegationTokenRequestData; import org.apache.kafka.common.message.RenewDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class RenewDelegationTokenRequest extends AbstractRequest { @@ -33,18 +33,9 @@ public RenewDelegationTokenRequest(RenewDelegationTokenRequestData data, short v this.data = data; } - public RenewDelegationTokenRequest(Struct struct, short version) { - super(ApiKeys.RENEW_DELEGATION_TOKEN, version); - this.data = new RenewDelegationTokenRequestData(struct, version); - } - public static RenewDelegationTokenRequest parse(ByteBuffer buffer, short version) { - return new RenewDelegationTokenRequest(ApiKeys.RENEW_DELEGATION_TOKEN.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new RenewDelegationTokenRequest(new RenewDelegationTokenRequestData( + new ByteBufferAccessor(buffer), version), version); } public RenewDelegationTokenRequestData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java index dfbb5c8518fad..2ef840378acb3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java @@ -21,23 +21,21 @@ import org.apache.kafka.common.message.RenewDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class RenewDelegationTokenResponse extends AbstractResponse { private final RenewDelegationTokenResponseData data; public RenewDelegationTokenResponse(RenewDelegationTokenResponseData data) { + super(ApiKeys.RENEW_DELEGATION_TOKEN); this.data = data; } - public RenewDelegationTokenResponse(Struct struct, short version) { - data = new RenewDelegationTokenResponseData(struct, version); - } - public static RenewDelegationTokenResponse parse(ByteBuffer buffer, short version) { - return new RenewDelegationTokenResponse(ApiKeys.RENEW_DELEGATION_TOKEN.responseSchema(version).read(buffer), version); + return new RenewDelegationTokenResponse(new RenewDelegationTokenResponseData( + new ByteBufferAccessor(buffer), version)); } @Override @@ -46,8 +44,8 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected RenewDelegationTokenResponseData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java index 7136584d93a88..c90e2e5ef7c36 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.auth.KafkaPrincipalSerde; import org.apache.kafka.common.security.auth.SecurityProtocol; @@ -93,9 +92,7 @@ public RequestAndSize parseRequest(ByteBuffer buffer) { ApiKeys apiKey = header.apiKey(); try { short apiVersion = header.apiVersion(); - Struct struct = apiKey.parseRequest(apiVersion, buffer); - AbstractRequest body = AbstractRequest.parseRequest(apiKey, apiVersion, struct); - return new RequestAndSize(body, struct.sizeOf()); + return AbstractRequest.parseRequest(apiKey, apiVersion, buffer); } catch (Throwable ex) { throw new InvalidRequestException("Error getting request for apiKey: " + apiKey + ", apiVersion: " + header.apiVersion() + @@ -111,8 +108,7 @@ public RequestAndSize parseRequest(ByteBuffer buffer) { * over the network. */ public Send buildResponseSend(AbstractResponse body) { - ResponseHeader responseHeader = header.toResponseHeader(); - return body.toSend(connectionId, responseHeader, apiVersion()); + return body.toSend(connectionId, header.toResponseHeader(), apiVersion()); } /** @@ -126,8 +122,7 @@ public Send buildResponseSend(AbstractResponse body) { * so we do not lose the benefit of "zero copy" transfers from disk. */ public ByteBuffer buildResponseEnvelopePayload(AbstractResponse body) { - ResponseHeader responseHeader = header.toResponseHeader(); - return RequestUtils.serialize(responseHeader.toStruct(), body.toStruct(header.apiVersion())); + return body.serializeWithHeader(header.toResponseHeader(), apiVersion()); } private boolean isUnsupportedApiVersionsRequest() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java index d145f49280852..29539491d1469 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java @@ -21,7 +21,7 @@ import org.apache.kafka.common.message.RequestHeaderData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ObjectSerializationCache; import java.nio.ByteBuffer; @@ -32,20 +32,13 @@ public class RequestHeader implements AbstractRequestResponse { private final RequestHeaderData data; private final short headerVersion; - public RequestHeader(Struct struct, short headerVersion) { - this(new RequestHeaderData(struct, headerVersion), headerVersion); - } - - public RequestHeader(ApiKeys requestApiKey, - short requestVersion, - String clientId, - int correlationId) { - this(new RequestHeaderData() - .setRequestApiKey(requestApiKey.id) - .setRequestApiVersion(requestVersion) - .setClientId(clientId) - .setCorrelationId(correlationId), - ApiKeys.forId(requestApiKey.id).requestHeaderVersion(requestVersion)); + public RequestHeader(ApiKeys requestApiKey, short requestVersion, String clientId, int correlationId) { + this(new RequestHeaderData(). + setRequestApiKey(requestApiKey.id). + setRequestApiVersion(requestVersion). + setClientId(clientId). + setCorrelationId(correlationId), + requestApiKey.requestHeaderVersion(requestVersion)); } public RequestHeader(RequestHeaderData data, short headerVersion) { @@ -53,10 +46,6 @@ public RequestHeader(RequestHeaderData data, short headerVersion) { this.headerVersion = headerVersion; } - public Struct toStruct() { - return this.data.toStruct(headerVersion); - } - public ApiKeys apiKey() { return ApiKeys.forId(data.requestApiKey()); } @@ -81,14 +70,23 @@ public RequestHeaderData data() { return data; } + public void write(ByteBuffer buffer, ObjectSerializationCache serializationCache) { + data.write(new ByteBufferAccessor(buffer), serializationCache, headerVersion); + } + + public int size(ObjectSerializationCache serializationCache) { + return data.size(serializationCache, headerVersion); + } + public ResponseHeader toResponseHeader() { - return new ResponseHeader(data.correlationId(), - apiKey().responseHeaderVersion(apiVersion())); + return new ResponseHeader(data.correlationId(), apiKey().responseHeaderVersion(apiVersion())); } public static RequestHeader parse(ByteBuffer buffer) { short apiKey = -1; try { + // We derive the header version from the request api version, so we read that first. + // The request api version is part of `RequestHeaderData`, so we reset the buffer position after the read. int position = buffer.position(); apiKey = buffer.getShort(); short apiVersion = buffer.getShort(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java index ff42914ebff34..4fa701f76476d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java @@ -17,8 +17,10 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.ProduceRequestData; -import org.apache.kafka.common.protocol.types.Field; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.MessageSizeAccumulator; +import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.record.BaseRecords; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.record.Records; @@ -32,28 +34,11 @@ public final class RequestUtils { private RequestUtils() {} - static void setLeaderEpochIfExists(Struct struct, Field.Int32 leaderEpochField, Optional leaderEpoch) { - struct.setIfExists(leaderEpochField, leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)); - } - - static Optional getLeaderEpoch(Struct struct, Field.Int32 leaderEpochField) { - int leaderEpoch = struct.getOrElse(leaderEpochField, RecordBatch.NO_PARTITION_LEADER_EPOCH); - return getLeaderEpoch(leaderEpoch); - } - static Optional getLeaderEpoch(int leaderEpoch) { return leaderEpoch == RecordBatch.NO_PARTITION_LEADER_EPOCH ? Optional.empty() : Optional.of(leaderEpoch); } - public static ByteBuffer serialize(Struct headerStruct, Struct bodyStruct) { - ByteBuffer buffer = ByteBuffer.allocate(headerStruct.sizeOf() + bodyStruct.sizeOf()); - headerStruct.writeTo(buffer); - bodyStruct.writeTo(buffer); - buffer.rewind(); - return buffer; - } - // visible for testing public static boolean hasIdempotentRecords(ProduceRequest request) { return flags(request).getKey(); @@ -72,7 +57,7 @@ public static boolean hasTransactionalRecords(ProduceRequest request) { public static AbstractMap.SimpleEntry flags(ProduceRequest request) { boolean hasIdempotentRecords = false; boolean hasTransactionalRecords = false; - for (ProduceRequestData.TopicProduceData tpd : request.dataOrException().topicData()) { + for (ProduceRequestData.TopicProduceData tpd : request.data().topicData()) { for (ProduceRequestData.PartitionProduceData ppd : tpd.partitionData()) { BaseRecords records = ppd.records(); if (records instanceof Records) { @@ -90,4 +75,37 @@ public static AbstractMap.SimpleEntry flags(ProduceRequest req } return new AbstractMap.SimpleEntry<>(hasIdempotentRecords, hasTransactionalRecords); } + + public static MessageSizeAccumulator size( + ObjectSerializationCache serializationCache, + Message header, + short headerVersion, + Message apiMessage, + short apiVersion + ) { + MessageSizeAccumulator messageSize = new MessageSizeAccumulator(); + if (header != null) + header.addSize(messageSize, serializationCache, headerVersion); + apiMessage.addSize(messageSize, serializationCache, apiVersion); + return messageSize; + } + + public static ByteBuffer serialize( + Message header, + short headerVersion, + Message apiMessage, + short apiVersion + ) { + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + MessageSizeAccumulator messageSize = RequestUtils.size(serializationCache, header, headerVersion, apiMessage, apiVersion); + + ByteBuffer buffer = ByteBuffer.allocate(messageSize.totalSize()); + ByteBufferAccessor bufferWritable = new ByteBufferAccessor(buffer); + if (header != null) + header.write(bufferWritable, serializationCache, headerVersion); + apiMessage.write(bufferWritable, serializationCache, apiVersion); + + buffer.rewind(); + return buffer; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java index 3fe7d9ebf4c7b..d45e28dc1d81d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java @@ -18,7 +18,7 @@ import org.apache.kafka.common.message.ResponseHeaderData; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ObjectSerializationCache; import java.nio.ByteBuffer; import java.util.Objects; @@ -30,10 +30,6 @@ public class ResponseHeader implements AbstractRequestResponse { private final ResponseHeaderData data; private final short headerVersion; - public ResponseHeader(Struct struct, short headerVersion) { - this(new ResponseHeaderData(struct, headerVersion), headerVersion); - } - public ResponseHeader(int correlationId, short headerVersion) { this(new ResponseHeaderData().setCorrelationId(correlationId), headerVersion); } @@ -43,12 +39,8 @@ public ResponseHeader(ResponseHeaderData data, short headerVersion) { this.headerVersion = headerVersion; } - public int sizeOf() { - return toStruct().sizeOf(); - } - - public Struct toStruct() { - return data.toStruct(headerVersion); + public int size(ObjectSerializationCache serializationCache) { + return data().size(serializationCache, headerVersion); } public int correlationId() { @@ -63,6 +55,18 @@ public ResponseHeaderData data() { return data; } + public void write(ByteBuffer buffer, ObjectSerializationCache serializationCache) { + data.write(new ByteBufferAccessor(buffer), serializationCache, headerVersion); + } + + @Override + public String toString() { + return "ResponseHeader(" + + "correlationId=" + data.correlationId() + + ", headerVersion=" + headerVersion + + ")"; + } + public static ResponseHeader parse(ByteBuffer buffer, short headerVersion) { return new ResponseHeader( new ResponseHeaderData(new ByteBufferAccessor(buffer), headerVersion), diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java index 5fdfbec66a923..35da4caf9c5c1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.message.SaslAuthenticateRequestData; import org.apache.kafka.common.message.SaslAuthenticateResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; @@ -49,28 +49,17 @@ public SaslAuthenticateRequest build(short version) { @Override public String toString() { - StringBuilder bld = new StringBuilder(); - bld.append("(type=SaslAuthenticateRequest)"); - return bld.toString(); + return "(type=SaslAuthenticateRequest)"; } } private final SaslAuthenticateRequestData data; - public SaslAuthenticateRequest(SaslAuthenticateRequestData data) { - this(data, ApiKeys.SASL_AUTHENTICATE.latestVersion()); - } - public SaslAuthenticateRequest(SaslAuthenticateRequestData data, short version) { super(ApiKeys.SASL_AUTHENTICATE, version); this.data = data; } - public SaslAuthenticateRequest(Struct struct, short version) { - super(ApiKeys.SASL_AUTHENTICATE, version); - this.data = new SaslAuthenticateRequestData(struct, version); - } - public SaslAuthenticateRequestData data() { return data; } @@ -85,12 +74,8 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static SaslAuthenticateRequest parse(ByteBuffer buffer, short version) { - return new SaslAuthenticateRequest(ApiKeys.SASL_AUTHENTICATE.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new SaslAuthenticateRequest(new SaslAuthenticateRequestData(new ByteBufferAccessor(buffer), version), + version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java index 650252bbd7200..2e1a2189a39be 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java @@ -18,13 +18,12 @@ import org.apache.kafka.common.message.SaslAuthenticateResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; - /** * Response from SASL server which for a SASL challenge as defined by the SASL protocol * for the mechanism configured for the client. @@ -34,13 +33,10 @@ public class SaslAuthenticateResponse extends AbstractResponse { private final SaslAuthenticateResponseData data; public SaslAuthenticateResponse(SaslAuthenticateResponseData data) { + super(ApiKeys.SASL_AUTHENTICATE); this.data = data; } - public SaslAuthenticateResponse(Struct struct, short version) { - this.data = new SaslAuthenticateResponseData(struct, version); - } - /** * Possible error codes: * SASL_AUTHENTICATION_FAILED(57) : Authentication failed @@ -67,11 +63,16 @@ public byte[] saslAuthBytes() { } @Override - public Struct toStruct(short version) { - return data.toStruct(version); + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + + @Override + protected SaslAuthenticateResponseData data() { + return data; } public static SaslAuthenticateResponse parse(ByteBuffer buffer, short version) { - return new SaslAuthenticateResponse(ApiKeys.SASL_AUTHENTICATE.parseResponse(version, buffer), version); + return new SaslAuthenticateResponse(new SaslAuthenticateResponseData(new ByteBufferAccessor(buffer), version)); } -} \ No newline at end of file +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java index c2b5f729256a8..e64dd6b2cb792 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java @@ -20,7 +20,7 @@ import org.apache.kafka.common.message.SaslHandshakeRequestData; import org.apache.kafka.common.message.SaslHandshakeResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; @@ -56,20 +56,11 @@ public String toString() { private final SaslHandshakeRequestData data; - public SaslHandshakeRequest(SaslHandshakeRequestData data) { - this(data, ApiKeys.SASL_HANDSHAKE.latestVersion()); - } - public SaslHandshakeRequest(SaslHandshakeRequestData data, short version) { super(ApiKeys.SASL_HANDSHAKE, version); this.data = data; } - public SaslHandshakeRequest(Struct struct, short version) { - super(ApiKeys.SASL_HANDSHAKE, version); - this.data = new SaslHandshakeRequestData(struct, version); - } - public SaslHandshakeRequestData data() { return data; } @@ -82,12 +73,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static SaslHandshakeRequest parse(ByteBuffer buffer, short version) { - return new SaslHandshakeRequest(ApiKeys.SASL_HANDSHAKE.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new SaslHandshakeRequest(new SaslHandshakeRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java index 161c075aa5318..b0ac2efcdec5f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.SaslHandshakeResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.List; @@ -34,13 +34,10 @@ public class SaslHandshakeResponse extends AbstractResponse { private final SaslHandshakeResponseData data; public SaslHandshakeResponse(SaslHandshakeResponseData data) { + super(ApiKeys.SASL_HANDSHAKE); this.data = data; } - public SaslHandshakeResponse(Struct struct, short version) { - this.data = new SaslHandshakeResponseData(struct, version); - } - /* * Possible error codes: * UNSUPPORTED_SASL_MECHANISM(33): Client mechanism not enabled in server @@ -56,8 +53,13 @@ public Map errorCounts() { } @Override - public Struct toStruct(short version) { - return data.toStruct(version); + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + + @Override + protected SaslHandshakeResponseData data() { + return data; } public List enabledMechanisms() { @@ -65,6 +67,6 @@ public List enabledMechanisms() { } public static SaslHandshakeResponse parse(ByteBuffer buffer, short version) { - return new SaslHandshakeResponse(ApiKeys.SASL_HANDSHAKE.parseResponse(version, buffer), version); + return new SaslHandshakeResponse(new SaslHandshakeResponseData(new ByteBufferAccessor(buffer), version)); } -} \ No newline at end of file +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java index 8ab8cc38ecad7..2b0aef766935b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java @@ -25,8 +25,8 @@ import org.apache.kafka.common.message.StopReplicaResponseData; import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.MappedIterator; import org.apache.kafka.common.utils.Utils; @@ -103,10 +103,6 @@ private StopReplicaRequest(StopReplicaRequestData data, short version) { this.data = data; } - public StopReplicaRequest(Struct struct, short version) { - this(new StopReplicaRequestData(struct, version), version); - } - @Override public StopReplicaResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); @@ -211,15 +207,11 @@ public long brokerEpoch() { } public static StopReplicaRequest parse(ByteBuffer buffer, short version) { - return new StopReplicaRequest(ApiKeys.STOP_REPLICA.parseRequest(version, buffer), version); - } - - public StopReplicaRequestData data() { - return data; + return new StopReplicaRequest(new StopReplicaRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected StopReplicaRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java index 78b0d98c51a4a..27a2502e75103 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.StopReplicaResponseData; import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -39,13 +39,10 @@ public class StopReplicaResponse extends AbstractResponse { private final StopReplicaResponseData data; public StopReplicaResponse(StopReplicaResponseData data) { + super(ApiKeys.STOP_REPLICA); this.data = data; } - public StopReplicaResponse(Struct struct, short version) { - data = new StopReplicaResponseData(struct, version); - } - public List partitionErrors() { return data.partitionErrors(); } @@ -65,12 +62,17 @@ public Map errorCounts() { } public static StopReplicaResponse parse(ByteBuffer buffer, short version) { - return new StopReplicaResponse(ApiKeys.STOP_REPLICA.parseResponse(version, buffer), version); + return new StopReplicaResponse(new StopReplicaResponseData(new ByteBufferAccessor(buffer), version)); + } + + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected StopReplicaResponseData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java index f3b648791f1ee..18e5eb8b80964 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.message.SyncGroupRequestData; import org.apache.kafka.common.message.SyncGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -60,11 +60,6 @@ public SyncGroupRequest(SyncGroupRequestData data, short version) { this.data = data; } - public SyncGroupRequest(Struct struct, short version) { - super(ApiKeys.SYNC_GROUP, version); - this.data = new SyncGroupRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new SyncGroupResponse(new SyncGroupResponseData() @@ -93,11 +88,11 @@ public boolean areMandatoryProtocolTypeAndNamePresent() { } public static SyncGroupRequest parse(ByteBuffer buffer, short version) { - return new SyncGroupRequest(ApiKeys.SYNC_GROUP.parseRequest(version, buffer), version); + return new SyncGroupRequest(new SyncGroupRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected SyncGroupRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java index 7f02d2d25bc7e..a26b6c900bbdf 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.SyncGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -29,18 +29,10 @@ public class SyncGroupResponse extends AbstractResponse { public final SyncGroupResponseData data; public SyncGroupResponse(SyncGroupResponseData data) { + super(ApiKeys.SYNC_GROUP); this.data = data; } - public SyncGroupResponse(Struct struct) { - short latestVersion = (short) (SyncGroupResponseData.SCHEMAS.length - 1); - this.data = new SyncGroupResponseData(struct, latestVersion); - } - - public SyncGroupResponse(Struct struct, short version) { - this.data = new SyncGroupResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -56,8 +48,8 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected SyncGroupResponseData data() { + return data; } @Override @@ -66,7 +58,7 @@ public String toString() { } public static SyncGroupResponse parse(ByteBuffer buffer, short version) { - return new SyncGroupResponse(ApiKeys.SYNC_GROUP.parseResponse(version, buffer), version); + return new SyncGroupResponse(new SyncGroupResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java index bee9c134cfa29..055793cf54b15 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java @@ -25,8 +25,8 @@ import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition; import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -129,11 +129,6 @@ public TxnOffsetCommitRequest(TxnOffsetCommitRequestData data, short version) { this.data = data; } - public TxnOffsetCommitRequest(Struct struct, short version) { - super(ApiKeys.TXN_OFFSET_COMMIT, version); - this.data = new TxnOffsetCommitRequestData(struct, version); - } - public Map offsets() { List topics = data.topics(); Map offsetMap = new HashMap<>(); @@ -173,8 +168,8 @@ static List getTopics(Map getErrorResponseTopics(List requestTopics, @@ -206,7 +201,8 @@ public TxnOffsetCommitResponse getErrorResponse(int throttleTimeMs, Throwable e) } public static TxnOffsetCommitRequest parse(ByteBuffer buffer, short version) { - return new TxnOffsetCommitRequest(ApiKeys.TXN_OFFSET_COMMIT.parseRequest(version, buffer), version); + return new TxnOffsetCommitRequest(new TxnOffsetCommitRequestData( + new ByteBufferAccessor(buffer), version), version); } public static class CommittedOffset { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java index c71719dad33a0..1ead0ec6d6073 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition; import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -50,14 +50,12 @@ public class TxnOffsetCommitResponse extends AbstractResponse { public final TxnOffsetCommitResponseData data; public TxnOffsetCommitResponse(TxnOffsetCommitResponseData data) { + super(ApiKeys.TXN_OFFSET_COMMIT); this.data = data; } - public TxnOffsetCommitResponse(Struct struct, short version) { - this.data = new TxnOffsetCommitResponseData(struct, version); - } - public TxnOffsetCommitResponse(int requestThrottleMs, Map responseData) { + super(ApiKeys.TXN_OFFSET_COMMIT); Map responseTopicDataMap = new HashMap<>(); for (Map.Entry entry : responseData.entrySet()) { @@ -80,8 +78,8 @@ public TxnOffsetCommitResponse(int requestThrottleMs, Map errors() { } public static TxnOffsetCommitResponse parse(ByteBuffer buffer, short version) { - return new TxnOffsetCommitResponse(ApiKeys.TXN_OFFSET_COMMIT.parseResponse(version, buffer), version); + return new TxnOffsetCommitResponse(new TxnOffsetCommitResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesRequest.java index 3276c0abbe438..d61056f64bedc 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesRequest.java @@ -23,7 +23,7 @@ import org.apache.kafka.common.message.UpdateFeaturesResponseData.UpdatableFeatureResult; import org.apache.kafka.common.message.UpdateFeaturesResponseData.UpdatableFeatureResultCollection; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; public class UpdateFeaturesRequest extends AbstractRequest { @@ -54,11 +54,6 @@ public UpdateFeaturesRequest(UpdateFeaturesRequestData data, short version) { this.data = data; } - public UpdateFeaturesRequest(Struct struct, short version) { - super(ApiKeys.UPDATE_FEATURES, version); - this.data = new UpdateFeaturesRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { final ApiError apiError = ApiError.fromThrowable(e); @@ -73,11 +68,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { final UpdateFeaturesResponseData responseData = new UpdateFeaturesResponseData() .setThrottleTimeMs(throttleTimeMs) .setResults(results); - return new UpdateFeaturesResponse(responseData); } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new UpdateFeaturesResponse(responseData); } public UpdateFeaturesRequestData data() { @@ -85,8 +76,7 @@ public UpdateFeaturesRequestData data() { } public static UpdateFeaturesRequest parse(ByteBuffer buffer, short version) { - return new UpdateFeaturesRequest( - ApiKeys.UPDATE_FEATURES.parseRequest(version, buffer), version); + return new UpdateFeaturesRequest(new UpdateFeaturesRequestData(new ByteBufferAccessor(buffer), version), version); } public static boolean isDeleteRequest(UpdateFeaturesRequestData.FeatureUpdateKey update) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesResponse.java index 5754f13595c8b..2c28842cd89d6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesResponse.java @@ -19,12 +19,13 @@ import java.nio.ByteBuffer; import java.util.Map; import java.util.stream.Collectors; + import org.apache.kafka.common.message.UpdateFeaturesResponseData; import org.apache.kafka.common.message.UpdateFeaturesResponseData.UpdatableFeatureResult; import org.apache.kafka.common.message.UpdateFeaturesResponseData.UpdatableFeatureResultCollection; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; /** @@ -40,18 +41,10 @@ public class UpdateFeaturesResponse extends AbstractResponse { private final UpdateFeaturesResponseData data; public UpdateFeaturesResponse(UpdateFeaturesResponseData data) { + super(ApiKeys.UPDATE_FEATURES); this.data = data; } - public UpdateFeaturesResponse(Struct struct) { - final short latestVersion = (short) (UpdateFeaturesResponseData.SCHEMAS.length - 1); - this.data = new UpdateFeaturesResponseData(struct, latestVersion); - } - - public UpdateFeaturesResponse(Struct struct, short version) { - this.data = new UpdateFeaturesResponseData(struct, version); - } - public Map errors() { return data.results().valuesSet().stream().collect( Collectors.toMap( @@ -69,11 +62,6 @@ public int throttleTimeMs() { return data.throttleTimeMs(); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public String toString() { return data.toString(); @@ -84,7 +72,7 @@ public UpdateFeaturesResponseData data() { } public static UpdateFeaturesResponse parse(ByteBuffer buffer, short version) { - return new UpdateFeaturesResponse(ApiKeys.UPDATE_FEATURES.parseResponse(version, buffer), version); + return new UpdateFeaturesResponse(new UpdateFeaturesResponseData(new ByteBufferAccessor(buffer), version)); } public static UpdateFeaturesResponse createWithErrors(ApiError topLevelError, Map updateErrors, int throttleTimeMs) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java index 7ae861b5c3df0..a0ef3471f12dc 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java @@ -25,8 +25,8 @@ import org.apache.kafka.common.message.UpdateMetadataResponseData; import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.utils.FlattenedIterator; import org.apache.kafka.common.utils.Utils; @@ -166,10 +166,6 @@ private static String listenerNameFromSecurityProtocol(UpdateMetadataEndpoint en return ListenerName.forSecurityProtocol(securityProtocol).value(); } - public UpdateMetadataRequest(Struct struct, short version) { - this(new UpdateMetadataRequestData(struct, version), version); - } - @Override public int controllerId() { return data.controllerId(); @@ -205,15 +201,11 @@ public List liveBrokers() { } @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - - public UpdateMetadataRequestData data() { + protected UpdateMetadataRequestData data() { return data; } public static UpdateMetadataRequest parse(ByteBuffer buffer, short version) { - return new UpdateMetadataRequest(ApiKeys.UPDATE_METADATA.parseRequest(version, buffer), version); + return new UpdateMetadataRequest(new UpdateMetadataRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java index c7d803f560eb1..4749067a9a786 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.UpdateMetadataResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -29,13 +29,10 @@ public class UpdateMetadataResponse extends AbstractResponse { private final UpdateMetadataResponseData data; public UpdateMetadataResponse(UpdateMetadataResponseData data) { + super(ApiKeys.UPDATE_METADATA); this.data = data; } - public UpdateMetadataResponse(Struct struct, short version) { - this(new UpdateMetadataResponseData(struct, version)); - } - public Errors error() { return Errors.forCode(data.errorCode()); } @@ -45,12 +42,18 @@ public Map errorCounts() { return errorCounts(error()); } + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static UpdateMetadataResponse parse(ByteBuffer buffer, short version) { - return new UpdateMetadataResponse(ApiKeys.UPDATE_METADATA.parseResponse(version, buffer), version); + return new UpdateMetadataResponse(new UpdateMetadataResponseData(new ByteBufferAccessor(buffer), version)); } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected UpdateMetadataResponseData data() { + return data; } + } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/VoteRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/VoteRequest.java index 9fe5cfc128a01..d31d019206f0b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/VoteRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/VoteRequest.java @@ -20,12 +20,14 @@ import org.apache.kafka.common.message.VoteRequestData; import org.apache.kafka.common.message.VoteResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import java.nio.ByteBuffer; import java.util.Collections; public class VoteRequest extends AbstractRequest { + public static class Builder extends AbstractRequest.Builder { private final VoteRequestData data; @@ -52,14 +54,9 @@ private VoteRequest(VoteRequestData data, short version) { this.data = data; } - public VoteRequest(Struct struct, short version) { - super(ApiKeys.VOTE, version); - this.data = new VoteRequestData(struct, version); - } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected VoteRequestData data() { + return data; } @Override @@ -68,6 +65,10 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { .setErrorCode(Errors.forException(e).code())); } + public static VoteRequest parse(ByteBuffer buffer, short version) { + return new VoteRequest(new VoteRequestData(new ByteBufferAccessor(buffer), version), version); + } + public static VoteRequestData singletonRequest(TopicPartition topicPartition, int candidateEpoch, int candidateId, diff --git a/clients/src/main/java/org/apache/kafka/common/requests/VoteResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/VoteResponse.java index ab45d918dc4e8..5956fe0077e99 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/VoteResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/VoteResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.VoteResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -45,23 +45,10 @@ public class VoteResponse extends AbstractResponse { public final VoteResponseData data; public VoteResponse(VoteResponseData data) { + super(ApiKeys.VOTE); this.data = data; } - public VoteResponse(Struct struct, short version) { - this.data = new VoteResponseData(struct, version); - } - - public VoteResponse(Struct struct) { - short latestVersion = (short) (VoteResponseData.SCHEMAS.length - 1); - this.data = new VoteResponseData(struct, latestVersion); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - public static VoteResponseData singletonResponse(Errors topLevelError, TopicPartition topicPartition, Errors partitionLevelError, @@ -95,7 +82,17 @@ public Map errorCounts() { return errors; } + @Override + protected VoteResponseData data() { + return data; + } + + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static VoteResponse parse(ByteBuffer buffer, short version) { - return new VoteResponse(ApiKeys.VOTE.responseSchema(version).read(buffer), version); + return new VoteResponse(new VoteResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java index 88bde08dc4488..5998ca7e07e7e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.WriteTxnMarkersRequestData.WritableTxnMarker; import org.apache.kafka.common.message.WriteTxnMarkersRequestData.WritableTxnMarkerTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -141,14 +141,9 @@ private WriteTxnMarkersRequest(WriteTxnMarkersRequestData data, short version) { this.data = data; } - public WriteTxnMarkersRequest(Struct struct, short version) { - super(ApiKeys.WRITE_TXN_MARKERS, version); - this.data = new WriteTxnMarkersRequestData(struct, version); - } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected WriteTxnMarkersRequestData data() { + return data; } @Override @@ -190,7 +185,7 @@ public List markers() { } public static WriteTxnMarkersRequest parse(ByteBuffer buffer, short version) { - return new WriteTxnMarkersRequest(ApiKeys.WRITE_TXN_MARKERS.parseRequest(version, buffer), version); + return new WriteTxnMarkersRequest(new WriteTxnMarkersRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java index 8fdcb99c9333a..fd2a834d24569 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java @@ -22,8 +22,8 @@ import org.apache.kafka.common.message.WriteTxnMarkersResponseData.WritableTxnMarkerResult; import org.apache.kafka.common.message.WriteTxnMarkersResponseData.WritableTxnMarkerTopicResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -49,10 +49,10 @@ */ public class WriteTxnMarkersResponse extends AbstractResponse { - private final Map> errors; - public final WriteTxnMarkersResponseData data; + private final WriteTxnMarkersResponseData data; public WriteTxnMarkersResponse(Map> errors) { + super(ApiKeys.WRITE_TXN_MARKERS); List markers = new ArrayList<>(); for (Map.Entry> markerEntry : errors.entrySet()) { Map responseTopicDataMap = new HashMap<>(); @@ -74,48 +74,53 @@ public WriteTxnMarkersResponse(Map> errors) { .setTopics(new ArrayList<>(responseTopicDataMap.values())) ); } - - this.errors = errors; this.data = new WriteTxnMarkersResponseData() .setMarkers(markers); } - public WriteTxnMarkersResponse(Struct struct, short version) { - this.data = new WriteTxnMarkersResponseData(struct, version); - this.errors = new HashMap<>(); + public WriteTxnMarkersResponse(WriteTxnMarkersResponseData data) { + super(ApiKeys.WRITE_TXN_MARKERS); + this.data = data; + } + + @Override + public WriteTxnMarkersResponseData data() { + return data; + } + + public Map> errorsByProducerId() { + Map> errors = new HashMap<>(); for (WritableTxnMarkerResult marker : data.markers()) { Map topicPartitionErrorsMap = new HashMap<>(); - for (WritableTxnMarkerTopicResult topic: marker.topics()) { - for (WritableTxnMarkerPartitionResult partitionResult: topic.partitions()) { + for (WritableTxnMarkerTopicResult topic : marker.topics()) { + for (WritableTxnMarkerPartitionResult partitionResult : topic.partitions()) { topicPartitionErrorsMap.put(new TopicPartition(topic.name(), partitionResult.partitionIndex()), - Errors.forCode(partitionResult.errorCode())); + Errors.forCode(partitionResult.errorCode())); } } errors.put(marker.producerId(), topicPartitionErrorsMap); } + return errors; } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - - public Map errors(long producerId) { - return errors.get(producerId); + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; } @Override public Map errorCounts() { Map errorCounts = new HashMap<>(); - errors.values().forEach(allErrors -> - allErrors.values().forEach(error -> - updateErrorCounts(errorCounts, error) - ) - ); + for (WritableTxnMarkerResult marker : data.markers()) { + for (WritableTxnMarkerTopicResult topic : marker.topics()) { + for (WritableTxnMarkerPartitionResult partitionResult : topic.partitions()) + updateErrorCounts(errorCounts, Errors.forCode(partitionResult.errorCode())); + } + } return errorCounts; } public static WriteTxnMarkersResponse parse(ByteBuffer buffer, short version) { - return new WriteTxnMarkersResponse(ApiKeys.WRITE_TXN_MARKERS.parseResponse(version, buffer), version); + return new WriteTxnMarkersResponse(new WriteTxnMarkersResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java index 00a4bfc64cfbe..5694d853a590f 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java @@ -58,6 +58,7 @@ import javax.security.sasl.SaslClient; import javax.security.sasl.SaslException; import java.io.IOException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.nio.channels.SelectionKey; import java.security.Principal; @@ -432,13 +433,16 @@ private boolean sendSaslClientToken(byte[] serverToken, boolean isInitial) throw byte[] saslToken = createSaslToken(serverToken, isInitial); if (saslToken != null) { ByteBuffer tokenBuf = ByteBuffer.wrap(saslToken); - if (saslAuthenticateVersion != DISABLE_KAFKA_SASL_AUTHENTICATE_HEADER) { + Send send; + if (saslAuthenticateVersion == DISABLE_KAFKA_SASL_AUTHENTICATE_HEADER) { + send = new NetworkSend(node, tokenBuf); + } else { SaslAuthenticateRequestData data = new SaslAuthenticateRequestData() .setAuthBytes(tokenBuf.array()); SaslAuthenticateRequest request = new SaslAuthenticateRequest.Builder(data).build(saslAuthenticateVersion); - tokenBuf = request.serialize(nextRequestHeader(ApiKeys.SASL_AUTHENTICATE, saslAuthenticateVersion)); + send = request.toSend(node, nextRequestHeader(ApiKeys.SASL_AUTHENTICATE, saslAuthenticateVersion)); } - send(new NetworkSend(node, tokenBuf)); + send(send); return true; } } @@ -573,7 +577,7 @@ private AbstractResponse receiveKafkaResponse() throws IOException { currentRequestHeader = null; return response; } - } catch (SchemaException | IllegalArgumentException e) { + } catch (BufferUnderflowException | SchemaException | IllegalArgumentException e) { /* * Account for the fact that during re-authentication there may be responses * arriving for requests that were sent in the past. diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java index befbe4ec0a360..e6a2c5693bd83 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java @@ -170,7 +170,7 @@ public SaslServerAuthenticator(Map configs, List enabledMechanisms = (List) this.configs.get(BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_CONFIG); if (enabledMechanisms == null || enabledMechanisms.isEmpty()) throw new IllegalArgumentException("No SASL mechanisms are enabled"); - this.enabledMechanisms = new ArrayList(new HashSet(enabledMechanisms)); + this.enabledMechanisms = new ArrayList<>(new HashSet<>(enabledMechanisms)); for (String mechanism : this.enabledMechanisms) { if (!callbackHandlers.containsKey(mechanism)) throw new IllegalArgumentException("Callback handler not specified for SASL mechanism " + mechanism); @@ -576,8 +576,8 @@ private void handleApiVersionsRequest(RequestContext context, ApiVersionsRequest else if (!apiVersionsRequest.isValid()) sendKafkaResponse(context, apiVersionsRequest.getErrorResponse(0, Errors.INVALID_REQUEST.exception())); else { - metadataRegistry.registerClientInformation(new ClientInformation(apiVersionsRequest.data.clientSoftwareName(), - apiVersionsRequest.data.clientSoftwareVersion())); + metadataRegistry.registerClientInformation(new ClientInformation(apiVersionsRequest.data().clientSoftwareName(), + apiVersionsRequest.data().clientSoftwareVersion())); sendKafkaResponse(context, apiVersionsResponse()); setSaslState(SaslState.HANDSHAKE_REQUEST); } @@ -676,8 +676,7 @@ else if (connectionsMaxReauthMs == null) retvalSessionLifetimeMs = zeroIfNegative(credentialExpirationMs - authenticationEndMs); else retvalSessionLifetimeMs = zeroIfNegative( - Math.min(credentialExpirationMs - authenticationEndMs, - connectionsMaxReauthMs)); + Math.min(credentialExpirationMs - authenticationEndMs, connectionsMaxReauthMs)); if (retvalSessionLifetimeMs > 0L) sessionExpirationTimeNanos = authenticationEndNanos + 1000 * 1000 * retvalSessionLifetimeMs; } diff --git a/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java b/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java index ba965db500d52..58180008d668d 100644 --- a/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java @@ -30,7 +30,7 @@ import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopicCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.utils.LogContext; @@ -41,6 +41,7 @@ import org.junit.Test; import java.net.InetSocketAddress; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -68,7 +69,7 @@ public class MetadataTest { new ClusterResourceListeners()); private static MetadataResponse emptyMetadataResponse() { - return MetadataResponse.prepareResponse( + return TestUtils.metadataResponse( Collections.emptyList(), null, -1, @@ -196,8 +197,8 @@ public void testIgnoreLeaderEpochInOlderMetadataResponse() { .setBrokers(new MetadataResponseBrokerCollection()); for (short version = ApiKeys.METADATA.oldestVersion(); version < 9; version++) { - Struct struct = data.toStruct(version); - MetadataResponse response = new MetadataResponse(struct, version); + ByteBuffer buffer = MessageTestUtil.messageToByteBuffer(data, version); + MetadataResponse response = MetadataResponse.parse(buffer, version); assertFalse(response.hasReliableLeaderEpochs()); metadata.updateWithCurrentRequestVersion(response, false, 100); assertTrue(metadata.partitionMetadataIfCurrent(tp).isPresent()); @@ -206,8 +207,8 @@ public void testIgnoreLeaderEpochInOlderMetadataResponse() { } for (short version = 9; version <= ApiKeys.METADATA.latestVersion(); version++) { - Struct struct = data.toStruct(version); - MetadataResponse response = new MetadataResponse(struct, version); + ByteBuffer buffer = MessageTestUtil.messageToByteBuffer(data, version); + MetadataResponse response = MetadataResponse.parse(buffer, version); assertTrue(response.hasReliableLeaderEpochs()); metadata.updateWithCurrentRequestVersion(response, false, 100); assertTrue(metadata.partitionMetadataIfCurrent(tp).isPresent()); @@ -244,7 +245,7 @@ public void testStaleMetadata() { .setTopics(topics) .setBrokers(new MetadataResponseBrokerCollection()); - metadata.updateWithCurrentRequestVersion(new MetadataResponse(data), false, 100); + metadata.updateWithCurrentRequestVersion(new MetadataResponse(data, ApiKeys.METADATA.latestVersion()), false, 100); // Older epoch with changed ISR should be ignored partitionMetadata @@ -256,7 +257,7 @@ public void testStaleMetadata() { .setOfflineReplicas(Collections.emptyList()) .setErrorCode(Errors.NONE.code()); - metadata.updateWithCurrentRequestVersion(new MetadataResponse(data), false, 101); + metadata.updateWithCurrentRequestVersion(new MetadataResponse(data, ApiKeys.METADATA.latestVersion()), false, 101); assertEquals(Optional.of(10), metadata.lastSeenLeaderEpoch(tp)); assertTrue(metadata.partitionMetadataIfCurrent(tp).isPresent()); @@ -757,12 +758,14 @@ public void testLeaderMetadataInconsistentWithBrokerMetadata() { metadata.updateWithCurrentRequestVersion(new MetadataResponse(new MetadataResponseData() .setTopics(buildTopicCollection(tp.topic(), firstPartitionMetadata)) - .setBrokers(buildBrokerCollection(Arrays.asList(node0, node1, node2)))), + .setBrokers(buildBrokerCollection(Arrays.asList(node0, node1, node2))), + ApiKeys.METADATA.latestVersion()), false, 10L); metadata.updateWithCurrentRequestVersion(new MetadataResponse(new MetadataResponseData() .setTopics(buildTopicCollection(tp.topic(), secondPartitionMetadata)) - .setBrokers(buildBrokerCollection(Arrays.asList(node1, node2)))), + .setBrokers(buildBrokerCollection(Arrays.asList(node1, node2))), + ApiKeys.METADATA.latestVersion()), false, 20L); assertNull(metadata.fetch().leaderFor(tp)); diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java index 0615c9c8f3427..dd7ca94c45f08 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java @@ -30,13 +30,13 @@ import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; import org.apache.kafka.common.requests.RequestHeader; -import org.apache.kafka.common.requests.ResponseHeader; import org.apache.kafka.common.security.authenticator.SaslClientAuthenticator; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; @@ -185,9 +185,10 @@ public void testUnsupportedVersionDuringInternalMetadataRequest() { private void checkSimpleRequestResponse(NetworkClient networkClient) { awaitReady(networkClient, node); // has to be before creating any request, as it may send ApiVersionsRequest and its response is mocked with correlation id 0 + short requestVersion = PRODUCE.latestVersion(); ProduceRequest.Builder builder = new ProduceRequest.Builder( - PRODUCE.latestVersion(), - PRODUCE.latestVersion(), + requestVersion, + requestVersion, new ProduceRequestData() .setAcks((short) 1) .setTimeoutMs(1000)); @@ -197,18 +198,9 @@ private void checkSimpleRequestResponse(NetworkClient networkClient) { networkClient.send(request, time.milliseconds()); networkClient.poll(1, time.milliseconds()); assertEquals(1, networkClient.inFlightRequestCount()); - ResponseHeader respHeader = - new ResponseHeader(request.correlationId(), - request.apiKey().responseHeaderVersion(PRODUCE.latestVersion())); - Struct resp = new ProduceResponseData() - .setThrottleTimeMs(100) - .toStruct(PRODUCE.latestVersion()); - Struct responseHeaderStruct = respHeader.toStruct(); - int size = responseHeaderStruct.sizeOf() + resp.sizeOf(); - ByteBuffer buffer = ByteBuffer.allocate(size); - responseHeaderStruct.writeTo(buffer); - resp.writeTo(buffer); - buffer.flip(); + ProduceResponse produceResponse = new ProduceResponse(new ProduceResponseData()); + ByteBuffer buffer = produceResponse.serializeWithHeader(requestVersion, request.correlationId()); + buffer.rewind(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); List responses = networkClient.poll(1, time.milliseconds()); assertEquals(1, responses.size()); @@ -219,7 +211,7 @@ private void checkSimpleRequestResponse(NetworkClient networkClient) { } private void delayedApiVersionsResponse(int correlationId, short version, ApiVersionsResponse response) { - ByteBuffer buffer = response.serialize(ApiKeys.API_VERSIONS, version, correlationId); + ByteBuffer buffer = response.serializeWithHeader(version, correlationId); selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); } @@ -502,9 +494,10 @@ public void testConnectionSetupTimeout() { public void testConnectionThrottling() { // Instrument the test to return a response with a 100ms throttle delay. awaitReady(client, node); + short requestVersion = PRODUCE.latestVersion(); ProduceRequest.Builder builder = new ProduceRequest.Builder( - PRODUCE.latestVersion(), - PRODUCE.latestVersion(), + requestVersion, + requestVersion, new ProduceRequestData() .setAcks((short) 1) .setTimeoutMs(1000)); @@ -513,18 +506,10 @@ public void testConnectionThrottling() { defaultRequestTimeoutMs, handler); client.send(request, time.milliseconds()); client.poll(1, time.milliseconds()); - ResponseHeader respHeader = - new ResponseHeader(request.correlationId(), - request.apiKey().responseHeaderVersion(PRODUCE.latestVersion())); - Struct resp = new ProduceResponseData() - .setThrottleTimeMs(100) - .toStruct(PRODUCE.latestVersion()); - Struct responseHeaderStruct = respHeader.toStruct(); - int size = responseHeaderStruct.sizeOf() + resp.sizeOf(); - ByteBuffer buffer = ByteBuffer.allocate(size); - responseHeaderStruct.writeTo(buffer); - resp.writeTo(buffer); - buffer.flip(); + int throttleTime = 100; + ProduceResponse produceResponse = new ProduceResponse(new ProduceResponseData().setThrottleTimeMs(throttleTime)); + ByteBuffer buffer = produceResponse.serializeWithHeader(requestVersion, request.correlationId()); + buffer.rewind(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); @@ -603,23 +588,15 @@ private int sendEmptyProduceRequest(String nodeId) { return request.correlationId(); } - private void sendResponse(ResponseHeader respHeader, Struct response) { - Struct responseHeaderStruct = respHeader.toStruct(); - int size = responseHeaderStruct.sizeOf() + response.sizeOf(); - ByteBuffer buffer = ByteBuffer.allocate(size); - responseHeaderStruct.writeTo(buffer); - response.writeTo(buffer); - buffer.flip(); + private void sendResponse(AbstractResponse response, short version, int correlationId) { + ByteBuffer buffer = response.serializeWithHeader(version, correlationId); + buffer.rewind(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); } private void sendThrottledProduceResponse(int correlationId, int throttleMs, short version) { - Struct resp = new ProduceResponseData() - .setThrottleTimeMs(throttleMs) - .toStruct(version); - sendResponse(new ResponseHeader(correlationId, - PRODUCE.responseHeaderVersion(version)), - resp); + ProduceResponse response = new ProduceResponse(new ProduceResponseData().setThrottleTimeMs(throttleMs)); + sendResponse(response, version, correlationId); } @Test @@ -706,7 +683,7 @@ public void testAuthenticationFailureWithInFlightMetadataRequest() { RequestHeader header = parseHeader(requestBuffer); assertEquals(ApiKeys.METADATA, header.apiKey()); - ByteBuffer responseBuffer = metadataResponse.serialize(ApiKeys.METADATA, header.apiVersion(), header.correlationId()); + ByteBuffer responseBuffer = metadataResponse.serializeWithHeader(header.apiVersion(), header.correlationId()); selector.delayedReceive(new DelayedReceive(node1.idString(), new NetworkReceive(node1.idString(), responseBuffer))); int initialUpdateVersion = metadata.updateVersion(); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 1d2dd4ae23997..071120e94015b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -116,7 +116,7 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.quota.ClientQuotaAlteration; import org.apache.kafka.common.quota.ClientQuotaEntity; import org.apache.kafka.common.quota.ClientQuotaFilter; @@ -458,11 +458,12 @@ private static MetadataResponse prepareMetadataResponse(Cluster cluster, Errors .setPartitions(pms); metadata.add(tm); } - return MetadataResponse.prepareResponse(0, - metadata, + return MetadataResponse.prepareResponse(true, + 0, cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), + metadata, MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); } @@ -560,7 +561,7 @@ public void testConnectionFailureOnMetadataUpdate() throws Exception { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); env.kafkaClient().prepareResponse(request -> request instanceof MetadataRequest, null, true); env.kafkaClient().prepareResponse(request -> request instanceof MetadataRequest, - MetadataResponse.prepareResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), + TestUtils.metadataResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), 1, Collections.emptyList())); env.kafkaClient().prepareResponse(body -> body instanceof CreateTopicsRequest, prepareCreateTopicsResponse("myTopic", Errors.NONE)); @@ -585,7 +586,7 @@ public void testUnreachableBootstrapServer() throws Exception { Cluster discoveredCluster = mockCluster(3, 0); env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); env.kafkaClient().prepareResponse(body -> body instanceof MetadataRequest, - MetadataResponse.prepareResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), + TestUtils.metadataResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), 1, Collections.emptyList())); env.kafkaClient().prepareResponse(body -> body instanceof CreateTopicsRequest, prepareCreateTopicsResponse("myTopic", Errors.NONE)); @@ -702,7 +703,7 @@ public void testCreateTopicsHandleNotControllerException() throws Exception { env.kafkaClient().prepareResponseFrom( prepareCreateTopicsResponse("myTopic", Errors.NOT_CONTROLLER), env.cluster().nodeById(0)); - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(env.cluster().nodes(), + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 1, Collections.emptyList())); @@ -1046,7 +1047,7 @@ public void testMetadataRetries() throws Exception { env.kafkaClient().prepareResponse(null, true); // The next one succeeds and gives us the controller id - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(initializedCluster.nodes(), + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(initializedCluster.nodes(), initializedCluster.clusterResource().clusterId(), initializedCluster.controller().id(), Collections.emptyList())); @@ -1056,7 +1057,7 @@ public void testMetadataRetries() throws Exception { MetadataResponse.PartitionMetadata partitionMetadata = new MetadataResponse.PartitionMetadata( Errors.NONE, new TopicPartition(topic, 0), Optional.of(leader.id()), Optional.of(10), singletonList(leader.id()), singletonList(leader.id()), singletonList(leader.id())); - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(initializedCluster.nodes(), + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(initializedCluster.nodes(), initializedCluster.clusterResource().clusterId(), 1, singletonList(new MetadataResponse.TopicMetadata(Errors.NONE, topic, false, singletonList(partitionMetadata), MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED)))); @@ -1175,17 +1176,18 @@ public void testDescribeAcls() throws Exception { // Test a call where we get back ACL1 and ACL2. env.kafkaClient().prepareResponse(new DescribeAclsResponse(new DescribeAclsResponseData() - .setResources(DescribeAclsResponse.aclsResources(asList(ACL1, ACL2))))); + .setResources(DescribeAclsResponse.aclsResources(asList(ACL1, ACL2))), ApiKeys.DESCRIBE_ACLS.latestVersion())); assertCollectionIs(env.adminClient().describeAcls(FILTER1).values().get(), ACL1, ACL2); // Test a call where we get back no results. - env.kafkaClient().prepareResponse(new DescribeAclsResponse(new DescribeAclsResponseData())); + env.kafkaClient().prepareResponse(new DescribeAclsResponse(new DescribeAclsResponseData(), + ApiKeys.DESCRIBE_ACLS.latestVersion())); assertTrue(env.adminClient().describeAcls(FILTER2).values().get().isEmpty()); // Test a call where we get back an error. env.kafkaClient().prepareResponse(new DescribeAclsResponse(new DescribeAclsResponseData() .setErrorCode(Errors.SECURITY_DISABLED.code()) - .setErrorMessage("Security is disabled"))); + .setErrorMessage("Security is disabled"), ApiKeys.DESCRIBE_ACLS.latestVersion())); TestUtils.assertFutureError(env.adminClient().describeAcls(FILTER2).values(), SecurityDisabledException.class); // Test a call where we supply an invalid filter. @@ -1238,7 +1240,8 @@ public void testDeleteAcls() throws Exception { DeleteAclsResponse.matchingAcl(ACL2, ApiError.NONE))), new DeleteAclsResponseData.DeleteAclsFilterResult() .setErrorCode(Errors.SECURITY_DISABLED.code()) - .setErrorMessage("No security"))))); + .setErrorMessage("No security"))), + ApiKeys.DELETE_ACLS.latestVersion())); DeleteAclsResult results = env.adminClient().deleteAcls(asList(FILTER1, FILTER2)); Map> filterResults = results.values(); FilterResults filter1Results = filterResults.get(FILTER1).get(); @@ -1258,8 +1261,13 @@ public void testDeleteAcls() throws Exception { DeleteAclsResponse.matchingAcl(ACL1, ApiError.NONE), new DeleteAclsResponseData.DeleteAclsMatchingAcl() .setErrorCode(Errors.SECURITY_DISABLED.code()) - .setErrorMessage("No security"))), - new DeleteAclsResponseData.DeleteAclsFilterResult())))); + .setErrorMessage("No security") + .setPermissionType(AclPermissionType.ALLOW.code()) + .setOperation(AclOperation.ALTER.code()) + .setResourceType(ResourceType.CLUSTER.code()) + .setPatternType(FILTER2.patternFilter().patternType().code()))), + new DeleteAclsResponseData.DeleteAclsFilterResult())), + ApiKeys.DELETE_ACLS.latestVersion())); results = env.adminClient().deleteAcls(asList(FILTER1, FILTER2)); assertTrue(results.values().get(FILTER2).get().values().isEmpty()); TestUtils.assertFutureError(results.all(), SecurityDisabledException.class); @@ -1271,7 +1279,8 @@ public void testDeleteAcls() throws Exception { new DeleteAclsResponseData.DeleteAclsFilterResult() .setMatchingAcls(asList(DeleteAclsResponse.matchingAcl(ACL1, ApiError.NONE))), new DeleteAclsResponseData.DeleteAclsFilterResult() - .setMatchingAcls(asList(DeleteAclsResponse.matchingAcl(ACL2, ApiError.NONE))))))); + .setMatchingAcls(asList(DeleteAclsResponse.matchingAcl(ACL2, ApiError.NONE))))), + ApiKeys.DELETE_ACLS.latestVersion())); results = env.adminClient().deleteAcls(asList(FILTER1, FILTER2)); Collection deleted = results.all().get(); assertCollectionIs(deleted, ACL1, ACL2); @@ -1307,7 +1316,8 @@ public void testElectLeaders() throws Exception { electionResults.add(electionResult); - env.kafkaClient().prepareResponse(new ElectLeadersResponse(0, Errors.NONE.code(), electionResults)); + env.kafkaClient().prepareResponse(new ElectLeadersResponse(0, Errors.NONE.code(), + electionResults, ApiKeys.ELECT_LEADERS.latestVersion())); ElectLeadersResult results = env.adminClient().electLeaders( electionType, new HashSet<>(asList(topic1, topic2))); @@ -1320,7 +1330,8 @@ public void testElectLeaders() throws Exception { partition2Result.setErrorCode(ApiError.NONE.error().code()); partition2Result.setErrorMessage(ApiError.NONE.message()); - env.kafkaClient().prepareResponse(new ElectLeadersResponse(0, Errors.NONE.code(), electionResults)); + env.kafkaClient().prepareResponse(new ElectLeadersResponse(0, Errors.NONE.code(), electionResults, + ApiKeys.ELECT_LEADERS.latestVersion())); results = env.adminClient().electLeaders(electionType, new HashSet<>(asList(topic1, topic2))); assertFalse(results.partitions().get().get(topic1).isPresent()); assertFalse(results.partitions().get().get(topic2).isPresent()); @@ -1848,7 +1859,7 @@ public void testDeleteRecordsTopicAuthorizationError() { topics.add(new MetadataResponse.TopicMetadata(Errors.TOPIC_AUTHORIZATION_FAILED, topic, false, Collections.emptyList())); - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(env.cluster().nodes(), + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), env.cluster().controller().id(), topics)); Map recordsToDelete = new HashMap<>(); @@ -1881,7 +1892,7 @@ public void testDeleteRecordsMultipleSends() throws Exception { List topicMetadata = new ArrayList<>(); topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.NONE, topic, false, partitionMetadata)); - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(env.cluster().nodes(), + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), env.cluster().controller().id(), topicMetadata)); env.kafkaClient().prepareResponseFrom(new DeleteRecordsResponse(new DeleteRecordsResponseData().setTopics( @@ -1969,7 +1980,7 @@ public void testDeleteRecords() throws Exception { t.add(new MetadataResponse.TopicMetadata(Errors.NONE, "my_topic", false, p)); - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), t)); + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), t)); env.kafkaClient().prepareResponse(new DeleteRecordsResponse(m)); Map recordsToDelete = new HashMap<>(); @@ -2032,20 +2043,20 @@ public void testDescribeCluster() throws Exception { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); // Prepare the metadata response used for the first describe cluster - MetadataResponse response = MetadataResponse.prepareResponse(0, - Collections.emptyList(), + MetadataResponse response = TestUtils.metadataResponse(0, env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 2, + Collections.emptyList(), MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); env.kafkaClient().prepareResponse(response); // Prepare the metadata response used for the second describe cluster - MetadataResponse response2 = MetadataResponse.prepareResponse(0, - Collections.emptyList(), + MetadataResponse response2 = TestUtils.metadataResponse(0, env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 3, + Collections.emptyList(), 1 << AclOperation.DESCRIBE.code() | 1 << AclOperation.ALTER.code()); env.kafkaClient().prepareResponse(response2); @@ -2072,14 +2083,14 @@ public void testListConsumerGroups() throws Exception { // Empty metadata response should be retried env.kafkaClient().prepareResponse( - MetadataResponse.prepareResponse( + TestUtils.metadataResponse( Collections.emptyList(), env.cluster().clusterResource().clusterId(), -1, Collections.emptyList())); env.kafkaClient().prepareResponse( - MetadataResponse.prepareResponse( + TestUtils.metadataResponse( env.cluster().nodes(), env.cluster().clusterResource().clusterId(), env.cluster().controller().id(), @@ -2185,7 +2196,7 @@ public void testListConsumerGroupsMetadataFailure() throws Exception { // Empty metadata causes the request to fail since we have no list of brokers // to send the ListGroups requests to env.kafkaClient().prepareResponse( - MetadataResponse.prepareResponse( + TestUtils.metadataResponse( Collections.emptyList(), env.cluster().clusterResource().clusterId(), -1, @@ -3446,7 +3457,7 @@ public void testAlterPartitionReassignments() throws Exception { .setName("B") .setPartitions(Collections.singletonList(normalPartitionResponse))) ); - MetadataResponse controllerNodeResponse = MetadataResponse.prepareResponse(env.cluster().nodes(), + MetadataResponse controllerNodeResponse = TestUtils.metadataResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 1, Collections.emptyList()); AlterPartitionReassignmentsResponseData normalResponse = new AlterPartitionReassignmentsResponseData() @@ -3574,7 +3585,7 @@ public void testListPartitionReassignments() throws Exception { ListPartitionReassignmentsResponseData notControllerData = new ListPartitionReassignmentsResponseData() .setErrorCode(Errors.NOT_CONTROLLER.code()) .setErrorMessage(Errors.NOT_CONTROLLER.message()); - MetadataResponse controllerNodeResponse = MetadataResponse.prepareResponse(env.cluster().nodes(), + MetadataResponse controllerNodeResponse = TestUtils.metadataResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 1, Collections.emptyList()); ListPartitionReassignmentsResponseData reassignmentsData = new ListPartitionReassignmentsResponseData() .setTopics(Arrays.asList(tp1Reassignment, tp2Reassignment)); @@ -4031,7 +4042,7 @@ public void testUpdateFeaturesHandleNotControllerException() throws Exception { 0), env.cluster().nodeById(0)); final int controllerId = 1; - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(env.cluster().nodes(), + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), controllerId, Collections.emptyList())); @@ -4539,7 +4550,7 @@ public void testDescribeClientQuotas() throws Exception { responseData.put(entity1, Collections.singletonMap("consumer_byte_rate", 10000.0)); responseData.put(entity2, Collections.singletonMap("producer_byte_rate", 20000.0)); - env.kafkaClient().prepareResponse(new DescribeClientQuotasResponse(responseData, 0)); + env.kafkaClient().prepareResponse(DescribeClientQuotasResponse.fromQuotaEntities(responseData, 0)); ClientQuotaFilter filter = ClientQuotaFilter.contains(asList(ClientQuotaFilterComponent.ofEntity(ClientQuotaEntity.USER, value))); @@ -4593,7 +4604,7 @@ public void testAlterClientQuotas() throws Exception { responseData.put(unauthorizedEntity, new ApiError(Errors.CLUSTER_AUTHORIZATION_FAILED, "Authorization failed")); responseData.put(invalidEntity, new ApiError(Errors.INVALID_REQUEST, "Invalid quota entity")); - env.kafkaClient().prepareResponse(new AlterClientQuotasResponse(responseData, 0)); + env.kafkaClient().prepareResponse(AlterClientQuotasResponse.fromQuotaEntities(responseData, 0)); List entries = new ArrayList<>(3); entries.add(new ClientQuotaAlteration(goodEntity, Collections.singleton(new ClientQuotaAlteration.Op("consumer_byte_rate", 10000.0)))); @@ -4915,15 +4926,21 @@ public void testDescribeLogDirsPartialFailure() throws Exception { @Test public void testHasCoordinatorMoved() { Map errors = new HashMap<>(); - AbstractResponse response = new AbstractResponse() { + AbstractResponse response = new AbstractResponse(ApiKeys.OFFSET_COMMIT) { @Override public Map errorCounts() { return errors; } + @Override - protected Struct toStruct(short version) { + protected Message data() { return null; } + + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } }; assertFalse(ConsumerGroupOperationContext.hasCoordinatorMoved(response)); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index ab36ddf1a4b62..ad8fc03ac631d 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -2408,7 +2408,7 @@ public void testSubscriptionOnInvalidTopic() { List topicMetadata = new ArrayList<>(); topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.INVALID_TOPIC_EXCEPTION, invalidTopicName, false, Collections.emptyList())); - MetadataResponse updateResponse = MetadataResponse.prepareResponse(cluster.nodes(), + MetadataResponse updateResponse = TestUtils.metadataResponse(cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), topicMetadata); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index ae2ab8a79b303..902401a77445f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -1595,7 +1595,7 @@ private void testInternalTopicInclusion(boolean includeInternalTopics) { MetadataResponse.TopicMetadata topicMetadata = new MetadataResponse.TopicMetadata(Errors.NONE, Topic.GROUP_METADATA_TOPIC_NAME, true, singletonList(partitionMetadata)); - client.updateMetadata(MetadataResponse.prepareResponse(singletonList(node), "clusterId", node.id(), + client.updateMetadata(TestUtils.metadataResponse(singletonList(node), "clusterId", node.id(), singletonList(topicMetadata))); coordinator.maybeUpdateSubscriptionMetadata(); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java index 330f88e210ead..f71aa2f5956fe 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java @@ -75,7 +75,7 @@ private void testPatternSubscription(boolean includeInternalTopics) { topics.add(topicMetadata("__matching_topic", false)); topics.add(topicMetadata("non_matching_topic", false)); - MetadataResponse response = MetadataResponse.prepareResponse(singletonList(node), + MetadataResponse response = TestUtils.metadataResponse(singletonList(node), "clusterId", node.id(), topics); metadata.updateWithCurrentRequestVersion(response, false, time.milliseconds()); @@ -151,7 +151,7 @@ private void testBasicSubscription(Set expectedTopics, Set expec for (String expectedInternalTopic : expectedInternalTopics) topics.add(topicMetadata(expectedInternalTopic, true)); - MetadataResponse response = MetadataResponse.prepareResponse(singletonList(node), + MetadataResponse response = TestUtils.metadataResponse(singletonList(node), "clusterId", node.id(), topics); metadata.updateWithCurrentRequestVersion(response, false, time.milliseconds()); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 4d31390e6792c..0ee5c76e2b22b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -2031,7 +2031,7 @@ public void testGetTopicMetadataOfflinePartitions() { altTopics.add(alteredTopic); } Node controller = originalResponse.controller(); - MetadataResponse altered = MetadataResponse.prepareResponse( + MetadataResponse altered = TestUtils.metadataResponse( originalResponse.brokers(), originalResponse.clusterId(), controller != null ? controller.id() : MetadataResponse.NO_CONTROLLER_ID, @@ -2065,7 +2065,8 @@ public void testQuotaMetrics() { time, true, new ApiVersions(), throttleTimeSensor, new LogContext()); ByteBuffer buffer = ApiVersionsResponse.createApiVersionsResponse( - 400, RecordBatch.CURRENT_MAGIC_VALUE).serialize(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion(), 0); + 400, RecordBatch.CURRENT_MAGIC_VALUE).serializeWithHeader(ApiKeys.API_VERSIONS.latestVersion(), 0); + selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); while (!client.ready(node, time.milliseconds())) { client.poll(1, time.milliseconds()); @@ -2082,9 +2083,7 @@ public void testQuotaMetrics() { client.send(request, time.milliseconds()); client.poll(1, time.milliseconds()); FetchResponse response = fullFetchResponse(tp0, nextRecords, Errors.NONE, i, throttleTimeMs); - buffer = response.serialize(ApiKeys.FETCH, - ApiKeys.FETCH.latestVersion(), - request.correlationId()); + buffer = response.serializeWithHeader(ApiKeys.FETCH.latestVersion(), request.correlationId()); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); // If a throttled response is received, advance the time to ensure progress. @@ -4507,7 +4506,7 @@ private MetadataResponse newMetadataResponse(String topic, Errors error) { MetadataResponse.TopicMetadata topicMetadata = new MetadataResponse.TopicMetadata(error, topic, false, partitionsMetadata); List brokers = new ArrayList<>(initialUpdateResponse.brokers()); - return MetadataResponse.prepareResponse(brokers, initialUpdateResponse.clusterId(), + return TestUtils.metadataResponse(brokers, initialUpdateResponse.clusterId(), initialUpdateResponse.controller().id(), Collections.singletonList(topicMetadata)); } diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java index 8153c2f7b4acf..96901c5c26b38 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java @@ -1086,7 +1086,7 @@ public void testSendToInvalidTopic() throws Exception { List topicMetadata = new ArrayList<>(); topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.INVALID_TOPIC_EXCEPTION, invalidTopicName, false, Collections.emptyList())); - MetadataResponse updateResponse = MetadataResponse.prepareResponse( + MetadataResponse updateResponse = TestUtils.metadataResponse( new ArrayList<>(initialUpdateResponse.brokers()), initialUpdateResponse.clusterId(), initialUpdateResponse.controller().id(), diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index a0c8d5639aafc..6ded3f72de291 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -160,7 +160,7 @@ public void tearDown() { private static Map partitionRecords(ProduceRequest request) { Map partitionRecords = new HashMap<>(); - request.dataOrException().topicData().forEach(tpData -> tpData.partitionData().forEach(p -> { + request.data().topicData().forEach(tpData -> tpData.partitionData().forEach(p -> { TopicPartition tp = new TopicPartition(tpData.name(), p.index()); partitionRecords.put(tp, (MemoryRecords) p.records()); })); @@ -285,8 +285,9 @@ public void testQuotaMetrics() { 1000, 1000, 64 * 1024, 64 * 1024, 1000, 10 * 1000, 127 * 1000, ClientDnsLookup.USE_ALL_DNS_IPS, time, true, new ApiVersions(), throttleTimeSensor, logContext); - ByteBuffer buffer = ApiVersionsResponse.createApiVersionsResponse( - 400, RecordBatch.CURRENT_MAGIC_VALUE).serialize(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion(), 0); + ByteBuffer buffer = ApiVersionsResponse.createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE). + serializeWithHeader(ApiKeys.API_VERSIONS.latestVersion(), 0); + selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); while (!client.ready(node, time.milliseconds())) { client.poll(1, time.milliseconds()); @@ -305,8 +306,7 @@ public void testQuotaMetrics() { client.send(request, time.milliseconds()); client.poll(1, time.milliseconds()); ProduceResponse response = produceResponse(tp0, i, Errors.NONE, throttleTimeMs); - buffer = response. - serialize(ApiKeys.PRODUCE, ApiKeys.PRODUCE.latestVersion(), request.correlationId()); + buffer = response.serializeWithHeader(ApiKeys.PRODUCE.latestVersion(), request.correlationId()); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); // If a throttled response is received, advance the time to ensure progress. diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java index cd71aabb3dfd0..70b52c5df05c2 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java @@ -3356,7 +3356,7 @@ private void prepareProduceResponse(Errors error, final long producerId, final s private MockClient.RequestMatcher produceRequestMatcher(final long producerId, final short epoch, TopicPartition tp) { return body -> { ProduceRequest produceRequest = (ProduceRequest) body; - MemoryRecords records = produceRequest.dataOrException().topicData() + MemoryRecords records = produceRequest.data().topicData() .stream() .filter(t -> t.name().equals(tp.topic())) .findAny() diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java b/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java index 945246d9b7538..a093148b891aa 100644 --- a/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java +++ b/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java @@ -20,16 +20,12 @@ import java.nio.ByteBuffer; public final class MessageTestUtil { - public static int messageSize(Message message, short version) { - return message.size(new ObjectSerializationCache(), version); - } - public static ByteBuffer messageToByteBuffer(Message message, short version) { ObjectSerializationCache cache = new ObjectSerializationCache(); int size = message.size(cache, version); ByteBuffer bytes = ByteBuffer.allocate(size); message.write(new ByteBufferAccessor(bytes), cache, version); - bytes.flip(); + bytes.rewind(); return bytes; } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java index 7d49fb5509f04..92c38e557b93d 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java @@ -67,7 +67,7 @@ public void testConstructorWithErrorResponse() { } @Test - public void testConstructorWithStruct() { + public void testParse() { AddPartitionsToTxnTopicResultCollection topicCollection = new AddPartitionsToTxnTopicResultCollection(); @@ -87,14 +87,13 @@ public void testConstructorWithStruct() { AddPartitionsToTxnResponseData data = new AddPartitionsToTxnResponseData() .setResults(topicCollection) .setThrottleTimeMs(throttleTimeMs); + AddPartitionsToTxnResponse response = new AddPartitionsToTxnResponse(data); for (short version = 0; version <= ApiKeys.ADD_PARTITIONS_TO_TXN.latestVersion(); version++) { - AddPartitionsToTxnResponse response = new AddPartitionsToTxnResponse(data.toStruct(version), version); - assertEquals(expectedErrorCounts, response.errorCounts()); - - assertEquals(throttleTimeMs, response.throttleTimeMs()); - - assertEquals(version >= 1, response.shouldClientThrottle(version)); + AddPartitionsToTxnResponse parsedResponse = AddPartitionsToTxnResponse.parse(response.serializeBody(version), version); + assertEquals(expectedErrorCounts, parsedResponse.errorCounts()); + assertEquals(throttleTimeMs, parsedResponse.throttleTimeMs()); + assertEquals(version >= 1, parsedResponse.shouldClientThrottle(version)); } } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/CreateAclsRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/CreateAclsRequestTest.java index 06b4b9f4bb2b2..02a24ba54eaba 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/CreateAclsRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/CreateAclsRequestTest.java @@ -23,12 +23,12 @@ import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.CreateAclsRequestData; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -53,30 +53,30 @@ public class CreateAclsRequestTest { @Test(expected = UnsupportedVersionException.class) public void shouldThrowOnV0IfNotLiteral() { - new CreateAclsRequest(V0, data(PREFIXED_ACL1)); + new CreateAclsRequest(data(PREFIXED_ACL1), V0); } @Test(expected = IllegalArgumentException.class) public void shouldThrowOnIfUnknown() { - new CreateAclsRequest(V0, data(UNKNOWN_ACL1)); + new CreateAclsRequest(data(UNKNOWN_ACL1), V0); } @Test public void shouldRoundTripV0() { - final CreateAclsRequest original = new CreateAclsRequest(V0, data(LITERAL_ACL1, LITERAL_ACL2)); - final Struct struct = original.toStruct(); + final CreateAclsRequest original = new CreateAclsRequest(data(LITERAL_ACL1, LITERAL_ACL2), V0); + final ByteBuffer buffer = original.serializeBody(); - final CreateAclsRequest result = new CreateAclsRequest(struct, V0); + final CreateAclsRequest result = CreateAclsRequest.parse(buffer, V0); assertRequestEquals(original, result); } @Test public void shouldRoundTripV1() { - final CreateAclsRequest original = new CreateAclsRequest(V1, data(LITERAL_ACL1, PREFIXED_ACL1)); - final Struct struct = original.toStruct(); + final CreateAclsRequest original = new CreateAclsRequest(data(LITERAL_ACL1, PREFIXED_ACL1), V1); + final ByteBuffer buffer = original.serializeBody(); - final CreateAclsRequest result = new CreateAclsRequest(struct, V1); + final CreateAclsRequest result = CreateAclsRequest.parse(buffer, V1); assertRequestEquals(original, result); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsRequestTest.java index 19b65ed088e1c..9d2e741fbe0bd 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsRequestTest.java @@ -23,12 +23,12 @@ import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.DeleteAclsRequestData; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.stream.Collectors; import static java.util.Arrays.asList; @@ -64,9 +64,9 @@ public void shouldThrowOnUnknownElements() { @Test public void shouldRoundTripLiteralV0() { final DeleteAclsRequest original = new DeleteAclsRequest.Builder(requestData(LITERAL_FILTER)).build(V0); - final Struct struct = original.toStruct(); + final ByteBuffer buffer = original.serializeBody(); - final DeleteAclsRequest result = new DeleteAclsRequest(struct, V0); + final DeleteAclsRequest result = DeleteAclsRequest.parse(buffer, V0); assertRequestEquals(original, result); } @@ -82,7 +82,7 @@ public void shouldRoundTripAnyV0AsLiteral() { ANY_FILTER.entryFilter())) ).build(V0); - final DeleteAclsRequest result = new DeleteAclsRequest(original.toStruct(), V0); + final DeleteAclsRequest result = DeleteAclsRequest.parse(original.serializeBody(), V0); assertRequestEquals(expected, result); } @@ -92,9 +92,9 @@ public void shouldRoundTripV1() { final DeleteAclsRequest original = new DeleteAclsRequest.Builder( requestData(LITERAL_FILTER, PREFIXED_FILTER, ANY_FILTER) ).build(V1); - final Struct struct = original.toStruct(); + final ByteBuffer buffer = original.serializeBody(); - final DeleteAclsRequest result = new DeleteAclsRequest(struct, V1); + final DeleteAclsRequest result = DeleteAclsRequest.parse(buffer, V1); assertRequestEquals(original, result); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java index aaebc0c91a005..96f04ebf29e61 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java @@ -23,14 +23,16 @@ import org.apache.kafka.common.message.DeleteAclsResponseData; import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsFilterResult; import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsMatchingAcl; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourceType; import org.junit.Test; +import java.nio.ByteBuffer; + import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; public class DeleteAclsResponseTest { private static final short V0 = 0; @@ -81,41 +83,47 @@ public class DeleteAclsResponseTest { private static final DeleteAclsFilterResult UNKNOWN_RESPONSE = new DeleteAclsFilterResult().setMatchingAcls(asList( UNKNOWN_ACL)); - @Test(expected = UnsupportedVersionException.class) + @Test public void shouldThrowOnV0IfNotLiteral() { - new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(singletonList(PREFIXED_RESPONSE)) - ).toStruct(V0); + assertThrows(UnsupportedVersionException.class, () -> new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(singletonList(PREFIXED_RESPONSE)), + V0)); } - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowOnIfUnknown() { - new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(singletonList(UNKNOWN_RESPONSE)) - ).toStruct(V1); + assertThrows(IllegalArgumentException.class, () -> new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(singletonList(UNKNOWN_RESPONSE)), + V1)); } @Test public void shouldRoundTripV0() { - final DeleteAclsResponse original = new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(singletonList(LITERAL_RESPONSE))); - final Struct struct = original.toStruct(V0); - - final DeleteAclsResponse result = new DeleteAclsResponse(struct, V0); + final DeleteAclsResponse original = new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(singletonList(LITERAL_RESPONSE)), + V0); + final ByteBuffer buffer = original.serializeBody(V0); + + final DeleteAclsResponse result = DeleteAclsResponse.parse(buffer, V0); assertEquals(original.filterResults(), result.filterResults()); } @Test public void shouldRoundTripV1() { - final DeleteAclsResponse original = new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(asList(LITERAL_RESPONSE, PREFIXED_RESPONSE))); - final Struct struct = original.toStruct(V1); - - final DeleteAclsResponse result = new DeleteAclsResponse(struct, V1); + final DeleteAclsResponse original = new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(asList(LITERAL_RESPONSE, PREFIXED_RESPONSE)), + V1); + final ByteBuffer buffer = original.serializeBody(V1); + + final DeleteAclsResponse result = DeleteAclsResponse.parse(buffer, V1); assertEquals(original.filterResults(), result.filterResults()); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsRequestTest.java index e9203aebe2e64..e346ba6c395de 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsRequestTest.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.errors.UnsupportedVersionException; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; @@ -60,9 +59,7 @@ public void shouldThrowIfUnknown() { @Test public void shouldRoundTripLiteralV0() { final DescribeAclsRequest original = new DescribeAclsRequest.Builder(LITERAL_FILTER).build(V0); - final Struct struct = original.toStruct(); - - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V0); + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V0); assertRequestEquals(original, result); } @@ -77,39 +74,28 @@ public void shouldRoundTripAnyV0AsLiteral() { PatternType.LITERAL), ANY_FILTER.entryFilter())).build(V0); - final Struct struct = original.toStruct(); - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V0); - + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V0); assertRequestEquals(expected, result); } @Test public void shouldRoundTripLiteralV1() { final DescribeAclsRequest original = new DescribeAclsRequest.Builder(LITERAL_FILTER).build(V1); - final Struct struct = original.toStruct(); - - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V1); - + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V1); assertRequestEquals(original, result); } @Test public void shouldRoundTripPrefixedV1() { final DescribeAclsRequest original = new DescribeAclsRequest.Builder(PREFIXED_FILTER).build(V1); - final Struct struct = original.toStruct(); - - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V1); - + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V1); assertRequestEquals(original, result); } @Test public void shouldRoundTripAnyV1() { final DescribeAclsRequest original = new DescribeAclsRequest.Builder(ANY_FILTER).build(V1); - final Struct struct = original.toStruct(); - - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V1); - + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V1); assertRequestEquals(original, result); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsResponseTest.java index 84500650d6cf6..56bfe8557ddfe 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsResponseTest.java @@ -26,12 +26,12 @@ import org.apache.kafka.common.message.DescribeAclsResponseData.AclDescription; import org.apache.kafka.common.message.DescribeAclsResponseData.DescribeAclsResource; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -82,21 +82,21 @@ public class DescribeAclsResponseTest { @Test(expected = UnsupportedVersionException.class) public void shouldThrowOnV0IfNotLiteral() { - buildResponse(10, Errors.NONE, Collections.singletonList(PREFIXED_ACL1)).toStruct(V0); + buildResponse(10, Errors.NONE, Collections.singletonList(PREFIXED_ACL1)).serializeBody(V0); } @Test(expected = IllegalArgumentException.class) public void shouldThrowIfUnknown() { - buildResponse(10, Errors.NONE, Collections.singletonList(UNKNOWN_ACL)).toStruct(V0); + buildResponse(10, Errors.NONE, Collections.singletonList(UNKNOWN_ACL)).serializeBody(V0); } @Test public void shouldRoundTripV0() { List resources = Arrays.asList(LITERAL_ACL1, LITERAL_ACL2); final DescribeAclsResponse original = buildResponse(10, Errors.NONE, resources); - final Struct struct = original.toStruct(V0); + final ByteBuffer buffer = original.serializeBody(V0); - final DescribeAclsResponse result = new DescribeAclsResponse(struct, V0); + final DescribeAclsResponse result = DescribeAclsResponse.parse(buffer, V0); assertResponseEquals(original, result); final DescribeAclsResponse result2 = buildResponse(10, Errors.NONE, DescribeAclsResponse.aclsResources( @@ -108,9 +108,9 @@ public void shouldRoundTripV0() { public void shouldRoundTripV1() { List resources = Arrays.asList(LITERAL_ACL1, PREFIXED_ACL1); final DescribeAclsResponse original = buildResponse(100, Errors.NONE, resources); - final Struct struct = original.toStruct(V1); + final ByteBuffer buffer = original.serializeBody(V1); - final DescribeAclsResponse result = new DescribeAclsResponse(struct, V1); + final DescribeAclsResponse result = DescribeAclsResponse.parse(buffer, V1); assertResponseEquals(original, result); final DescribeAclsResponse result2 = buildResponse(100, Errors.NONE, DescribeAclsResponse.aclsResources( diff --git a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java index f7b5850916c23..c1d4367c929e0 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java @@ -29,21 +29,24 @@ public class EndTxnResponseTest { @Test - public void testConstructorWithStruct() { + public void testConstructor() { int throttleTimeMs = 10; EndTxnResponseData data = new EndTxnResponseData() - .setErrorCode(Errors.NOT_COORDINATOR.code()) - .setThrottleTimeMs(throttleTimeMs); + .setErrorCode(Errors.NOT_COORDINATOR.code()) + .setThrottleTimeMs(throttleTimeMs); Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1); for (short version = 0; version <= ApiKeys.END_TXN.latestVersion(); version++) { - EndTxnResponse response = new EndTxnResponse(data.toStruct(version), version); + EndTxnResponse response = new EndTxnResponse(data); assertEquals(expectedErrorCounts, response.errorCounts()); - assertEquals(throttleTimeMs, response.throttleTimeMs()); + assertEquals(version >= 1, response.shouldClientThrottle(version)); + response = EndTxnResponse.parse(response.serializeBody(version), version); + assertEquals(expectedErrorCounts, response.errorCounts()); + assertEquals(throttleTimeMs, response.throttleTimeMs()); assertEquals(version >= 1, response.shouldClientThrottle(version)); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java index 2235a8f643cf8..a8c0bd5e778cb 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java @@ -25,7 +25,6 @@ import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.MessageTestUtil; import org.apache.kafka.test.TestUtils; import org.junit.Test; @@ -129,7 +128,7 @@ public void testVersionLogic() { assertEquals(2, request.controllerEpoch()); assertEquals(3, request.brokerEpoch()); - ByteBuffer byteBuffer = MessageTestUtil.messageToByteBuffer(request.data(), request.version()); + ByteBuffer byteBuffer = request.serializeBody(); LeaderAndIsrRequest deserializedRequest = new LeaderAndIsrRequest(new LeaderAndIsrRequestData( new ByteBufferAccessor(byteBuffer), version), version); @@ -163,10 +162,7 @@ public void testTopicPartitionGroupingSizeReduction() { LeaderAndIsrRequest v2 = builder.build((short) 2); LeaderAndIsrRequest v1 = builder.build((short) 1); - int size2 = MessageTestUtil.messageSize(v2.data(), v2.version()); - int size1 = MessageTestUtil.messageSize(v1.data(), v1.version()); - - assertTrue("Expected v2 < v1: v2=" + size2 + ", v1=" + size1, size2 < size1); + assertTrue("Expected v2 < v1: v2=" + v2.sizeInBytes() + ", v1=" + v1.sizeInBytes(), v2.sizeInBytes() < v1.sizeInBytes()); } private Set iterableToSet(Iterable iterable) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java index a620ed0d08230..99c863627b9f2 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java @@ -20,9 +20,11 @@ import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.junit.Before; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -58,29 +60,6 @@ public void setUp() { ); } - @Test - public void testConstructorWithStruct() { - Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1); - - LeaveGroupResponseData responseData = new LeaveGroupResponseData() - .setErrorCode(Errors.NOT_COORDINATOR.code()) - .setThrottleTimeMs(throttleTimeMs); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { - LeaveGroupResponse leaveGroupResponse = new LeaveGroupResponse(responseData.toStruct(version), version); - - assertEquals(expectedErrorCounts, leaveGroupResponse.errorCounts()); - - if (version >= 1) { - assertEquals(throttleTimeMs, leaveGroupResponse.throttleTimeMs()); - } else { - assertEquals(DEFAULT_THROTTLE_TIME, leaveGroupResponse.throttleTimeMs()); - } - - assertEquals(Errors.NOT_COORDINATOR, leaveGroupResponse.error()); - } - } - - @Test public void testConstructorWithMemberResponses() { Map expectedErrorCounts = new HashMap<>(); @@ -126,19 +105,42 @@ public void testShouldThrottle() { } @Test - public void testEqualityWithStruct() { + public void testEqualityWithSerialization() { LeaveGroupResponseData responseData = new LeaveGroupResponseData() - .setErrorCode(Errors.NONE.code()) - .setThrottleTimeMs(throttleTimeMs); + .setErrorCode(Errors.NONE.code()) + .setThrottleTimeMs(throttleTimeMs); for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { - LeaveGroupResponse primaryResponse = new LeaveGroupResponse(responseData.toStruct(version), version); - - LeaveGroupResponse secondaryResponse = new LeaveGroupResponse(responseData.toStruct(version), version); + LeaveGroupResponse primaryResponse = LeaveGroupResponse.parse( + MessageTestUtil.messageToByteBuffer(responseData, version), version); + LeaveGroupResponse secondaryResponse = LeaveGroupResponse.parse( + MessageTestUtil.messageToByteBuffer(responseData, version), version); assertEquals(primaryResponse, primaryResponse); assertEquals(primaryResponse, secondaryResponse); assertEquals(primaryResponse.hashCode(), secondaryResponse.hashCode()); + } + } + + @Test + public void testParse() { + Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1); + LeaveGroupResponseData data = new LeaveGroupResponseData() + .setErrorCode(Errors.NOT_COORDINATOR.code()) + .setThrottleTimeMs(throttleTimeMs); + + for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + ByteBuffer buffer = MessageTestUtil.messageToByteBuffer(data, version); + LeaveGroupResponse leaveGroupResponse = LeaveGroupResponse.parse(buffer, version); + assertEquals(expectedErrorCounts, leaveGroupResponse.errorCounts()); + + if (version >= 1) { + assertEquals(throttleTimeMs, leaveGroupResponse.throttleTimeMs()); + } else { + assertEquals(DEFAULT_THROTTLE_TIME, leaveGroupResponse.throttleTimeMs()); + } + + assertEquals(Errors.NOT_COORDINATOR, leaveGroupResponse.error()); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ListOffsetRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ListOffsetRequestTest.java index 9d7c2aa7cfd42..22e7c5736286e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ListOffsetRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ListOffsetRequestTest.java @@ -35,6 +35,7 @@ import org.apache.kafka.common.message.ListOffsetResponseData.ListOffsetTopicResponse; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.junit.Test; public class ListOffsetRequestTest { @@ -52,7 +53,7 @@ public void testDuplicatePartitions() { ListOffsetRequestData data = new ListOffsetRequestData() .setTopics(topics) .setReplicaId(-1); - ListOffsetRequest request = new ListOffsetRequest(data.toStruct((short) 0), (short) 0); + ListOffsetRequest request = ListOffsetRequest.parse(MessageTestUtil.messageToByteBuffer(data, (short) 0), (short) 0); assertEquals(Collections.singleton(new TopicPartition("topic", 0)), request.duplicatePartitions()); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java index 62229168b2405..5e3a85739e50d 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java @@ -22,9 +22,11 @@ import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.junit.Before; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -69,22 +71,23 @@ public void testConstructorWithErrorResponse() { } @Test - public void testConstructorWithStruct() { + public void testParse() { OffsetCommitResponseData data = new OffsetCommitResponseData() - .setTopics(Arrays.asList( - new OffsetCommitResponseTopic().setPartitions( - Collections.singletonList(new OffsetCommitResponsePartition() - .setPartitionIndex(partitionOne) - .setErrorCode(errorOne.code()))), - new OffsetCommitResponseTopic().setPartitions( - Collections.singletonList(new OffsetCommitResponsePartition() - .setPartitionIndex(partitionTwo) - .setErrorCode(errorTwo.code()))) - )) - .setThrottleTimeMs(throttleTimeMs); + .setTopics(Arrays.asList( + new OffsetCommitResponseTopic().setPartitions( + Collections.singletonList(new OffsetCommitResponsePartition() + .setPartitionIndex(partitionOne) + .setErrorCode(errorOne.code()))), + new OffsetCommitResponseTopic().setPartitions( + Collections.singletonList(new OffsetCommitResponsePartition() + .setPartitionIndex(partitionTwo) + .setErrorCode(errorTwo.code()))) + )) + .setThrottleTimeMs(throttleTimeMs); for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { - OffsetCommitResponse response = new OffsetCommitResponse(data.toStruct(version), version); + ByteBuffer buffer = MessageTestUtil.messageToByteBuffer(data, version); + OffsetCommitResponse response = OffsetCommitResponse.parse(buffer, version); assertEquals(expectedErrorCounts, response.errorCounts()); if (version >= 3) { @@ -96,4 +99,5 @@ public void testConstructorWithStruct() { assertEquals(version >= 4, response.shouldClientThrottle(version)); } } + } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java index 2bfa8b594525d..f20a74412880a 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java @@ -108,7 +108,7 @@ public void testStructBuild() { for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { Struct struct = latestResponse.data.toStruct(version); - OffsetFetchResponse oldResponse = new OffsetFetchResponse(struct, version); + OffsetFetchResponse oldResponse = OffsetFetchResponse.parse(latestResponse.serializeBody(version), version); if (version <= 1) { assertFalse(struct.hasField(ERROR_CODE)); @@ -148,9 +148,7 @@ public void testStructBuild() { Map responseData = oldResponse.responseData(); assertEquals(expectedDataMap, responseData); - responseData.forEach( - (tp, data) -> assertTrue(data.hasError()) - ); + responseData.forEach((tp, data) -> assertTrue(data.hasError())); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java index 37460cc3a5bae..f11a4837747ce 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java @@ -49,8 +49,7 @@ public void testDefaultReplicaId() { OffsetsForLeaderEpochRequest.Builder builder = OffsetsForLeaderEpochRequest.Builder.forFollower( version, Collections.emptyMap(), replicaId); OffsetsForLeaderEpochRequest request = builder.build(); - OffsetsForLeaderEpochRequest parsed = (OffsetsForLeaderEpochRequest) AbstractRequest.parseRequest( - ApiKeys.OFFSET_FOR_LEADER_EPOCH, version, request.toStruct()); + OffsetsForLeaderEpochRequest parsed = OffsetsForLeaderEpochRequest.parse(request.serializeBody(), version); if (version < 3) assertEquals(OffsetsForLeaderEpochRequest.DEBUGGING_REPLICA_ID, parsed.replicaId()); else @@ -58,4 +57,4 @@ public void testDefaultReplicaId() { } } -} \ No newline at end of file +} diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java index 156af86b53756..7bf53eceb3038 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java @@ -299,7 +299,7 @@ private void assertThrowsInvalidRecordExceptionForAllVersions(ProduceRequest.Bui private void assertThrowsInvalidRecordException(ProduceRequest.Builder builder, short version) { try { - builder.build(version).toStruct(); + builder.build(version).serializeBody(); fail("Builder did not raise " + InvalidRecordException.class.getName() + " as expected"); } catch (RuntimeException e) { assertTrue("Unexpected exception type " + e.getClass().getName(), diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java index 37cac84db0c15..9f3f1e877e33a 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java @@ -18,10 +18,8 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.message.ProduceResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import org.junit.Test; @@ -48,15 +46,12 @@ public void produceResponseV5Test() { ProduceResponse v5Response = new ProduceResponse(responseData, 10); short version = 5; - ByteBuffer buffer = v5Response.serialize(ApiKeys.PRODUCE, version, 0); + ByteBuffer buffer = v5Response.serializeWithHeader(version, 0); buffer.rewind(); ResponseHeader.parse(buffer, ApiKeys.PRODUCE.responseHeaderVersion(version)); // throw away. - - Struct deserializedStruct = ApiKeys.PRODUCE.parseResponse(version, buffer); - ProduceResponse v5FromBytes = (ProduceResponse) AbstractResponse.parseResponse(ApiKeys.PRODUCE, - deserializedStruct, version); + buffer, version); assertEquals(1, v5FromBytes.responses().size()); assertTrue(v5FromBytes.responses().containsKey(tp0)); @@ -79,12 +74,6 @@ public void produceResponseVersionTest() { assertEquals("Throttle time must be zero", 0, v0Response.throttleTimeMs()); assertEquals("Throttle time must be 10", 10, v1Response.throttleTimeMs()); assertEquals("Throttle time must be 10", 10, v2Response.throttleTimeMs()); - assertEquals("Should use schema version 0", ApiKeys.PRODUCE.responseSchema((short) 0), - v0Response.toStruct((short) 0).schema()); - assertEquals("Should use schema version 1", ApiKeys.PRODUCE.responseSchema((short) 1), - v1Response.toStruct((short) 1).schema()); - assertEquals("Should use schema version 2", ApiKeys.PRODUCE.responseSchema((short) 2), - v2Response.toStruct((short) 2).schema()); assertEquals("Response data does not match", responseData, v0Response.responses()); assertEquals("Response data does not match", responseData, v1Response.responses()); assertEquals("Response data does not match", responseData, v2Response.responses()); @@ -103,9 +92,7 @@ public void produceResponseRecordErrorsTest() { for (short ver = 0; ver <= PRODUCE.latestVersion(); ver++) { ProduceResponse response = new ProduceResponse(responseData); - Struct struct = response.toStruct(ver); - assertEquals("Should use schema version " + ver, ApiKeys.PRODUCE.responseSchema(ver), struct.schema()); - ProduceResponse.PartitionResponse deserialized = new ProduceResponse(new ProduceResponseData(struct, ver)).responses().get(tp); + ProduceResponse.PartitionResponse deserialized = ProduceResponse.parse(response.serializeBody(ver), ver).responses().get(tp); if (ver >= 8) { assertEquals(1, deserialized.recordErrors.size()); assertEquals(3, deserialized.recordErrors.get(0).batchIndex); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java index 47d49e618e9ab..1f2b567639832 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java @@ -24,7 +24,6 @@ import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.junit.Test; @@ -73,9 +72,8 @@ public void testSerdeUnsupportedApiVersionRequest() throws Exception { ApiKeys.API_VERSIONS.responseHeaderVersion(header.apiVersion())); assertEquals(correlationId, responseHeader.correlationId()); - Struct struct = ApiKeys.API_VERSIONS.parseResponse((short) 0, responseBuffer); - ApiVersionsResponse response = (ApiVersionsResponse) - AbstractResponse.parseResponse(ApiKeys.API_VERSIONS, struct, (short) 0); + ApiVersionsResponse response = (ApiVersionsResponse) AbstractResponse.parseResponse(ApiKeys.API_VERSIONS, + responseBuffer, (short) 0); assertEquals(Errors.UNSUPPORTED_VERSION.code(), response.data.errorCode()); assertTrue(response.data.apiKeys().isEmpty()); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java index a3fa922359b4d..8cf5fb2c0b72b 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java @@ -17,12 +17,12 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ObjectSerializationCache; +import org.apache.kafka.test.TestUtils; import org.junit.Test; import java.nio.ByteBuffer; -import static org.apache.kafka.test.TestUtils.toBuffer; import static org.junit.Assert.assertEquals; public class RequestHeaderTest { @@ -30,11 +30,11 @@ public class RequestHeaderTest { @Test public void testSerdeControlledShutdownV0() { // Verify that version 0 of controlled shutdown does not include the clientId field - + short apiVersion = 0; int correlationId = 2342; ByteBuffer rawBuffer = ByteBuffer.allocate(32); rawBuffer.putShort(ApiKeys.CONTROLLED_SHUTDOWN.id); - rawBuffer.putShort((short) 0); + rawBuffer.putShort(apiVersion); rawBuffer.putInt(correlationId); rawBuffer.flip(); @@ -45,8 +45,7 @@ public void testSerdeControlledShutdownV0() { assertEquals("", deserialized.clientId()); assertEquals(0, deserialized.headerVersion()); - Struct serialized = deserialized.toStruct(); - ByteBuffer serializedBuffer = toBuffer(serialized); + ByteBuffer serializedBuffer = TestUtils.serializeRequestHeader(deserialized); assertEquals(ApiKeys.CONTROLLED_SHUTDOWN.id, serializedBuffer.getShort(0)); assertEquals(0, serializedBuffer.getShort(2)); @@ -56,9 +55,11 @@ public void testSerdeControlledShutdownV0() { @Test public void testRequestHeaderV1() { - RequestHeader header = new RequestHeader(ApiKeys.FIND_COORDINATOR, (short) 1, "", 10); + short apiVersion = 1; + RequestHeader header = new RequestHeader(ApiKeys.FIND_COORDINATOR, apiVersion, "", 10); assertEquals(1, header.headerVersion()); - ByteBuffer buffer = toBuffer(header.toStruct()); + + ByteBuffer buffer = TestUtils.serializeRequestHeader(header); assertEquals(10, buffer.remaining()); RequestHeader deserialized = RequestHeader.parse(buffer); assertEquals(header, deserialized); @@ -66,9 +67,11 @@ public void testRequestHeaderV1() { @Test public void testRequestHeaderV2() { - RequestHeader header = new RequestHeader(ApiKeys.CREATE_DELEGATION_TOKEN, (short) 2, "", 10); + short apiVersion = 2; + RequestHeader header = new RequestHeader(ApiKeys.CREATE_DELEGATION_TOKEN, apiVersion, "", 10); assertEquals(2, header.headerVersion()); - ByteBuffer buffer = toBuffer(header.toStruct()); + + ByteBuffer buffer = TestUtils.serializeRequestHeader(header); assertEquals(11, buffer.remaining()); RequestHeader deserialized = RequestHeader.parse(buffer); assertEquals(header, deserialized); @@ -80,7 +83,10 @@ public void parseHeaderFromBufferWithNonZeroPosition() { buffer.position(10); RequestHeader header = new RequestHeader(ApiKeys.FIND_COORDINATOR, (short) 1, "", 10); - header.toStruct().writeTo(buffer); + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + // size must be called before write to avoid an NPE with the current implementation + header.size(serializationCache); + header.write(buffer, serializationCache); int limit = buffer.position(); buffer.position(10); buffer.limit(limit); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 40d9b32876c3c..b62376c8881d6 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -35,6 +35,7 @@ import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.AddOffsetsToTxnRequestData; import org.apache.kafka.common.message.AddOffsetsToTxnResponseData; +import org.apache.kafka.common.message.AlterClientQuotasResponseData; import org.apache.kafka.common.message.AlterConfigsResponseData; import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData; import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData; @@ -80,6 +81,7 @@ import org.apache.kafka.common.message.DescribeAclsResponseData; import org.apache.kafka.common.message.DescribeAclsResponseData.AclDescription; import org.apache.kafka.common.message.DescribeAclsResponseData.DescribeAclsResource; +import org.apache.kafka.common.message.DescribeClientQuotasResponseData; import org.apache.kafka.common.message.DescribeConfigsRequestData; import org.apache.kafka.common.message.DescribeConfigsResponseData; import org.apache.kafka.common.message.DescribeConfigsResponseData.DescribeConfigsResourceResult; @@ -94,6 +96,7 @@ import org.apache.kafka.common.message.ExpireDelegationTokenRequestData; import org.apache.kafka.common.message.ExpireDelegationTokenResponseData; import org.apache.kafka.common.message.FetchRequestData; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.message.FindCoordinatorRequestData; import org.apache.kafka.common.message.HeartbeatRequestData; import org.apache.kafka.common.message.HeartbeatResponseData; @@ -160,8 +163,6 @@ import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.ObjectSerializationCache; -import org.apache.kafka.common.protocol.types.SchemaException; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.quota.ClientQuotaAlteration; import org.apache.kafka.common.quota.ClientQuotaEntity; import org.apache.kafka.common.quota.ClientQuotaFilter; @@ -182,17 +183,16 @@ import org.apache.kafka.common.security.token.delegation.TokenInformation; import org.apache.kafka.common.utils.SecurityUtils; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.test.TestUtils; import org.junit.Test; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -210,7 +210,6 @@ import static org.apache.kafka.common.protocol.ApiKeys.LIST_OFFSETS; import static org.apache.kafka.common.protocol.ApiKeys.SYNC_GROUP; import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; -import static org.apache.kafka.test.TestUtils.toBuffer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -455,7 +454,7 @@ public void testSerialization() throws Exception { checkResponse(createCreateAclsResponse(), ApiKeys.CREATE_ACLS.latestVersion(), true); checkRequest(createDeleteAclsRequest(), true); checkErrorResponse(createDeleteAclsRequest(), new SecurityDisabledException("Security is not enabled."), true); - checkResponse(createDeleteAclsResponse(), ApiKeys.DELETE_ACLS.latestVersion(), true); + checkResponse(createDeleteAclsResponse(ApiKeys.DELETE_ACLS.latestVersion()), ApiKeys.DELETE_ACLS.latestVersion(), true); checkRequest(createAlterConfigsRequest(), false); checkErrorResponse(createAlterConfigsRequest(), unknownServerException, true); checkResponse(createAlterConfigsResponse(), 0, false); @@ -514,12 +513,15 @@ public void testSerialization() throws Exception { @Test public void testResponseHeader() { ResponseHeader header = createResponseHeader((short) 1); - ByteBuffer buffer = toBuffer(header.toStruct()); + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + ByteBuffer buffer = ByteBuffer.allocate(header.size(serializationCache)); + header.write(buffer, serializationCache); + buffer.flip(); ResponseHeader deserialized = ResponseHeader.parse(buffer, header.headerVersion()); assertEquals(header.correlationId(), deserialized.correlationId()); } - private void checkOlderFetchVersions() throws Exception { + private void checkOlderFetchVersions() { int latestVersion = FETCH.latestVersion(); for (int i = 0; i < latestVersion; ++i) { if (i > 7) { @@ -530,12 +532,15 @@ private void checkOlderFetchVersions() throws Exception { } } - private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, DescribeConfigsResponse actual, int version) throws Exception { + private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, DescribeConfigsResponse actual, + int version) { for (Map.Entry resource : expected.resultMap().entrySet()) { List actualEntries = actual.resultMap().get(resource.getKey()).configs(); - Iterator expectedEntries = expected.resultMap().get(resource.getKey()).configs().iterator(); - for (DescribeConfigsResourceResult actualEntry : actualEntries) { - DescribeConfigsResourceResult expectedEntry = expectedEntries.next(); + List expectedEntries = expected.resultMap().get(resource.getKey()).configs(); + assertEquals(expectedEntries.size(), actualEntries.size()); + for (int i = 0; i < actualEntries.size(); ++i) { + DescribeConfigsResourceResult actualEntry = actualEntries.get(i); + DescribeConfigsResourceResult expectedEntry = expectedEntries.get(i); assertEquals(expectedEntry.name(), actualEntry.name()); assertEquals("Non-matching values for " + actualEntry.name() + " in version " + version, expectedEntry.value(), actualEntry.value()); @@ -550,84 +555,69 @@ private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, Des assertEquals("Non-matching configType for " + actualEntry.name() + " in version " + version, expectedEntry.configType(), actualEntry.configType()); } - if (version == 1 || version == 3 || (expectedEntry.configSource() != DescribeConfigsResponse.ConfigSource.DYNAMIC_BROKER_CONFIG.id() && - expectedEntry.configSource() != DescribeConfigsResponse.ConfigSource.DYNAMIC_DEFAULT_BROKER_CONFIG.id())) - assertEquals("Non-matching configSource for " + actualEntry.name() + " in version " + version, - expectedEntry.configSource(), actualEntry.configSource()); - else + if (version == 0) { assertEquals("Non matching configSource for " + actualEntry.name() + " in version " + version, DescribeConfigsResponse.ConfigSource.STATIC_BROKER_CONFIG.id(), actualEntry.configSource()); + } else { + assertEquals("Non-matching configSource for " + actualEntry.name() + " in version " + version, + expectedEntry.configSource(), actualEntry.configSource()); + } } } } - private void checkDescribeConfigsResponseVersions() throws Exception { - DescribeConfigsResponse response = createDescribeConfigsResponse((short) 0); - DescribeConfigsResponse deserialized0 = (DescribeConfigsResponse) deserialize(response, - response.toStruct((short) 0), (short) 0); - verifyDescribeConfigsResponse(response, deserialized0, 0); - - response = createDescribeConfigsResponse((short) 1); - DescribeConfigsResponse deserialized1 = (DescribeConfigsResponse) deserialize(response, - response.toStruct((short) 1), (short) 1); - verifyDescribeConfigsResponse(response, deserialized1, 1); - - response = createDescribeConfigsResponse((short) 3); - DescribeConfigsResponse deserialized3 = (DescribeConfigsResponse) deserialize(response, - response.toStruct((short) 3), (short) 3); - verifyDescribeConfigsResponse(response, deserialized3, 3); + private void checkDescribeConfigsResponseVersions() { + for (int version = ApiKeys.DESCRIBE_CONFIGS.oldestVersion(); version < ApiKeys.DESCRIBE_CONFIGS.latestVersion(); ++version) { + short apiVersion = (short) version; + DescribeConfigsResponse response = createDescribeConfigsResponse(apiVersion); + DescribeConfigsResponse deserialized0 = (DescribeConfigsResponse) AbstractResponse.parseResponse(ApiKeys.DESCRIBE_CONFIGS, + response.serializeBody(apiVersion), apiVersion); + verifyDescribeConfigsResponse(response, deserialized0, apiVersion); + } } private void checkErrorResponse(AbstractRequest req, Throwable e, boolean checkEqualityAndHashCode) { AbstractResponse response = req.getErrorResponse(e); checkResponse(response, req.version(), checkEqualityAndHashCode); if (e instanceof UnknownServerException) { - String responseStr = response.toStruct(req.version()).toString(); - assertFalse(String.format("Unknown message included in response for %s: %s ", req.api, responseStr), + String responseStr = response.toString(); + assertFalse(String.format("Unknown message included in response for %s: %s ", req.apiKey(), responseStr), responseStr.contains(e.getMessage())); } } - private void checkRequest(AbstractRequest req, boolean checkEqualityAndHashCode) { + private void checkRequest(AbstractRequest req, boolean checkEquality) { // Check that we can serialize, deserialize and serialize again - // Check for equality and hashCode of the Struct only if indicated (it is likely to fail if any of the fields + // Check for equality of the ByteBuffer only if indicated (it is likely to fail if any of the fields // in the request is a HashMap with multiple elements since ordering of the elements may vary) try { - Struct struct = req.toStruct(); - AbstractRequest deserialized = AbstractRequest.parseRequest(req.api, req.version(), struct); - Struct struct2 = deserialized.toStruct(); - if (checkEqualityAndHashCode) { - assertEquals(struct, struct2); - assertEquals(struct.hashCode(), struct2.hashCode()); - } + ByteBuffer serializedBytes = req.serializeBody(); + AbstractRequest deserialized = AbstractRequest.parseRequest(req.apiKey(), req.version(), serializedBytes).request; + ByteBuffer serializedBytes2 = deserialized.serializeBody(); + serializedBytes.rewind(); + if (checkEquality) + assertEquals("Request " + req + "failed equality test", serializedBytes, serializedBytes2); } catch (Exception e) { throw new RuntimeException("Failed to deserialize request " + req + " with type " + req.getClass(), e); } } - private void checkResponse(AbstractResponse response, int version, boolean checkEqualityAndHashCode) { + private void checkResponse(AbstractResponse response, int version, boolean checkEquality) { // Check that we can serialize, deserialize and serialize again // Check for equality and hashCode of the Struct only if indicated (it is likely to fail if any of the fields // in the response is a HashMap with multiple elements since ordering of the elements may vary) try { - Struct struct = response.toStruct((short) version); - AbstractResponse deserialized = (AbstractResponse) deserialize(response, struct, (short) version); - Struct struct2 = deserialized.toStruct((short) version); - if (checkEqualityAndHashCode) { - assertEquals(struct, struct2); - assertEquals(struct.hashCode(), struct2.hashCode()); - } + ByteBuffer serializedBytes = response.serializeBody((short) version); + AbstractResponse deserialized = AbstractResponse.parseResponse(response.apiKey(), serializedBytes, (short) version); + ByteBuffer serializedBytes2 = deserialized.serializeBody((short) version); + serializedBytes.rewind(); + if (checkEquality) + assertEquals("Response " + response + "failed equality test", serializedBytes, serializedBytes2); } catch (Exception e) { throw new RuntimeException("Failed to deserialize response " + response + " with type " + response.getClass(), e); } } - private AbstractRequestResponse deserialize(AbstractRequestResponse req, Struct struct, short version) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { - ByteBuffer buffer = toBuffer(struct); - Method deserializer = req.getClass().getDeclaredMethod("parse", ByteBuffer.class, Short.TYPE); - return (AbstractRequestResponse) deserializer.invoke(null, buffer, version); - } - @Test(expected = UnsupportedVersionException.class) public void cannotUseFindCoordinatorV0ToFindTransactionCoordinator() { FindCoordinatorRequest.Builder builder = new FindCoordinatorRequest.Builder( @@ -665,7 +655,7 @@ public void testPartitionSize() { @Test public void produceRequestToStringTest() { ProduceRequest request = createProduceRequest(ApiKeys.PRODUCE.latestVersion()); - assertEquals(1, request.dataOrException().topicData().size()); + assertEquals(1, request.data().topicData().size()); assertFalse(request.toString(false).contains("partitionSizes")); assertTrue(request.toString(false).contains("numPartitions=1")); assertTrue(request.toString(true).contains("partitionSizes")); @@ -673,7 +663,7 @@ public void produceRequestToStringTest() { request.clearPartitionRecords(); try { - request.dataOrException(); + request.data(); fail("dataOrException should fail after clearPartitionRecords()"); } catch (IllegalStateException e) { // OK @@ -723,10 +713,6 @@ public void fetchResponseVersionTest() { FetchResponse v1Response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID); assertEquals("Throttle time must be zero", 0, v0Response.throttleTimeMs()); assertEquals("Throttle time must be 10", 10, v1Response.throttleTimeMs()); - assertEquals("Should use schema version 0", FETCH.responseSchema((short) 0), - v0Response.toStruct((short) 0).schema()); - assertEquals("Should use schema version 1", FETCH.responseSchema((short) 1), - v1Response.toStruct((short) 1).schema()); assertEquals("Response data does not match", responseData, v0Response.responseData()); assertEquals("Response data does not match", responseData, v1Response.responseData()); } @@ -748,7 +734,7 @@ public void testFetchResponseV4() { 6, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), emptyList(), records)); FetchResponse response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID); - FetchResponse deserialized = FetchResponse.parse(toBuffer(response.toStruct((short) 4)), (short) 4); + FetchResponse deserialized = FetchResponse.parse(response.serializeBody((short) 4), (short) 4); assertEquals(responseData, deserialized.responseData()); } @@ -781,19 +767,17 @@ private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse ApiVersionsResponse.parse(ByteBuffer.allocate(0), version)); } @Test - public void testApiVersionResponseStructParsing() { - Struct struct = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.toStruct(ApiKeys.API_VERSIONS.latestVersion()); - ApiVersionsResponse response = ApiVersionsResponse. - fromStruct(struct, ApiKeys.API_VERSIONS.latestVersion()); + public void testApiVersionResponseParsing() { + ByteBuffer buffer = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.serializeBody(ApiKeys.API_VERSIONS.latestVersion()); + ApiVersionsResponse response = ApiVersionsResponse.parse(buffer, ApiKeys.API_VERSIONS.latestVersion()); assertEquals(Errors.NONE.code(), response.data.errorCode()); } @@ -1003,7 +985,7 @@ public void testInitProducerIdRequestVersions() { setTransactionalId("abracadabra"). setProducerId(123)); final UnsupportedVersionException exception = assertThrows( - UnsupportedVersionException.class, () -> bld.build((short) 2).toStruct()); + UnsupportedVersionException.class, () -> bld.build((short) 2).serializeBody()); assertTrue(exception.getMessage().contains("Attempted to write a non-default producerId at version 2")); bld.build((short) 3); } @@ -1367,7 +1349,7 @@ private MetadataResponse createMetadataResponse() { new TopicPartition("topic3", 0), Optional.empty(), Optional.empty(), replicas, isr, offlineReplicas)))); - return MetadataResponse.prepareResponse(asList(node), null, MetadataResponse.NO_CONTROLLER_ID, allTopicMetadata); + return TestUtils.metadataResponse(asList(node), null, MetadataResponse.NO_CONTROLLER_ID, allTopicMetadata); } private OffsetCommitRequest createOffsetCommitRequest(int version) { @@ -1679,7 +1661,7 @@ private SaslHandshakeResponse createSaslHandshakeResponse() { private SaslAuthenticateRequest createSaslAuthenticateRequest() { SaslAuthenticateRequestData data = new SaslAuthenticateRequestData().setAuthBytes(new byte[0]); - return new SaslAuthenticateRequest(data); + return new SaslAuthenticateRequest(data, ApiKeys.SASL_AUTHENTICATE.latestVersion()); } private SaslAuthenticateResponse createSaslAuthenticateResponse() { @@ -2040,7 +2022,7 @@ private DeleteAclsRequest createDeleteAclsRequest() { return new DeleteAclsRequest.Builder(data).build(); } - private DeleteAclsResponse createDeleteAclsResponse() { + private DeleteAclsResponse createDeleteAclsResponse(int version) { List filterResults = new ArrayList<>(); filterResults.add(new DeleteAclsResponseData.DeleteAclsFilterResult().setMatchingAcls(asList( new DeleteAclsResponseData.DeleteAclsMatchingAcl() @@ -2064,7 +2046,7 @@ private DeleteAclsResponse createDeleteAclsResponse() { .setErrorMessage("No security")); return new DeleteAclsResponse(new DeleteAclsResponseData() .setThrottleTimeMs(0) - .setFilterResults(filterResults)); + .setFilterResults(filterResults), (short) version); } private DescribeConfigsRequest createDescribeConfigsRequest(int version) { @@ -2350,7 +2332,7 @@ private ElectLeadersResponse createElectLeadersResponse() { partitionResult.setErrorMessage(Errors.UNKNOWN_TOPIC_OR_PARTITION.message()); electionResult.partitionResult().add(partitionResult); - return new ElectLeadersResponse(200, Errors.NONE.code(), electionResults); + return new ElectLeadersResponse(200, Errors.NONE.code(), electionResults, ApiKeys.ELECT_LEADERS.latestVersion()); } private IncrementalAlterConfigsRequest createIncrementalAlterConfigsRequest() { @@ -2511,8 +2493,15 @@ private DescribeClientQuotasRequest createDescribeClientQuotasRequest() { } private DescribeClientQuotasResponse createDescribeClientQuotasResponse() { - ClientQuotaEntity entity = new ClientQuotaEntity(Collections.singletonMap(ClientQuotaEntity.USER, "user")); - return new DescribeClientQuotasResponse(Collections.singletonMap(entity, Collections.singletonMap("request_percentage", 1.0)), 0); + DescribeClientQuotasResponseData data = new DescribeClientQuotasResponseData().setEntries(asList( + new DescribeClientQuotasResponseData.EntryData() + .setEntity(asList(new DescribeClientQuotasResponseData.EntityData() + .setEntityType(ClientQuotaEntity.USER) + .setEntityName("user"))) + .setValues(asList(new DescribeClientQuotasResponseData.ValueData() + .setKey("request_percentage") + .setValue(1.0))))); + return new DescribeClientQuotasResponse(data); } private AlterClientQuotasRequest createAlterClientQuotasRequest() { @@ -2523,8 +2512,12 @@ private AlterClientQuotasRequest createAlterClientQuotasRequest() { } private AlterClientQuotasResponse createAlterClientQuotasResponse() { - ClientQuotaEntity entity = new ClientQuotaEntity(Collections.singletonMap(ClientQuotaEntity.USER, "user")); - return new AlterClientQuotasResponse(Collections.singletonMap(entity, ApiError.NONE), 0); + AlterClientQuotasResponseData data = new AlterClientQuotasResponseData() + .setEntries(asList(new AlterClientQuotasResponseData.EntryData() + .setEntity(asList(new AlterClientQuotasResponseData.EntityData() + .setEntityType(ClientQuotaEntity.USER) + .setEntityName("user"))))); + return new AlterClientQuotasResponse(data); } /** @@ -2544,7 +2537,7 @@ public void testErrorCountsIncludesNone() { assertEquals(Integer.valueOf(1), createCreatePartitionsResponse().errorCounts().get(Errors.NONE)); assertEquals(Integer.valueOf(1), createCreateTokenResponse().errorCounts().get(Errors.NONE)); assertEquals(Integer.valueOf(1), createCreateTopicResponse().errorCounts().get(Errors.NONE)); - assertEquals(Integer.valueOf(1), createDeleteAclsResponse().errorCounts().get(Errors.NONE)); + assertEquals(Integer.valueOf(1), createDeleteAclsResponse(ApiKeys.DELETE_ACLS.latestVersion()).errorCounts().get(Errors.NONE)); assertEquals(Integer.valueOf(1), createDeleteGroupsResponse().errorCounts().get(Errors.NONE)); assertEquals(Integer.valueOf(1), createDeleteTopicsResponse().errorCounts().get(Errors.NONE)); assertEquals(Integer.valueOf(1), createDescribeAclsResponse().errorCounts().get(Errors.NONE)); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java index a038aaf9de479..243efa9bbf4b9 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java @@ -129,12 +129,12 @@ public void testTopicStatesNormalization() { List topicStates = topicStates(true); for (short version = STOP_REPLICA.oldestVersion(); version < STOP_REPLICA.latestVersion(); version++) { - // Create a request for version to get its struct + // Create a request for version to get its serialized form StopReplicaRequest baseRequest = new StopReplicaRequest.Builder(version, 0, 1, 0, true, topicStates).build(version); - // Construct the request from the struct - StopReplicaRequest request = new StopReplicaRequest(baseRequest.toStruct(), version); + // Construct the request from the buffer + StopReplicaRequest request = StopReplicaRequest.parse(baseRequest.serializeBody(), version); Map partitionStates = StopReplicaRequestTest.partitionStates(request.topicStates()); @@ -164,12 +164,12 @@ public void testPartitionStatesNormalization() { List topicStates = topicStates(true); for (short version = STOP_REPLICA.oldestVersion(); version < STOP_REPLICA.latestVersion(); version++) { - // Create a request for version to get its struct + // Create a request for version to get its serialized form StopReplicaRequest baseRequest = new StopReplicaRequest.Builder(version, 0, 1, 0, true, topicStates).build(version); - // Construct the request from the struct - StopReplicaRequest request = new StopReplicaRequest(baseRequest.toStruct(), version); + // Construct the request from the buffer + StopReplicaRequest request = StopReplicaRequest.parse(baseRequest.serializeBody(), version); Map partitionStates = request.partitionStates(); assertEquals(6, partitionStates.size()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java index be4f5df359a05..bf3e9c8d2115e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java @@ -17,10 +17,8 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.TxnOffsetCommitResponseData; -import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition; -import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; - +import org.apache.kafka.common.protocol.MessageTestUtil; import org.junit.Test; import java.util.Arrays; @@ -42,26 +40,27 @@ public void testConstructorWithErrorResponse() { @Test @Override - public void testConstructorWithStruct() { + public void testParse() { TxnOffsetCommitResponseData data = new TxnOffsetCommitResponseData() .setThrottleTimeMs(throttleTimeMs) .setTopics(Arrays.asList( - new TxnOffsetCommitResponseTopic().setPartitions( - Collections.singletonList(new TxnOffsetCommitResponsePartition() - .setPartitionIndex(partitionOne) - .setErrorCode(errorOne.code()))), - new TxnOffsetCommitResponseTopic().setPartitions( - Collections.singletonList(new TxnOffsetCommitResponsePartition() - .setPartitionIndex(partitionTwo) - .setErrorCode(errorTwo.code())) - ) - )); + new TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic().setPartitions( + Collections.singletonList(new TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition() + .setPartitionIndex(partitionOne) + .setErrorCode(errorOne.code()))), + new TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic().setPartitions( + Collections.singletonList(new TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition() + .setPartitionIndex(partitionTwo) + .setErrorCode(errorTwo.code()))) + )); for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) { - TxnOffsetCommitResponse response = new TxnOffsetCommitResponse(data.toStruct(version), version); + TxnOffsetCommitResponse response = TxnOffsetCommitResponse.parse( + MessageTestUtil.messageToByteBuffer(data, version), version); assertEquals(expectedErrorCounts, response.errorCounts()); assertEquals(throttleTimeMs, response.throttleTimeMs()); assertEquals(version >= 1, response.shouldClientThrottle(version)); } } + } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java index 6bd83c6eeb80c..8da3fedef2a08 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java @@ -26,7 +26,6 @@ import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.MessageTestUtil; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.test.TestUtils; import org.junit.Test; @@ -157,7 +156,7 @@ public void testVersionLogic() { assertEquals(2, request.controllerEpoch()); assertEquals(3, request.brokerEpoch()); - ByteBuffer byteBuffer = MessageTestUtil.messageToByteBuffer(request.data(), request.version()); + ByteBuffer byteBuffer = request.serializeBody(); UpdateMetadataRequest deserializedRequest = new UpdateMetadataRequest(new UpdateMetadataRequestData( new ByteBufferAccessor(byteBuffer), version), version); @@ -207,8 +206,7 @@ public void testTopicPartitionGroupingSizeReduction() { UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder((short) 5, 0, 0, 0, partitionStates, Collections.emptyList()); - assertTrue(MessageTestUtil.messageSize(builder.build((short) 5).data(), (short) 5) < - MessageTestUtil.messageSize(builder.build((short) 4).data(), (short) 4)); + assertTrue(builder.build((short) 5).sizeInBytes() < builder.build((short) 4).sizeInBytes()); } private Set iterableToSet(Iterable iterable) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java index d869201b52662..12fa71b92e015 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java @@ -72,7 +72,7 @@ public void testGetErrorResponse() { request.getErrorResponse(throttleTimeMs, Errors.UNKNOWN_PRODUCER_ID.exception()); assertEquals(Collections.singletonMap( - topicPartition, Errors.UNKNOWN_PRODUCER_ID), errorResponse.errors(producerId)); + topicPartition, Errors.UNKNOWN_PRODUCER_ID), errorResponse.errorsByProducerId().get(producerId)); assertEquals(Collections.singletonMap(Errors.UNKNOWN_PRODUCER_ID, 1), errorResponse.errorCounts()); // Write txn marker has no throttle time defined in response. assertEquals(0, errorResponse.throttleTimeMs()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java index 4cec88c22e923..3860b186f072e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java @@ -46,15 +46,15 @@ public void setUp() { errorMap.put(producerIdOne, Collections.singletonMap(tp1, pidOneError)); errorMap.put(producerIdTwo, Collections.singletonMap(tp2, pidTwoError)); } - @Test - public void testConstructorWithStruct() { + @Test + public void testConstructor() { Map expectedErrorCounts = new HashMap<>(); expectedErrorCounts.put(Errors.UNKNOWN_PRODUCER_ID, 1); expectedErrorCounts.put(Errors.INVALID_PRODUCER_EPOCH, 1); WriteTxnMarkersResponse response = new WriteTxnMarkersResponse(errorMap); assertEquals(expectedErrorCounts, response.errorCounts()); - assertEquals(Collections.singletonMap(tp1, pidOneError), response.errors(producerIdOne)); - assertEquals(Collections.singletonMap(tp2, pidTwoError), response.errors(producerIdTwo)); + assertEquals(Collections.singletonMap(tp1, pidOneError), response.errorsByProducerId().get(producerIdOne)); + assertEquals(Collections.singletonMap(tp2, pidTwoError), response.errorsByProducerId().get(producerIdTwo)); } } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java index 5c1ce3cfc0653..77bd9197476a9 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java @@ -132,6 +132,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -1602,7 +1603,7 @@ public void testConvertListOffsetResponseToSaslHandshakeResponse() { .setOffset(0) .setTimestamp(0))))); ListOffsetResponse response = new ListOffsetResponse(data); - ByteBuffer buffer = response.serialize(ApiKeys.LIST_OFFSETS, LIST_OFFSETS.latestVersion(), 0); + ByteBuffer buffer = response.serializeWithHeader(LIST_OFFSETS.latestVersion(), 0); final RequestHeader header0 = new RequestHeader(LIST_OFFSETS, LIST_OFFSETS.latestVersion(), "id", SaslClientAuthenticator.MIN_RESERVED_CORRELATION_ID); Assert.assertThrows(SchemaException.class, () -> NetworkClient.parseResponse(buffer.duplicate(), header0)); final RequestHeader header1 = new RequestHeader(LIST_OFFSETS, LIST_OFFSETS.latestVersion(), "id", 1); @@ -1681,19 +1682,18 @@ public void testCannotReauthenticateAgainFasterThanOneSecond() throws Exception * instead */ time.sleep((long) (CONNECTIONS_MAX_REAUTH_MS_VALUE * 1.1)); - NetworkTestUtils.checkClientConnection(selector, node, 1, 1); - fail("Expected a failure when trying to re-authenticate to quickly, but that did not occur"); - } catch (AssertionError e) { + AssertionError exception = assertThrows(AssertionError.class, + () -> NetworkTestUtils.checkClientConnection(selector, node, 1, 1)); String expectedResponseTextRegex = "\\w-" + node; String receivedResponseTextRegex = ".*" + OAuthBearerLoginModule.OAUTHBEARER_MECHANISM; assertTrue( "Should have received the SaslHandshakeRequest bytes back since we re-authenticated too quickly, " + "but instead we got our generated message echoed back, implying re-auth succeeded when it " + - "should not have: " + e, - e.getMessage().matches( + "should not have: " + exception, + exception.getMessage().matches( ".*<\\[" + expectedResponseTextRegex + "]>.*<\\[" + receivedResponseTextRegex + ".*?]>")); server.verifyReauthenticationMetrics(1, 0); // unchanged - } finally { + } finally { selector.close(); selector = null; } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java index 0b4c63bc42cb1..a4d2d771f5c17 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java @@ -29,11 +29,11 @@ import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; import org.apache.kafka.common.security.auth.SecurityProtocol; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.security.plain.PlainLoginModule; import org.apache.kafka.common.utils.AppInfoParser; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.test.TestUtils; import org.junit.Test; import javax.security.auth.Subject; @@ -79,16 +79,16 @@ public void testUnexpectedRequestType() throws IOException { SaslServerAuthenticator authenticator = setupAuthenticator(configs, transportLayer, SCRAM_SHA_256.mechanismName(), new DefaultChannelMetadataRegistry()); - final RequestHeader header = new RequestHeader(ApiKeys.METADATA, (short) 0, "clientId", 13243); - final Struct headerStruct = header.toStruct(); + RequestHeader header = new RequestHeader(ApiKeys.METADATA, (short) 0, "clientId", 13243); + ByteBuffer headerBuffer = TestUtils.serializeRequestHeader(header); when(transportLayer.read(any(ByteBuffer.class))).then(invocation -> { - invocation.getArgument(0).putInt(headerStruct.sizeOf()); + invocation.getArgument(0).putInt(headerBuffer.remaining()); return 4; }).then(invocation -> { // serialize only the request header. the authenticator should not parse beyond this - headerStruct.writeTo(invocation.getArgument(0)); - return headerStruct.sizeOf(); + invocation.getArgument(0).put(headerBuffer.duplicate()); + return headerBuffer.remaining(); }); try { @@ -113,7 +113,7 @@ public void testLatestApiVersionsRequest() throws IOException { "apache-kafka-java", AppInfoParser.getVersion()); } - public void testApiVersionsRequest(short version, String expectedSoftwareName, + private void testApiVersionsRequest(short version, String expectedSoftwareName, String expectedSoftwareVersion) throws IOException { TransportLayer transportLayer = mock(TransportLayer.class, Answers.RETURNS_DEEP_STUBS); Map configs = Collections.singletonMap(BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_CONFIG, @@ -122,21 +122,23 @@ public void testApiVersionsRequest(short version, String expectedSoftwareName, SaslServerAuthenticator authenticator = setupAuthenticator(configs, transportLayer, SCRAM_SHA_256.mechanismName(), metadataRegistry); - final RequestHeader header = new RequestHeader(ApiKeys.API_VERSIONS, version, "clientId", 0); - final Struct headerStruct = header.toStruct(); + RequestHeader header = new RequestHeader(ApiKeys.API_VERSIONS, version, "clientId", 0); + ByteBuffer headerBuffer = TestUtils.serializeRequestHeader(header); - final ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version); - final Struct requestStruct = request.data.toStruct(version); + ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version); + ByteBuffer requestBuffer = request.serializeBody(); + requestBuffer.rewind(); when(transportLayer.socketChannel().socket().getInetAddress()).thenReturn(InetAddress.getLoopbackAddress()); when(transportLayer.read(any(ByteBuffer.class))).then(invocation -> { - invocation.getArgument(0).putInt(headerStruct.sizeOf() + requestStruct.sizeOf()); + invocation.getArgument(0).putInt(headerBuffer.remaining() + requestBuffer.remaining()); return 4; }).then(invocation -> { - headerStruct.writeTo(invocation.getArgument(0)); - requestStruct.writeTo(invocation.getArgument(0)); - return headerStruct.sizeOf() + requestStruct.sizeOf(); + invocation.getArgument(0) + .put(headerBuffer.duplicate()) + .put(requestBuffer.duplicate()); + return headerBuffer.remaining() + requestBuffer.remaining(); }); authenticator.authenticate(); diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java index f006a20ba2dda..50a79c7dbb19e 100644 --- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java +++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java @@ -24,11 +24,13 @@ import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.MetadataResponseData; import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ObjectSerializationCache; +import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.requests.ByteBufferChannel; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.RequestHeader; @@ -116,6 +118,41 @@ public static Cluster clusterWith(final int nodes, final Map to return new Cluster("kafka-cluster", asList(ns), parts, Collections.emptySet(), Collections.emptySet()); } + public static MetadataResponse metadataResponse(Collection brokers, + String clusterId, int controllerId, + List topicMetadataList) { + return metadataResponse(MetadataResponse.DEFAULT_THROTTLE_TIME, brokers, clusterId, controllerId, + topicMetadataList, MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + } + + public static MetadataResponse metadataResponse(int throttleTimeMs, Collection brokers, + String clusterId, int controllerId, + List topicMetadatas, + int clusterAuthorizedOperations) { + List topics = new ArrayList<>(); + topicMetadatas.forEach(topicMetadata -> { + MetadataResponseData.MetadataResponseTopic metadataResponseTopic = new MetadataResponseData.MetadataResponseTopic(); + metadataResponseTopic + .setErrorCode(topicMetadata.error().code()) + .setName(topicMetadata.topic()) + .setIsInternal(topicMetadata.isInternal()) + .setTopicAuthorizedOperations(topicMetadata.authorizedOperations()); + + for (MetadataResponse.PartitionMetadata partitionMetadata : topicMetadata.partitionMetadata()) { + metadataResponseTopic.partitions().add(new MetadataResponseData.MetadataResponsePartition() + .setErrorCode(partitionMetadata.error.code()) + .setPartitionIndex(partitionMetadata.partition()) + .setLeaderId(partitionMetadata.leaderId.orElse(MetadataResponse.NO_LEADER_ID)) + .setLeaderEpoch(partitionMetadata.leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)) + .setReplicaNodes(partitionMetadata.replicaIds) + .setIsrNodes(partitionMetadata.inSyncReplicaIds) + .setOfflineReplicas(partitionMetadata.offlineReplicaIds)); + } + topics.add(metadataResponseTopic); + }); + return MetadataResponse.prepareResponse(true, throttleTimeMs, brokers, clusterId, controllerId, + topics, clusterAuthorizedOperations); } + public static MetadataResponse metadataUpdateWith(final int numNodes, final Map topicPartitionCounts) { return metadataUpdateWith("kafka-cluster", numNodes, topicPartitionCounts); @@ -197,7 +234,15 @@ public static MetadataResponse metadataUpdateWith(final String clusterId, Topic.isInternal(topic), Collections.emptyList())); } - return MetadataResponse.prepareResponse(nodes, clusterId, 0, topicMetadata, responseVersion); + return metadataResponse(nodes, clusterId, 0, topicMetadata); + } + + public static ByteBuffer serializeRequestHeader(RequestHeader header) { + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + ByteBuffer buffer = ByteBuffer.allocate(header.size(serializationCache)); + header.write(buffer, serializationCache); + buffer.rewind(); + return buffer; } @FunctionalInterface @@ -526,13 +571,6 @@ public static Set toSet(Collection collection) { return new HashSet<>(collection); } - public static ByteBuffer toBuffer(Struct struct) { - ByteBuffer buffer = ByteBuffer.allocate(struct.sizeOf()); - struct.writeTo(buffer); - buffer.rewind(); - return buffer; - } - public static ByteBuffer toBuffer(Send send) { ByteBufferChannel channel = new ByteBufferChannel(send.size()); try { diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java index dd3062e4159bc..e4b456df61663 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java @@ -38,6 +38,7 @@ import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult; import org.apache.kafka.common.message.DescribeConfigsResponseData; import org.apache.kafka.common.message.MetadataResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ApiError; import org.apache.kafka.common.requests.CreateTopicsResponse; @@ -553,7 +554,7 @@ private MetadataResponse describeTopicResponse(ApiError error, NewTopic... topic .setName(topic.name()) .setErrorCode(error.error().code())); } - return new MetadataResponse(response); + return new MetadataResponse(response, ApiKeys.METADATA.latestVersion()); } private DescribeConfigsResponse describeConfigsResponseWithUnsupportedVersion(NewTopic... topics) { diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala index 1391d345c685f..f217776109cad 100755 --- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala +++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala @@ -271,7 +271,7 @@ class RequestSendThread(val controllerId: Int, val response = clientResponse.responseBody stateChangeLogger.withControllerEpoch(controllerContext.epoch).trace(s"Received response " + - s"${response.toString(requestHeader.apiVersion)} for request $api with correlation id " + + s"$response for request $api with correlation id " + s"${requestHeader.correlationId} sent to broker $brokerNode") if (callback != null) { diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala index c2efdbd934672..848e0fa65ceeb 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala @@ -89,10 +89,11 @@ class TransactionMarkerRequestCompletionHandler(brokerId: Int, val writeTxnMarkerResponse = response.responseBody.asInstanceOf[WriteTxnMarkersResponse] + val responseErrors = writeTxnMarkerResponse.errorsByProducerId; for (txnIdAndMarker <- txnIdAndMarkerEntries.asScala) { val transactionalId = txnIdAndMarker.txnId val txnMarker = txnIdAndMarker.txnMarkerEntry - val errors = writeTxnMarkerResponse.errors(txnMarker.producerId) + val errors = responseErrors.get(txnMarker.producerId) if (errors == null) throw new IllegalStateException(s"WriteTxnMarkerResponse does not contain expected error map for producer id ${txnMarker.producerId}") diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 3945e8e15c17c..39ca69f1f1893 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -20,7 +20,6 @@ package kafka.network import java.net.InetAddress import java.nio.ByteBuffer import java.util.concurrent._ - import com.typesafe.scalalogging.Logger import com.yammer.metrics.core.Meter import kafka.log.LogConfig @@ -34,7 +33,7 @@ import org.apache.kafka.common.memory.MemoryPool import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData._ import org.apache.kafka.common.network.Send -import org.apache.kafka.common.protocol.{ApiKeys, Errors} +import org.apache.kafka.common.protocol.{ApiKeys, Errors, ObjectSerializationCache} import org.apache.kafka.common.requests._ import org.apache.kafka.common.security.auth.KafkaPrincipal import org.apache.kafka.common.utils.{Sanitizer, Time} @@ -99,11 +98,14 @@ object RequestChannel extends Logging { private val bodyAndSize: RequestAndSize = context.parseRequest(buffer) def header: RequestHeader = context.header + def sizeOfBodyInBytes: Int = bodyAndSize.size - // most request types are parsed entirely into objects at this point. for those we can release the underlying buffer. - // some (like produce, or any time the schema contains fields of types BYTES or NULLABLE_BYTES) retain a reference - // to the buffer. for those requests we cannot release the buffer early, but only when request processing is done. + def sizeInBytes: Int = header.size(new ObjectSerializationCache) + sizeOfBodyInBytes + + //most request types are parsed entirely into objects at this point. for those we can release the underlying buffer. + //some (like produce, or any time the schema contains fields of types BYTES or NULLABLE_BYTES) retain a reference + //to the buffer. for those requests we cannot release the buffer early, but only when request processing is done. if (!header.apiKey.requiresDelayedAllocation) { releaseBuffer() } @@ -123,7 +125,7 @@ object RequestChannel extends Logging { def responseString(response: AbstractResponse): Option[String] = { if (RequestChannel.isRequestLoggingEnabled) - Some(response.toString(context.apiVersion)) + Some(response.toString) else None } diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 072ea5304bbbb..6aa71505acf4f 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -24,7 +24,6 @@ import java.util import java.util.{Collections, Optional} import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger - import kafka.admin.{AdminUtils, RackAwareMode} import kafka.api.ElectLeadersRequestOps import kafka.api.{ApiVersion, KAFKA_0_11_0_IV0, KAFKA_2_3_IV0} @@ -51,7 +50,7 @@ import org.apache.kafka.common.internals.{FatalExitError, Topic} import org.apache.kafka.common.internals.Topic.{GROUP_METADATA_TOPIC_NAME, TRANSACTION_STATE_TOPIC_NAME, isInternal} import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic import org.apache.kafka.common.message.CreatePartitionsResponseData.CreatePartitionsTopicResult -import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetResponseData, ListPartitionReassignmentsResponseData, MetadataResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} +import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterClientQuotasResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeClientQuotasResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetResponseData, ListPartitionReassignmentsResponseData, MetadataResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} import org.apache.kafka.common.message.CreateTopicsResponseData.{CreatableTopicResult, CreatableTopicResultCollection} import org.apache.kafka.common.message.DeleteGroupsResponseData.{DeletableGroupResult, DeletableGroupResultCollection} import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.{ReassignablePartitionResponse, ReassignableTopicResponse} @@ -569,7 +568,7 @@ class KafkaApis(val requestChannel: RequestChannel, */ def handleProduceRequest(request: RequestChannel.Request): Unit = { val produceRequest = request.body[ProduceRequest] - val numBytesAppended = request.header.toStruct.sizeOf + request.sizeOfBodyInBytes + val requestSize = request.sizeInBytes val (hasIdempotentRecords, hasTransactionalRecords) = { val flags = RequestUtils.flags(produceRequest) @@ -595,9 +594,9 @@ class KafkaApis(val requestChannel: RequestChannel, val authorizedRequestInfo = mutable.Map[TopicPartition, MemoryRecords]() // cache the result to avoid redundant authorization calls val authorizedTopics = filterByAuthorized(request.context, WRITE, TOPIC, - produceRequest.dataOrException().topicData().asScala)(_.name()) + produceRequest.data().topicData().asScala)(_.name()) - produceRequest.dataOrException.topicData.forEach(topic => topic.partitionData.forEach { partition => + produceRequest.data.topicData.forEach(topic => topic.partitionData.forEach { partition => val topicPartition = new TopicPartition(topic.name, partition.index) // This caller assumes the type is MemoryRecords and that is true on current serialization // We cast the type to avoid causing big change to code base. @@ -641,7 +640,7 @@ class KafkaApis(val requestChannel: RequestChannel, // have been violated. If both quotas have been violated, use the max throttle time between the two quotas. Note // that the request quota is not enforced if acks == 0. val timeMs = time.milliseconds() - val bandwidthThrottleTimeMs = quotas.produce.maybeRecordAndGetThrottleTimeMs(request, numBytesAppended, timeMs) + val bandwidthThrottleTimeMs = quotas.produce.maybeRecordAndGetThrottleTimeMs(request, requestSize, timeMs) val requestThrottleTimeMs = if (produceRequest.acks == 0) 0 else quotas.request.maybeRecordAndGetThrottleTimeMs(request, timeMs) @@ -1345,11 +1344,12 @@ class KafkaApis(val requestChannel: RequestChannel, sendResponseMaybeThrottle(request, requestThrottleMs => MetadataResponse.prepareResponse( + requestVersion, requestThrottleMs, - completeTopicMetadata.asJava, brokers.flatMap(_.getNode(request.context.listenerName)).asJava, clusterId, metadataCache.getControllerId.getOrElse(MetadataResponse.NO_CONTROLLER_ID), + completeTopicMetadata.asJava, clusterAuthorizedOperations )) } @@ -2506,7 +2506,8 @@ class KafkaApis(val requestChannel: RequestChannel, new DescribeAclsResponse(new DescribeAclsResponseData() .setErrorCode(Errors.SECURITY_DISABLED.code) .setErrorMessage("No Authorizer is configured on the broker") - .setThrottleTimeMs(requestThrottleMs))) + .setThrottleTimeMs(requestThrottleMs), + describeAclsRequest.version)) case Some(auth) => val filter = describeAclsRequest.filter val returnedAcls = new util.HashSet[AclBinding]() @@ -2514,7 +2515,8 @@ class KafkaApis(val requestChannel: RequestChannel, sendResponseMaybeThrottle(request, requestThrottleMs => new DescribeAclsResponse(new DescribeAclsResponseData() .setThrottleTimeMs(requestThrottleMs) - .setResources(DescribeAclsResponse.aclsResources(returnedAcls)))) + .setResources(DescribeAclsResponse.aclsResources(returnedAcls)), + describeAclsRequest.version)) } } @@ -2585,9 +2587,11 @@ class KafkaApis(val requestChannel: RequestChannel, def sendResponseCallback(): Unit = { val filterResults = deleteResults.map(_.get).map(DeleteAclsResponse.filterResult).asJava sendResponseMaybeThrottle(request, requestThrottleMs => - new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(requestThrottleMs) - .setFilterResults(filterResults))) + new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(requestThrottleMs) + .setFilterResults(filterResults), + deleteAclsRequest.version)) } alterAclsPurgatory.tryCompleteElseWatch(config.connectionsMaxIdleMs, deleteResults, sendResponseCallback) } @@ -3125,11 +3129,30 @@ class KafkaApis(val requestChannel: RequestChannel, val describeClientQuotasRequest = request.body[DescribeClientQuotasRequest] if (authorize(request.context, DESCRIBE_CONFIGS, CLUSTER, CLUSTER_NAME)) { - val result = adminManager.describeClientQuotas(describeClientQuotasRequest.filter).map { case (quotaEntity, quotaConfigs) => - quotaEntity -> quotaConfigs.map { case (key, value) => key -> Double.box(value) }.asJava - }.asJava + val result = adminManager.describeClientQuotas(describeClientQuotasRequest.filter) + + val entriesData = result.iterator.map { case (quotaEntity, quotaValues) => + val entityData = quotaEntity.entries.asScala.iterator.map { case (entityType, entityName) => + new DescribeClientQuotasResponseData.EntityData() + .setEntityType(entityType) + .setEntityName(entityName) + }.toBuffer + + val valueData = quotaValues.iterator.map { case (key, value) => + new DescribeClientQuotasResponseData.ValueData() + .setKey(key) + .setValue(value) + }.toBuffer + + new DescribeClientQuotasResponseData.EntryData() + .setEntity(entityData.asJava) + .setValues(valueData.asJava) + }.toBuffer + sendResponseMaybeThrottle(request, requestThrottleMs => - new DescribeClientQuotasResponse(result, requestThrottleMs)) + new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData() + .setThrottleTimeMs(requestThrottleMs) + .setEntries(entriesData.asJava))) } else { sendResponseMaybeThrottle(request, requestThrottleMs => describeClientQuotasRequest.getErrorResponse(requestThrottleMs, Errors.CLUSTER_AUTHORIZATION_FAILED.exception)) @@ -3140,10 +3163,26 @@ class KafkaApis(val requestChannel: RequestChannel, val alterClientQuotasRequest = request.body[AlterClientQuotasRequest] if (authorize(request.context, ALTER_CONFIGS, CLUSTER, CLUSTER_NAME)) { - val result = adminManager.alterClientQuotas(alterClientQuotasRequest.entries().asScala.toSeq, - alterClientQuotasRequest.validateOnly()).asJava + val result = adminManager.alterClientQuotas(alterClientQuotasRequest.entries.asScala, + alterClientQuotasRequest.validateOnly) + + val entriesData = result.iterator.map { case (quotaEntity, apiError) => + val entityData = quotaEntity.entries.asScala.iterator.map { case (key, value) => + new AlterClientQuotasResponseData.EntityData() + .setEntityType(key) + .setEntityName(value) + }.toBuffer + + new AlterClientQuotasResponseData.EntryData() + .setErrorCode(apiError.error.code) + .setErrorMessage(apiError.message) + .setEntity(entityData.asJava) + }.toBuffer + sendResponseMaybeThrottle(request, requestThrottleMs => - new AlterClientQuotasResponse(result, requestThrottleMs)) + new AlterClientQuotasResponse(new AlterClientQuotasResponseData() + .setThrottleTimeMs(requestThrottleMs) + .setEntries(entriesData.asJava))) } else { sendResponseMaybeThrottle(request, requestThrottleMs => alterClientQuotasRequest.getErrorResponse(requestThrottleMs, Errors.CLUSTER_AUTHORIZATION_FAILED.exception)) diff --git a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala index a8f7cd93838c3..fefd274927df2 100644 --- a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala +++ b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala @@ -94,7 +94,7 @@ class TestRaftRequestHandler( case Some(response) => val responseSend = request.context.buildResponseSend(response) val responseString = - if (RequestChannel.isRequestLoggingEnabled) Some(response.toString(request.context.apiVersion)) + if (RequestChannel.isRequestLoggingEnabled) Some(response.toString) else None new RequestChannel.SendResponse(request, responseSend, responseString, None) case None => diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala index 26fec0cc81df9..e8cce7abe8118 100644 --- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala @@ -196,7 +196,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.ALTER_CONFIGS -> ((resp: AlterConfigsResponse) => resp.errors.get(new ConfigResource(ConfigResource.Type.TOPIC, tp.topic)).error), ApiKeys.INIT_PRODUCER_ID -> ((resp: InitProducerIdResponse) => resp.error), - ApiKeys.WRITE_TXN_MARKERS -> ((resp: WriteTxnMarkersResponse) => resp.errors(producerId).get(tp)), + ApiKeys.WRITE_TXN_MARKERS -> ((resp: WriteTxnMarkersResponse) => resp.errorsByProducerId.get(producerId).get(tp)), ApiKeys.ADD_PARTITIONS_TO_TXN -> ((resp: AddPartitionsToTxnResponse) => resp.errors.get(tp)), ApiKeys.ADD_OFFSETS_TO_TXN -> ((resp: AddOffsetsToTxnResponse) => Errors.forCode(resp.data.errorCode)), ApiKeys.END_TXN -> ((resp: EndTxnResponse) => resp.error), @@ -1808,7 +1808,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { resources: Set[ResourceType], isAuthorized: Boolean, topicExists: Boolean = true): AbstractResponse = { - val apiKey = request.api + val apiKey = request.apiKey val response = connectAndReceive[AbstractResponse](request) val error = requestKeyToError(apiKey).asInstanceOf[AbstractResponse => Errors](response) diff --git a/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala b/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala index 18b838232048f..aa1abb4026557 100644 --- a/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala +++ b/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala @@ -165,7 +165,7 @@ class RequestChannelTest { } def request(req: AbstractRequest): RequestChannel.Request = { - val buffer = req.serialize(new RequestHeader(req.api, req.version, "client-id", 1)) + val buffer = req.serializeWithHeader(new RequestHeader(req.apiKey, req.version, "client-id", 1)) val requestContext = newRequestContext(buffer) new network.RequestChannel.Request(processor = 1, requestContext, @@ -195,4 +195,4 @@ class RequestChannelTest { private def toMap(config: IncrementalAlterConfigsRequestData.AlterableConfigCollection): Map[String, String] = { config.asScala.map(e => e.name -> e.value).toMap } -} \ No newline at end of file +} diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index d8b76e05a9217..49d1490c96e75 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -112,7 +112,7 @@ class SocketServerTest { } def sendApiRequest(socket: Socket, request: AbstractRequest, header: RequestHeader) = { - val byteBuffer = request.serialize(header) + val byteBuffer = request.serializeWithHeader(header) byteBuffer.rewind() val serializedBytes = new Array[Byte](byteBuffer.remaining) byteBuffer.get(serializedBytes) @@ -141,7 +141,7 @@ class SocketServerTest { } def processRequest(channel: RequestChannel, request: RequestChannel.Request): Unit = { - val byteBuffer = request.body[AbstractRequest].serialize(request.header) + val byteBuffer = request.body[AbstractRequest].serializeWithHeader(request.header) byteBuffer.rewind() val send = new NetworkSend(request.context.connectionId, byteBuffer) @@ -214,7 +214,7 @@ class SocketServerTest { .setTransactionalId(null)) .build() val emptyHeader = new RequestHeader(ApiKeys.PRODUCE, emptyRequest.version, clientId, correlationId) - val byteBuffer = emptyRequest.serialize(emptyHeader) + val byteBuffer = emptyRequest.serializeWithHeader(emptyHeader) byteBuffer.rewind() val serializedBytes = new Array[Byte](byteBuffer.remaining) @@ -225,7 +225,7 @@ class SocketServerTest { private def apiVersionRequestBytes(clientId: String, version: Short): Array[Byte] = { val request = new ApiVersionsRequest.Builder().build(version) val header = new RequestHeader(ApiKeys.API_VERSIONS, request.version(), clientId, -1) - val buffer = request.serialize(header) + val buffer = request.serializeWithHeader(header) buffer.rewind() val bytes = new Array[Byte](buffer.remaining()) buffer.get(bytes) @@ -378,7 +378,7 @@ class SocketServerTest { val correlationId = 57 val header = new RequestHeader(ApiKeys.VOTE, 0, "", correlationId) val request = new VoteRequest.Builder(new VoteRequestData()).build() - val byteBuffer = request.serialize(header) + val byteBuffer = request.serializeWithHeader(header) byteBuffer.rewind() val socket = connect() @@ -687,7 +687,7 @@ class SocketServerTest { // Mimic a primitive request handler that fetches the request from RequestChannel and place a response with a // throttled channel. val request = receiveRequest(server.dataPlaneRequestChannel) - val byteBuffer = request.body[AbstractRequest].serialize(request.header) + val byteBuffer = request.body[AbstractRequest].serializeWithHeader(request.header) val send = new NetworkSend(request.context.connectionId, byteBuffer) def channelThrottlingCallback(response: RequestChannel.Response): Unit = { server.dataPlaneRequestChannel.sendResponse(response) @@ -967,7 +967,7 @@ class SocketServerTest { .build() val emptyHeader = new RequestHeader(ApiKeys.PRODUCE, emptyRequest.version, clientId, correlationId) - val byteBuffer = emptyRequest.serialize(emptyHeader) + val byteBuffer = emptyRequest.serializeWithHeader(emptyHeader) byteBuffer.rewind() val serializedBytes = new Array[Byte](byteBuffer.remaining) byteBuffer.get(serializedBytes) diff --git a/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala b/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala index cfe799ac9e717..10f73a88ae40d 100644 --- a/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala @@ -24,7 +24,6 @@ import kafka.network.SocketServer import kafka.utils.TestUtils import org.apache.kafka.common.message.CreateTopicsRequestData import org.apache.kafka.common.message.CreateTopicsRequestData._ -import org.apache.kafka.common.protocol.types.Struct import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests._ import org.junit.Assert.{assertEquals, assertFalse, assertNotNull, assertTrue} @@ -151,15 +150,6 @@ abstract class AbstractCreateTopicsRequestTest extends BaseRequestTest { protected def error(error: Errors, errorMessage: Option[String] = None): ApiError = new ApiError(error, errorMessage.orNull) - protected def addPartitionsAndReplicationFactorToFirstTopic(request: CreateTopicsRequest) = { - val struct = request.toStruct - val topics = struct.getArray("create_topic_requests") - val firstTopic = topics(0).asInstanceOf[Struct] - firstTopic.set("num_partitions", 1) - firstTopic.set("replication_factor", 1.toShort) - new CreateTopicsRequest(struct, request.version) - } - protected def validateErrorCreateTopicsRequests(request: CreateTopicsRequest, expectedResponse: Map[String, ApiError], checkErrorMessage: Boolean = true): Unit = { diff --git a/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala index 3a4dd9487d181..d9f64e8e416f7 100644 --- a/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala @@ -63,7 +63,7 @@ class BaseClientQuotaManagerTest { listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): (T, RequestChannel.Request) = { val request = builder.build() - val buffer = request.serialize(new RequestHeader(builder.apiKey, request.version, "", 0)) + val buffer = request.serializeWithHeader(new RequestHeader(builder.apiKey, request.version, "", 0)) val requestChannelMetrics: RequestChannel.Metrics = EasyMock.createNiceMock(classOf[RequestChannel.Metrics]) // read the header from the buffer first so that the body can be read next from the Request constructor diff --git a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala index bf54cdb19157f..a78b37e6a50ba 100644 --- a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala @@ -97,8 +97,7 @@ abstract class BaseRequestTest extends IntegrationTestHarness { val responseBuffer = ByteBuffer.wrap(responseBytes) ResponseHeader.parse(responseBuffer, apiKey.responseHeaderVersion(version)) - val responseStruct = apiKey.parseResponse(version, responseBuffer) - AbstractResponse.parseResponse(apiKey, responseStruct, version) match { + AbstractResponse.parseResponse(apiKey, responseBuffer, version) match { case response: T => response case response => throw new ClassCastException(s"Expected response with type ${classTag.runtimeClass}, but found ${response.getClass}") @@ -111,7 +110,7 @@ abstract class BaseRequestTest extends IntegrationTestHarness { correlationId: Option[Int] = None) (implicit classTag: ClassTag[T], nn: NotNothing[T]): T = { send(request, socket, clientId, correlationId) - receive[T](socket, request.api, request.version) + receive[T](socket, request.apiKey, request.version) } def connectAndReceive[T <: AbstractResponse](request: AbstractRequest, @@ -130,12 +129,12 @@ abstract class BaseRequestTest extends IntegrationTestHarness { socket: Socket, clientId: String = "client-id", correlationId: Option[Int] = None): Unit = { - val header = nextRequestHeader(request.api, request.version, clientId, correlationId) + val header = nextRequestHeader(request.apiKey, request.version, clientId, correlationId) sendWithHeader(request, header, socket) } def sendWithHeader(request: AbstractRequest, header: RequestHeader, socket: Socket): Unit = { - val serializedBytes = request.serialize(header).array + val serializedBytes = request.serializeWithHeader(header).array sendRequest(socket, serializedBytes) } diff --git a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala index 2a3cd0f611c37..2ed64509c459a 100755 --- a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala @@ -139,10 +139,11 @@ class EdgeCaseRequestTest extends KafkaServerTestHarness { .setTimeoutMs(10000) .setTransactionalId(null)) .build() - val byteBuffer = ByteBuffer.allocate(headerBytes.length + request.toStruct.sizeOf) + val bodyBytes = request.serializeBody + val byteBuffer = ByteBuffer.allocate(headerBytes.length + bodyBytes.remaining()) byteBuffer.put(headerBytes) - request.toStruct.writeTo(byteBuffer) - (byteBuffer.array(), request.api.responseHeaderVersion(version)) + byteBuffer.put(bodyBytes) + (byteBuffer.array(), request.apiKey.responseHeaderVersion(version)) } val response = requestAndReceive(serializedBytes) diff --git a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala index 4d9451e2a31b4..b6624d6c2a205 100644 --- a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala @@ -60,8 +60,7 @@ class ForwardingManagerTest { val request = buildRequest(requestHeader, requestBuffer, clientPrincipal) val responseBody = new AlterConfigsResponse(new AlterConfigsResponseData()) - val responseBuffer = responseBody.serialize(ApiKeys.ALTER_CONFIGS, - requestBody.version, requestCorrelationId + 1) + val responseBuffer = responseBody.serializeWithHeader(requestBody.version, requestCorrelationId + 1) Mockito.when(brokerToController.sendRequest( any(classOf[EnvelopeRequest.Builder]), @@ -113,12 +112,12 @@ class ForwardingManagerTest { correlationId: Int ): (RequestHeader, ByteBuffer) = { val header = new RequestHeader( - body.api, + body.apiKey, body.version, "clientId", correlationId ) - val buffer = body.serialize(header) + val buffer = body.serializeWithHeader(header) // Fast-forward buffer to start of the request as `RequestChannel.Request` expects RequestHeader.parse(buffer) diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 5cc7f4bc66974..b89fb2efdb1b3 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -352,7 +352,7 @@ class KafkaApisTest { configResource -> new AlterConfigsRequest.Config( Seq(new AlterConfigsRequest.ConfigEntry("foo", "bar")).asJava)) val alterConfigsRequest = new AlterConfigsRequest.Builder(configs.asJava, false).build(requestHeader.apiVersion) - val serializedRequestData = alterConfigsRequest.serialize(requestHeader) + val serializedRequestData = alterConfigsRequest.serializeWithHeader(requestHeader) val capturedResponse = expectNoThrottling() @@ -375,7 +375,7 @@ class KafkaApisTest { clientId, 0) val leaveGroupRequest = new LeaveGroupRequest.Builder("group", Collections.singletonList(new MemberIdentity())).build(requestHeader.apiVersion) - val serializedRequestData = leaveGroupRequest.serialize(requestHeader) + val serializedRequestData = leaveGroupRequest.serializeWithHeader(requestHeader) val capturedResponse = expectNoThrottling() @@ -1358,7 +1358,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors(1)) + assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) } @Test @@ -1377,7 +1377,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors(1)) + assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) } @Test @@ -1412,7 +1412,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors(1)) + assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) EasyMock.verify(replicaManager) } @@ -1549,7 +1549,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors(1)) + assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) EasyMock.verify(replicaManager) } @@ -2826,8 +2826,8 @@ class KafkaApisTest { ): RequestChannel.Request = { val listenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT) - val requestHeader = new RequestHeader(request.api, request.version, clientId, 0) - val requestBuffer = request.serialize(requestHeader) + val requestHeader = new RequestHeader(request.apiKey, request.version, clientId, 0) + val requestBuffer = request.serializeWithHeader(requestHeader) val requestContext = new RequestContext(requestHeader, "1", InetAddress.getLocalHost, KafkaPrincipal.ANONYMOUS, listenerName, SecurityProtocol.PLAINTEXT, ClientInformation.EMPTY, fromPrivilegedListener) @@ -2837,7 +2837,7 @@ class KafkaApisTest { requestBuffer, new Array[Byte](0), InetAddress.getLocalHost.getAddress - ).build().serialize(envelopeHeader) + ).build().serializeWithHeader(envelopeHeader) val envelopeContext = new RequestContext(envelopeHeader, "1", InetAddress.getLocalHost, KafkaPrincipal.ANONYMOUS, listenerName, SecurityProtocol.PLAINTEXT, ClientInformation.EMPTY, fromPrivilegedListener, principalSerde.asJava) @@ -2868,8 +2868,8 @@ class KafkaApisTest { listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT), fromPrivilegedListener: Boolean = false, requestHeader: Option[RequestHeader] = None): RequestChannel.Request = { - val buffer = request.serialize(requestHeader.getOrElse( - new RequestHeader(request.api, request.version, clientId, 0))) + val buffer = request.serializeWithHeader(requestHeader.getOrElse( + new RequestHeader(request.apiKey, request.version, clientId, 0))) // read the header from the buffer first so that the body can be read next from the Request constructor val header = RequestHeader.parse(buffer) @@ -2881,7 +2881,7 @@ class KafkaApisTest { } private def readResponse(request: AbstractRequest, capturedResponse: Capture[RequestChannel.Response]) = { - val api = request.api + val api = request.apiKey val response = capturedResponse.getValue assertTrue(s"Unexpected response type: ${response.getClass}", response.isInstanceOf[SendResponse]) val sendResponse = response.asInstanceOf[SendResponse] @@ -2891,8 +2891,7 @@ class KafkaApisTest { channel.close() channel.buffer.getInt() // read the size ResponseHeader.parse(channel.buffer, api.responseHeaderVersion(request.version)) - val struct = api.responseSchema(request.version).read(channel.buffer) - AbstractResponse.parseResponse(api, struct, request.version) + AbstractResponse.parseResponse(api, channel.buffer, request.version) } private def expectNoThrottling(): Capture[RequestChannel.Response] = { diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index ccd8904198c7b..aedb5194e2870 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -610,7 +610,7 @@ class RequestQuotaTest extends BaseRequestTest { 0 ) val embedRequestData = new AlterClientQuotasRequest.Builder( - List.empty.asJava, false).build().serialize(requestHeader) + List.empty.asJava, false).build().serializeWithHeader(requestHeader) new EnvelopeRequest.Builder(embedRequestData, new Array[Byte](0), InetAddress.getByName("192.168.1.1").getAddress) @@ -670,11 +670,9 @@ class RequestQuotaTest extends BaseRequestTest { } private def checkRequestThrottleTime(apiKey: ApiKeys): Unit = { - // Request until throttled using client-id with default small quota val clientId = apiKey.toString val client = Client(clientId, apiKey) - val throttled = client.runUntil(_.throttleTimeMs > 0) assertTrue(s"Response not throttled: $client", throttled) diff --git a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala index 6d1f8cafeae18..ca276d982d538 100644 --- a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala @@ -18,11 +18,10 @@ package kafka.server import java.net.Socket import java.util.Collections - import kafka.api.{KafkaSasl, SaslSetup} import kafka.utils.JaasTestUtils import org.apache.kafka.common.message.SaslHandshakeRequestData -import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.requests.{ApiVersionsRequest, ApiVersionsResponse, SaslHandshakeRequest, SaslHandshakeResponse} import org.apache.kafka.common.security.auth.SecurityProtocol import org.junit.Assert._ @@ -91,7 +90,8 @@ class SaslApiVersionsRequestTest extends AbstractApiVersionsRequestTest with Sas } private def sendSaslHandshakeRequestValidateResponse(socket: Socket): Unit = { - val request = new SaslHandshakeRequest(new SaslHandshakeRequestData().setMechanism("PLAIN")) + val request = new SaslHandshakeRequest(new SaslHandshakeRequestData().setMechanism("PLAIN"), + ApiKeys.SASL_HANDSHAKE.latestVersion) val response = sendAndReceive[SaslHandshakeResponse](request, socket) assertEquals(Errors.NONE, response.error) assertEquals(Collections.singletonList("PLAIN"), response.enabledMechanisms) diff --git a/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala b/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala index 02e36413b860a..7e92084086e55 100644 --- a/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala +++ b/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala @@ -50,7 +50,7 @@ class ThrottledChannelExpirationTest { listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): (T, RequestChannel.Request) = { val request = builder.build() - val buffer = request.serialize(new RequestHeader(builder.apiKey, request.version, "", 0)) + val buffer = request.serializeWithHeader(new RequestHeader(builder.apiKey, request.version, "", 0)) val requestChannelMetrics: RequestChannel.Metrics = EasyMock.createNiceMock(classOf[RequestChannel.Metrics]) // read the header from the buffer first so that the body can be read next from the Request constructor diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchRequestBenchmark.java index 5f68a89ef4971..0d0b1e46a86dd 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchRequestBenchmark.java @@ -20,7 +20,6 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.ByteBufferChannel; import org.apache.kafka.common.requests.FetchRequest; @@ -39,6 +38,7 @@ import org.openjdk.jmh.annotations.Warmup; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -67,7 +67,7 @@ public class FetchRequestBenchmark { FetchRequest replicaRequest; - Struct requestStruct; + ByteBuffer requestBuffer; @Setup(Level.Trial) public void setup() { @@ -86,14 +86,13 @@ public void setup() { .build(ApiKeys.FETCH.latestVersion()); this.replicaRequest = FetchRequest.Builder.forReplica(ApiKeys.FETCH.latestVersion(), 1, 0, 0, fetchData) .build(ApiKeys.FETCH.latestVersion()); - this.requestStruct = this.consumerRequest.data().toStruct(ApiKeys.FETCH.latestVersion()); + this.requestBuffer = this.consumerRequest.serializeBody(); } @Benchmark - public short testFetchRequestFromStruct() { - AbstractRequest request = AbstractRequest.parseRequest(ApiKeys.FETCH, ApiKeys.FETCH.latestVersion(), requestStruct); - return request.version(); + public short testFetchRequestFromBuffer() { + return AbstractRequest.parseRequest(ApiKeys.FETCH, ApiKeys.FETCH.latestVersion(), requestBuffer).request.version(); } @Benchmark diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java index 413781fa365d5..4ac4c56d4c18f 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java @@ -200,7 +200,7 @@ public void tearDown() { private RequestChannel.Request buildAllTopicMetadataRequest() { MetadataRequest metadataRequest = MetadataRequest.Builder.allTopics().build(); - ByteBuffer buffer = metadataRequest.serialize(new RequestHeader(metadataRequest.api, + ByteBuffer buffer = metadataRequest.serializeWithHeader(new RequestHeader(metadataRequest.apiKey(), metadataRequest.version(), "", 0)); RequestHeader header = RequestHeader.parse(buffer); diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerRequestBenchmark.java index 58b35f354f92e..22d49551dd5e8 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerRequestBenchmark.java @@ -19,7 +19,6 @@ import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.RecordBatch; @@ -85,9 +84,4 @@ public ProduceResponse constructorErrorResponse() { return REQUEST.getErrorResponse(0, Errors.INVALID_REQUEST.exception()); } - @Benchmark - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public Struct constructorStruct() { - return REQUEST.toStruct(); - } } diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerResponseBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerResponseBenchmark.java index 307e194431818..431880a96dc18 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerResponseBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerResponseBenchmark.java @@ -19,7 +19,6 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ProduceResponse; import org.openjdk.jmh.annotations.Benchmark; @@ -38,8 +37,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.apache.kafka.common.protocol.ApiKeys.PRODUCE; - @State(Scope.Benchmark) @Fork(value = 1) @Warmup(iterations = 5) @@ -79,10 +76,4 @@ private static ProduceResponse response() { public AbstractResponse constructorProduceResponse() { return response(); } - - @Benchmark - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public Struct constructorStruct() { - return RESPONSE.toStruct(PRODUCE.latestVersion()); - } } From 1d84f543678c4c08800bc3ea18c04a9db8adf7e4 Mon Sep 17 00:00:00 2001 From: Kowshik Prakasam Date: Mon, 7 Dec 2020 16:42:19 -0800 Subject: [PATCH 12/14] MINOR: Remove redundant default parameter values in call to LogSegment.open (#9710) Reviewers: Jun Rao --- core/src/main/scala/kafka/log/Log.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala index a490c97a650ae..9677dc4f6c7a7 100644 --- a/core/src/main/scala/kafka/log/Log.scala +++ b/core/src/main/scala/kafka/log/Log.scala @@ -770,9 +770,7 @@ class Log(@volatile private var _dir: File, baseOffset = 0, config, time = time, - fileAlreadyExists = false, - initFileSize = this.initFileSize, - preallocate = false)) + initFileSize = this.initFileSize)) } 0 } @@ -858,7 +856,6 @@ class Log(@volatile private var _dir: File, baseOffset = logStartOffset, config, time = time, - fileAlreadyExists = false, initFileSize = this.initFileSize, preallocate = config.preallocate)) } @@ -2124,7 +2121,6 @@ class Log(@volatile private var _dir: File, baseOffset = newOffset, config = config, time = time, - fileAlreadyExists = false, initFileSize = initFileSize, preallocate = config.preallocate)) updateLogEndOffset(newOffset) From 32adca324218e407cef8eba47a40f6e267076bfb Mon Sep 17 00:00:00 2001 From: bertber <75388552+bertber@users.noreply.github.com> Date: Sat, 5 Dec 2020 02:51:27 +0800 Subject: [PATCH 13/14] MINOR: add a description for calling resetToDatetime's function to resetByDuration Add a request that it will call resetToDatetime of description to resetByDuration of function. --- .../src/main/scala/kafka/tools/StreamsResetter.java | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/core/src/main/scala/kafka/tools/StreamsResetter.java b/core/src/main/scala/kafka/tools/StreamsResetter.java index d356681e11242..adaee6694510d 100644 --- a/core/src/main/scala/kafka/tools/StreamsResetter.java +++ b/core/src/main/scala/kafka/tools/StreamsResetter.java @@ -491,17 +491,8 @@ private void resetByDuration(final Consumer client, final Duration duration) { final Instant now = Instant.now(); final long timestamp = now.minus(duration).toEpochMilli(); - - final Map topicPartitionsAndTimes = new HashMap<>(inputTopicPartitions.size()); - for (final TopicPartition topicPartition : inputTopicPartitions) { - topicPartitionsAndTimes.put(topicPartition, timestamp); - } - - final Map topicPartitionsAndOffset = client.offsetsForTimes(topicPartitionsAndTimes); - - for (final TopicPartition topicPartition : inputTopicPartitions) { - client.seek(topicPartition, topicPartitionsAndOffset.get(topicPartition).offset()); - } + + resetToDatetime(client, inputTopicPartitions, timestamp); } private void resetToDatetime(final Consumer client, From 59a880349e4bd5b93ceaac713572979f2fb3e2a1 Mon Sep 17 00:00:00 2001 From: bertber <75388552+bertber@users.noreply.github.com> Date: Wed, 9 Dec 2020 22:25:00 +0800 Subject: [PATCH 14/14] remove duplicate code from resetByDuration remove local variables from resetByDuration --- core/src/main/scala/kafka/tools/StreamsResetter.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core/src/main/scala/kafka/tools/StreamsResetter.java b/core/src/main/scala/kafka/tools/StreamsResetter.java index adaee6694510d..f72a3b6905400 100644 --- a/core/src/main/scala/kafka/tools/StreamsResetter.java +++ b/core/src/main/scala/kafka/tools/StreamsResetter.java @@ -489,10 +489,7 @@ private Map getTopicPartitionOffsetFromResetPlan(final Str private void resetByDuration(final Consumer client, final Set inputTopicPartitions, final Duration duration) { - final Instant now = Instant.now(); - final long timestamp = now.minus(duration).toEpochMilli(); - - resetToDatetime(client, inputTopicPartitions, timestamp); + resetToDatetime(client, inputTopicPartitions, Instant.now().minus(duration).toEpochMilli()); } private void resetToDatetime(final Consumer client,