Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -644,7 +644,7 @@ void runOnce() {
if (records != null && !records.isEmpty()) {
pollSensor.record(pollLatency, now);
pollRecordsSensor.record(records.count(), now);
addRecordsToTasks(records);
taskManager.addRecordsToTasks(records);
}

// Shutdown hook could potentially be triggered and transit the thread state to PENDING_SHUTDOWN during #pollRequests().
Expand Down Expand Up @@ -819,25 +819,6 @@ private void addToResetList(final TopicPartition partition, final Set<TopicParti
partitions.add(partition);
}

/**
* Take records and add them to each respective task
*
* @param records Records, can be null
*/
private void addRecordsToTasks(final ConsumerRecords<byte[], byte[]> records) {
for (final TopicPartition partition : records.partitions()) {
final Task task = taskManager.taskForInputPartition(partition);

if (task == null) {
log.error("Unable to locate active task for received-record partition {}. Current tasks: {}",
partition, taskManager.toString(">"));
throw new NullPointerException("Task was unexpectedly missing for partition " + partition);
}

task.addRecords(partition, records.records(partition));
}
}

/**
* Try to commit all active tasks owned by this thread.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.kafka.clients.admin.RecordsToDelete;
import org.apache.kafka.clients.consumer.CommitFailedException;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.Metric;
Expand Down Expand Up @@ -86,7 +87,7 @@ public class TaskManager {
private boolean rebalanceInProgress = false; // if we are in the middle of a rebalance, it is not safe to commit

// includes assigned & initialized tasks and unassigned tasks we locked temporarily during rebalance
private Set<TaskId> lockedTaskDirectories = new HashSet<>();
private final Set<TaskId> lockedTaskDirectories = new HashSet<>();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


TaskManager(final ChangelogReader changelogReader,
final UUID processId,
Expand Down Expand Up @@ -743,10 +744,6 @@ Set<TaskId> standbyTaskIds() {
.collect(Collectors.toSet());
}

Task taskForInputPartition(final TopicPartition partition) {
return partitionToTask.get(partition);
}

Map<TaskId, Task> tasks() {
// not bothering with an unmodifiable map, since the tasks themselves are mutable, but
// if any outside code modifies the map or the tasks, it would be a severe transgression.
Expand Down Expand Up @@ -778,6 +775,25 @@ int commitAll() {
return commit(tasks.values());
}

/**
* Take records and add them to each respective task
*
* @param records Records, can be null
*/
void addRecordsToTasks(final ConsumerRecords<byte[], byte[]> records) {
for (final TopicPartition partition : records.partitions()) {
final Task task = partitionToTask.get(partition);

if (task == null) {
log.error("Unable to locate active task for received-record partition {}. Current tasks: {}",
partition, toString(">"));
throw new NullPointerException("Task was unexpectedly missing for partition " + partition);
}

task.addRecords(partition, records.records(partition));
}
}

/**
* @throws TaskMigratedException if committing offsets failed (non-EOS)
* or if the task producer got fenced (EOS)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -694,7 +694,6 @@ public void shouldUpdateInputPartitionsAfterRebalance() {
assertThat(taskManager.tryToCompleteRestoration(), is(true));
assertThat(task00.state(), is(Task.State.RUNNING));
assertEquals(newPartitionsSet, task00.inputPartitions());
assertEquals(task00, taskManager.taskForInputPartition(t1p1));
verify(activeTaskCreator, consumer, changeLogReader);
}

Expand Down