Skip to content

KAFKA-14659 source-record-write-[rate|total] metrics include filtered records #13193

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
C0urante merged 2 commits into from
Feb 28, 2023
Merged

KAFKA-14659 source-record-write-[rate|total] metrics include filtered records #13193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
07d4f4d
KAFKA-14659 Exclude filtered records from source-record-write-[rate|t…
hgeraldino Jan 31, 2023
c2523da
PR feedback
hgeraldino Feb 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,7 @@ subprojects {
"**/KafkaConfigBackingStoreTest.*",
"**/KafkaBasedLogTest.*", "**/StandaloneHerderTest.*",
"**/WorkerSinkTaskTest.*", "**/WorkerSinkTaskThreadedTest.*",
"**/WorkerSourceTaskTest.*", "**/AbstractWorkerSourceTaskTest.*"
"**/WorkerSourceTaskTest.*"
])
}

Expand Down
9 changes: 6 additions & 3 deletions connect/runtime/src/main/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,8 @@ static class SourceRecordWriteCounter {
private final int batchSize;
private boolean completed = false;
private int counter;
private int skipped; // Keeps track of filtered records

public SourceRecordWriteCounter(int batchSize, SourceTaskMetricsGroup metricsGroup) {
assert batchSize > 0;
assert metricsGroup != null;
Expand All @@ -586,6 +588,7 @@ public SourceRecordWriteCounter(int batchSize, SourceTaskMetricsGroup metricsGro
this.metricsGroup = metricsGroup;
}
public void skipRecord() {
skipped += 1;
if (counter > 0 && --counter == 0) {
finishedAllWrites();
}
Expand All @@ -600,7 +603,7 @@ public void retryRemaining() {
}
private void finishedAllWrites() {
if (!completed) {
metricsGroup.recordWrite(batchSize - counter);
metricsGroup.recordWrite(batchSize - counter, skipped);
completed = true;
}
}
Expand Down Expand Up @@ -652,8 +655,8 @@ void recordPoll(int batchSize, long duration) {
sourceRecordActiveCount.record(activeRecordCount);
}

void recordWrite(int recordCount) {
sourceRecordWrite.record(recordCount);
void recordWrite(int recordCount, int skippedCount) {
sourceRecordWrite.record(recordCount - skippedCount);
activeRecordCount -= recordCount;
activeRecordCount = Math.max(0, activeRecordCount);
sourceRecordActiveCount.record(activeRecordCount);
Expand Down
13 changes: 6 additions & 7 deletions connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectMetricsRegistry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,15 +179,14 @@ public ConnectMetricsRegistry(Set<String> tags) {
"belonging to the named source connector in this worker.",
sourceTaskTags);
sourceRecordWriteRate = createTemplate("source-record-write-rate", SOURCE_TASK_GROUP_NAME,
"The average per-second number of records output from the transformations and written" +
" to Kafka for this task belonging to the named source connector in this worker. This" +
" is after transformations are applied and excludes any records filtered out by the " +
"transformations.",
"The average per-second number of records written to Kafka for this task belonging to the " +
"named source connector in this worker, since the task was last restarted. This is after " +
"transformations are applied, and excludes any records filtered out by the transformations.",
sourceTaskTags);
sourceRecordWriteTotal = createTemplate("source-record-write-total", SOURCE_TASK_GROUP_NAME,
"The number of records output from the transformations and written to Kafka for this" +
" task belonging to the named source connector in this worker, since the task was " +
"last restarted.",
"The number of records output written to Kafka for this task belonging to the " +
"named source connector in this worker, since the task was last restarted. This is after " +
"transformations are applied, and excludes any records filtered out by the transformations.",
sourceTaskTags);
sourceRecordPollBatchTimeMax = createTemplate("poll-batch-max-time-ms", SOURCE_TASK_GROUP_NAME,
"The maximum time in milliseconds taken by this task to poll for a batch of " +
Expand Down
12 changes: 6 additions & 6 deletions .../runtime/src/test/java/org/apache/kafka/connect/runtime/AbstractWorkerSourceTaskTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,18 +192,18 @@ public void testMetricsGroup() {
AbstractWorkerSourceTask.SourceTaskMetricsGroup group1 = new AbstractWorkerSourceTask.SourceTaskMetricsGroup(taskId1, metrics);
for (int i = 0; i != 10; ++i) {
group.recordPoll(100, 1000 + i * 100);
group.recordWrite(10);
group.recordWrite(10, 2);
}
for (int i = 0; i != 20; ++i) {
group1.recordPoll(100, 1000 + i * 100);
group1.recordWrite(10);
group1.recordWrite(10, 4);
}
assertEquals(1900.0, metrics.currentMetricValueAsDouble(group.metricGroup(), "poll-batch-max-time-ms"), 0.001d);
assertEquals(1450.0, metrics.currentMetricValueAsDouble(group.metricGroup(), "poll-batch-avg-time-ms"), 0.001d);
assertEquals(33.333, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-poll-rate"), 0.001d);
assertEquals(1000, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-poll-total"), 0.001d);
assertEquals(3.3333, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-write-rate"), 0.001d);
assertEquals(100, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-write-total"), 0.001d);
assertEquals(2.666, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-write-rate"), 0.001d);
assertEquals(80, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-write-total"), 0.001d);
assertEquals(900.0, metrics.currentMetricValueAsDouble(group.metricGroup(), "source-record-active-count"), 0.001d);

// Close the group
Expand All @@ -227,8 +227,8 @@ public void testMetricsGroup() {
assertEquals(1950.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "poll-batch-avg-time-ms"), 0.001d);
assertEquals(66.667, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-poll-rate"), 0.001d);
assertEquals(2000, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-poll-total"), 0.001d);
assertEquals(6.667, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-write-rate"), 0.001d);
assertEquals(200, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-write-total"), 0.001d);
assertEquals(4.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-write-rate"), 0.001d);
assertEquals(120, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-write-total"), 0.001d);
assertEquals(1800.0, metrics.currentMetricValueAsDouble(group1.metricGroup(), "source-record-active-count"), 0.001d);
}

Expand Down