Skip to content

Kafka 17792 header parsing times out processing and using large quantities of memory if the string looks like a number #17510

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gharris1727 merged 7 commits into from
Jan 27, 2025
Merged

Kafka 17792 header parsing times out processing and using large quantities of memory if the string looks like a number #17510

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
07a4bd3
KAFKA-17792 header parsing times out processing and using large quant…
msillence Oct 16, 2024
91639cf
fix code and tests
msillence Jan 24, 2025
bb00f70
KAFKA-17792 header parsing times out processing and using large quant…
msillence Jan 24, 2025
37fae21
still convert to int
msillence Jan 24, 2025
5225155
spotless
msillence Jan 24, 2025
b407e58
add missing new line
msillence Jan 24, 2025
a4d376b
spotless
msillence Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 14 additions & 6 deletions connect/api/src/main/java/org/apache/kafka/connect/data/Values.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import java.io.Serializable;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
import java.nio.ByteBuffer;
import java.text.CharacterIterator;
import java.text.DateFormat;
Expand Down Expand Up @@ -1012,6 +1011,7 @@ private SchemaAndValue parseMultipleTokensAsTemporal(String token) {
return parseAsTemporal(token);
}


private static SchemaAndValue parseAsNumber(String token) {
// Try to parse as a number ...
BigDecimal decimal = new BigDecimal(token);
Expand All @@ -1032,12 +1032,20 @@ private static SchemaAndValue parseAsNumber(String token) {
}
}

private static boolean isWholeNumber(BigDecimal bd) {
return bd.signum() == 0 || bd.scale() <= 0 || bd.stripTrailingZeros().scale() <= 0;
}

private static final BigDecimal BIGGER_THAN_LONG = new BigDecimal("1e19");

private static SchemaAndValue parseAsExactDecimal(BigDecimal decimal) {
BigDecimal ceil = decimal.setScale(0, RoundingMode.CEILING);
BigDecimal floor = decimal.setScale(0, RoundingMode.FLOOR);
if (ceil.equals(floor)) {
BigInteger num = ceil.toBigIntegerExact();
if (ceil.precision() >= 19 && (num.compareTo(LONG_MIN) < 0 || num.compareTo(LONG_MAX) > 0)) {
BigDecimal abs = decimal.abs();
Comment thread
gharris1727 marked this conversation as resolved.
if (abs.compareTo(BIGGER_THAN_LONG) > 0 || (abs.compareTo(BigDecimal.ONE) < 0 && abs.compareTo(BigDecimal.ZERO) != 0)) {
return null;
}
if (isWholeNumber(decimal)) {
BigInteger num = decimal.toBigIntegerExact();
if (num.compareTo(LONG_MIN) < 0 || num.compareTo(LONG_MAX) > 0) {
return null;
}
long integral = num.longValue();
Expand Down
11 changes: 11 additions & 0 deletions connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1181,6 +1181,17 @@ public void shouldParseFractionalPartsAsIntegerWhenNoFractionalPart() {
assertEquals(new SchemaAndValue(Schema.FLOAT32_SCHEMA, 66000.0008f), Values.parseString("66000.0008"));
}

@Test
Comment thread
gharris1727 marked this conversation as resolved.
public void avoidCpuAndMemoryIssuesConvertingExtremeBigDecimals() {
String parsingBig = "1e+100000000"; // new BigDecimal().setScale(0, RoundingMode.FLOOR) takes around two minutes and uses 3GB;
BigDecimal valueBig = new BigDecimal(parsingBig);
assertEquals(new SchemaAndValue(Decimal.schema(-100000000), valueBig), Values.parseString(parsingBig), "parsing number that's too big");

String parsingSmall = "1e-100000000";
BigDecimal valueSmall = new BigDecimal(parsingSmall);
assertEquals(new SchemaAndValue(Schema.FLOAT32_SCHEMA, (float) valueSmall.doubleValue()), Values.parseString(parsingSmall), "parsing number that's too big, strictly this should return a bigdecimal");
}

protected void assertParsed(String input) {
assertParsed(input, input);
}
Expand Down