Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -232,9 +232,10 @@ private byte[] getInternal(final Bytes key) {
@Override
public KeyValueIterator<Bytes, byte[]> range(final Bytes from,
final Bytes to) {
// Make sure this is a valid query
if (from.compareTo(to) > 0) {
LOG.warn("Returning empty iterator for range query with invalid range: keyFrom > keyTo.");
LOG.warn("Returning empty iterator for fetch with invalid key range: from > to. "
+ "This may be due to serdes that don't preserve ordering when lexicographically comparing the serialized bytes. " +
"Note that the built-in numerical serdes do not follow this for negative numbers");
return KeyValueIterators.emptyIterator();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,9 +157,10 @@ public KeyValueIterator<Windowed<Bytes>, byte[]> findSessions(final Bytes keyFro
final Bytes keyTo,
final long earliestSessionEndTime,
final long latestSessionStartTime) {
// Make sure this is a valid query
if (keyFrom.compareTo(keyTo) > 0) {
LOG.warn("Returning empty iterator for findSessions call with invalid range: keyFrom > keyTo.");
LOG.warn("Returning empty iterator for fetch with invalid key range: from > to. "
+ "This may be due to serdes that don't preserve ordering when lexicographically comparing the serialized bytes. " +
"Note that the built-in numerical serdes do not follow this for negative numbers");
return KeyValueIterators.emptyIterator();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,10 @@ public KeyValueIterator<Windowed<Bytes>, byte[]> fetch(final Bytes from,
final Bytes to,
final long timeFrom,
final long timeTo) {
// Make sure this is a valid query
if (from.compareTo(to) > 0) {
LOG.warn("Returning empty iterator for fetch with invalid range: keyFrom > keyTo.");
LOG.warn("Returning empty iterator for fetch with invalid key range: from > to. "
+ "This may be due to serdes that don't preserve ordering when lexicographically comparing the serialized bytes. " +
"Note that the built-in numerical serdes do not follow this for negative numbers");
return KeyValueIterators.emptyIterator();

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto here and below

}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,11 @@ public byte[] delete(final Bytes key) {

@Override
public KeyValueIterator<Bytes, byte[]> range(final Bytes from, final Bytes to) {
// Make sure this is a valid query

if (from.compareTo(to) > 0) {
LOG.warn("Returning empty iterator for range query with invalid range: keyFrom > keyTo.");
LOG.warn("Returning empty iterator for fetch with invalid key range: from > to. "
+ "This may be due to serdes that don't preserve ordering when lexicographically comparing the serialized bytes. " +
"Note that the built-in numerical serdes do not follow this for negative numbers");
return KeyValueIterators.emptyIterator();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,9 +185,10 @@ public KeyValueIterator<Windowed<Bytes>, byte[]> fetch(final Bytes from,
final long timeTo) {
removeExpiredSegments();

// Make sure this is a valid query
if (from.compareTo(to) > 0) {
LOG.debug("Returning empty iterator for fetch with invalid range: keyFrom > keyTo.");
LOG.warn("Returning empty iterator for fetch with invalid key range: from > to. "
+ "This may be due to serdes that don't preserve ordering when lexicographically comparing the serialized bytes. " +
"Note that the built-in numerical serdes do not follow this for negative numbers");
return KeyValueIterators.emptyIterator();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,27 @@
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class MemoryNavigableLRUCache extends MemoryLRUCache {

private static final Logger LOG = LoggerFactory.getLogger(MemoryNavigableLRUCache.class);

public MemoryNavigableLRUCache(final String name, final int maxCacheSize) {
super(name, maxCacheSize);
}

@Override
public KeyValueIterator<Bytes, byte[]> range(final Bytes from, final Bytes to) {

if (from.compareTo(to) > 0) {
LOG.warn("Returning empty iterator for fetch with invalid key range: from > to. "
+ "This may be due to serdes that don't preserve ordering when lexicographically comparing the serialized bytes. " +
"Note that the built-in numerical serdes do not follow this for negative numbers");
return KeyValueIterators.emptyIterator();
}

final TreeMap<Bytes, byte[]> treeMap = toTreeMap();
return new DelegatingPeekingKeyValueIterator<>(name(),
new MemoryNavigableLRUCache.CacheIterator(treeMap.navigableKeySet()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.kafka.common.serialization.StringSerializer;
import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.processor.ProcessorContext;
import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.streams.state.KeyValueStore;
import org.apache.kafka.streams.state.KeyValueStoreTestDriver;
Expand All @@ -38,6 +39,7 @@
import java.util.List;
import java.util.Map;

import static org.hamcrest.CoreMatchers.hasItem;
import static org.hamcrest.core.IsEqual.equalTo;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
Expand Down Expand Up @@ -389,4 +391,17 @@ public void shouldNotThrowConcurrentModificationException() {

assertEquals(new KeyValue<>(0, "zero"), results.next());
}

@Test
public void shouldNotThrowInvalidRangeExceptionWithNegativeFromKey() {
LogCaptureAppender.setClassLoggerToDebug(InMemoryWindowStore.class);
final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();

store.range(-1, 1);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can use org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender to assert the correct log message

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah thanks, will add to tests

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add a check to verify that the returned iterator is empty. Something along the lines of assertThat(iterator.hasNext(), is(false))?

Could you also add a test for a range query where the start key is equal to the end key? Such a unit test ensures correct behaviour for this special case.

nit: I would rename the test to shouldReturnEmptyIteratorForRangeQueryWithInvalidKeyRange. Correct me, if I am wrong, but I think the empty iterator and the invalid key range are the points here, not the negative starting key. I would even change the range from (-1, 1) to (5, 3). It took me a bit to understand why (-1, 1) is an invalid range.

These comments apply also to the unit tests below.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ack Re: verify returned iterator is empty, add unit tests for equal start/end keys

Regarding your third point, this patch is mostly aimed at the bug in [https://issues.apache.org/jira/browse/KAFKA-8159]

which went undiscovered for a while because there were no tests of range queries with a negative key. I actually think it's fair to say we make no guarantees about what will happen if your app makes an invalid query; however we definitely shouldn't crash on what appears to be a valid query range (ie [-1,1]), which is the key point here

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fair enough


final List<String> messages = appender.getMessages();
assertThat(messages, hasItem("Returning empty iterator for fetch with invalid key range: from > to. "
+ "This may be due to serdes that don't preserve ordering when lexicographically comparing the serialized bytes. "
+ "Note that the built-in numerical serdes do not follow this for negative numbers"));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.apache.kafka.streams.state.internals;

import org.apache.kafka.common.metrics.Metrics;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.utils.Bytes;
import org.apache.kafka.common.utils.LogContext;
Expand All @@ -28,6 +29,7 @@
import org.apache.kafka.streams.kstream.internals.SessionWindow;
import org.apache.kafka.streams.processor.internals.MockStreamsMetrics;
import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.test.InternalMockProcessorContext;
import org.apache.kafka.test.TestUtils;
Expand All @@ -50,6 +52,7 @@
import static org.apache.kafka.test.StreamsTestUtils.toList;
import static org.apache.kafka.test.StreamsTestUtils.verifyKeyValueList;
import static org.apache.kafka.test.StreamsTestUtils.verifyWindowedKeyValue;
import static org.hamcrest.CoreMatchers.hasItem;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertArrayEquals;
Expand Down Expand Up @@ -412,6 +415,21 @@ public void shouldThrowNullPointerExceptionOnPutNullKey() {
cachingStore.put(null, "1".getBytes());
}

@Test
public void shouldNotThrowInvalidRangeExceptionWithNegativeFromKey() {
LogCaptureAppender.setClassLoggerToDebug(InMemoryWindowStore.class);
final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();

final Bytes keyFrom = Bytes.wrap(Serdes.Integer().serializer().serialize("", -1));
final Bytes keyTo = Bytes.wrap(Serdes.Integer().serializer().serialize("", 1));
cachingStore.findSessions(keyFrom, keyTo, 0L, 10L);

final List<String> messages = appender.getMessages();
assertThat(messages, hasItem("Returning empty iterator for fetch with invalid key range: from > to. "
+ "This may be due to serdes that don't preserve ordering when lexicographically comparing the serialized bytes. "
+ "Note that the built-in numerical serdes do not follow this for negative numbers"));
}

private List<KeyValue<Windowed<Bytes>, byte[]>> addSessionsUntilOverflow(final String... sessionIds) {
final Random random = new Random();
final List<KeyValue<Windowed<Bytes>, byte[]>> results = new ArrayList<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.kafka.streams.processor.ProcessorContext;
import org.apache.kafka.streams.processor.internals.MockStreamsMetrics;
import org.apache.kafka.streams.processor.internals.ProcessorRecordContext;
import org.apache.kafka.streams.processor.internals.testutil.LogCaptureAppender;
import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.streams.state.StoreBuilder;
import org.apache.kafka.streams.state.Stores;
Expand All @@ -61,6 +62,7 @@
import static org.apache.kafka.test.StreamsTestUtils.verifyKeyValueList;
import static org.apache.kafka.test.StreamsTestUtils.verifyWindowedKeyValue;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.hasItem;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
Expand Down Expand Up @@ -568,6 +570,21 @@ public void shouldThrowNullPointerExceptionOnRangeNullToKey() {
cachingStore.fetch(bytesKey("anyFrom"), null, ofEpochMilli(1L), ofEpochMilli(2L));
}

@Test
public void shouldNotThrowInvalidRangeExceptionWithNegativeFromKey() {
LogCaptureAppender.setClassLoggerToDebug(InMemoryWindowStore.class);
final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();

final Bytes keyFrom = Bytes.wrap(Serdes.Integer().serializer().serialize("", -1));
final Bytes keyTo = Bytes.wrap(Serdes.Integer().serializer().serialize("", 1));
cachingStore.fetch(keyFrom, keyTo, 0L, 10L);

final List<String> messages = appender.getMessages();
assertThat(messages, hasItem("Returning empty iterator for fetch with invalid key range: from > to. "
+ "This may be due to serdes that don't preserve ordering when lexicographically comparing the serialized bytes. "
+ "Note that the built-in numerical serdes do not follow this for negative numbers"));
}

private static KeyValue<Windowed<Bytes>, byte[]> windowedPair(final String key, final String value, final long timestamp) {
return KeyValue.pair(
new Windowed<>(bytesKey(key), new TimeWindow(timestamp, timestamp + WINDOW_SIZE)),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -579,4 +579,19 @@ public void shouldNotThrowExceptionWhenFetchRangeIsExpired() {

assertFalse(iterator.hasNext());
}

@Test
public void shouldNotThrowInvalidRangeExceptionWithNegativeFromKey() {
windowStore = createInMemoryWindowStore(context, false);

LogCaptureAppender.setClassLoggerToDebug(InMemoryWindowStore.class);
final LogCaptureAppender appender = LogCaptureAppender.createAndRegister();

windowStore.fetch(-1, 1, 0L, 10L);

final List<String> messages = appender.getMessages();
assertThat(messages, hasItem("Returning empty iterator for fetch with invalid key range: from > to. "
+ "This may be due to serdes that don't preserve ordering when lexicographically comparing the serialized bytes. "
+ "Note that the built-in numerical serdes do not follow this for negative numbers"));
}
}