Skip to content

Optimize null representation in encoded VariableBlockWidthBlock #15760

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
raunaqmorarka merged 1 commit into from
Jan 24, 2023
Merged

Optimize null representation in encoded VariableBlockWidthBlock #15760

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions core/trino-main/src/test/java/io/trino/execution/buffer/TestPagesSerde.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,14 +200,14 @@ public void testVarcharSerializedSize()
// empty page
Page page = new Page(builder.build());
int pageSize = serializedSize(ImmutableList.of(VARCHAR), page);
assertEquals(pageSize, 44);
assertEquals(pageSize, 48);

// page with one value
VARCHAR.writeString(builder, "alice");
pageSize = 44; // Now we have moved to the normal block implementation so the page size overhead is 44
page = new Page(builder.build());
int firstValueSize = serializedSize(ImmutableList.of(VARCHAR), page) - pageSize;
assertEquals(firstValueSize, 4 + 5); // length + "alice"
assertEquals(firstValueSize, 8 + 5); // length + nonNullsCount + "alice"

// page with two values
VARCHAR.writeString(builder, "bob");
Expand Down
26 changes: 23 additions & 3 deletions core/trino-spi/src/main/java/io/trino/spi/block/VariableWidthBlockEncoding.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,22 @@ public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceO
int positionCount = variableWidthBlock.getPositionCount();
sliceOutput.appendInt(positionCount);

// offsets
// lengths
int[] lengths = new int[positionCount];
int totalLength = 0;
int nonNullsCount = 0;

for (int position = 0; position < positionCount; position++) {
int length = variableWidthBlock.getSliceLength(position);
totalLength += length;
sliceOutput.appendInt(totalLength);
lengths[nonNullsCount] = length;
nonNullsCount += variableWidthBlock.isNull(position) ? 0 : 1;
}

sliceOutput
.appendInt(nonNullsCount)
.writeBytes(Slices.wrappedIntArray(lengths, 0, nonNullsCount));

encodeNullsAsBits(sliceOutput, variableWidthBlock);

sliceOutput
Expand All @@ -61,16 +69,28 @@ public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceO
public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput)
{
int positionCount = sliceInput.readInt();
int nonNullsCount = sliceInput.readInt();

int[] offsets = new int[positionCount + 1];
sliceInput.readBytes(Slices.wrappedIntArray(offsets), SIZE_OF_INT, positionCount * SIZE_OF_INT);
int[] lengths = new int[nonNullsCount];
sliceInput.readBytes(Slices.wrappedIntArray(lengths), 0, nonNullsCount * SIZE_OF_INT);

boolean[] valueIsNull = decodeNullBits(sliceInput, positionCount).orElse(null);

int blockSize = sliceInput.readInt();
Slice slice = Slices.allocate(blockSize);
sliceInput.readBytes(slice);

int nonNullPosition = 0;
int offset = 0;

for (int i = 0; i < positionCount; i++) {
if (valueIsNull == null || !valueIsNull[i]) {
offset += lengths[nonNullPosition];
nonNullPosition++;
}
offsets[i + 1] = offset;
}
return new VariableWidthBlock(0, positionCount, slice, offsets, valueIsNull);
}
}
6 changes: 3 additions & 3 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/optimizer/TestHivePlans.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ public void testPruneSimplePartitionLikeFilter()
assertDistributedPlan(
"SELECT * FROM table_str_partitioned WHERE str_part LIKE 't%'",
output(
filter("\"$like\"(STR_PART, \"$literal$\"(from_base64('DgAAAFZBUklBQkxFX1dJRFRIAQAAAAcAAAAABwAAAAIAAAB0JQA=')))",
filter("\"$like\"(STR_PART, \"$literal$\"(from_base64('DgAAAFZBUklBQkxFX1dJRFRIAQAAAAEAAAAHAAAAAAcAAAACAAAAdCUA')))",
tableScan("table_str_partitioned", Map.of("INT_COL", "int_col", "STR_PART", "str_part")))));
}

Expand All @@ -151,12 +151,12 @@ public void testPrunePartitionLikeFilter()
.left(
exchange(REMOTE, REPARTITION,
project(
filter("\"$like\"(L_STR_PART, \"$literal$\"(from_base64('DgAAAFZBUklBQkxFX1dJRFRIAQAAAAcAAAAABwAAAAIAAAB0JQA=')))",
filter("\"$like\"(L_STR_PART, \"$literal$\"(from_base64('DgAAAFZBUklBQkxFX1dJRFRIAQAAAAEAAAAHAAAAAAcAAAACAAAAdCUA')))",
tableScan("table_str_partitioned", Map.of("L_INT_COL", "int_col", "L_STR_PART", "str_part"))))))
.right(exchange(LOCAL,
exchange(REMOTE, REPARTITION,
project(
filter("R_STR_COL IN ('three', CAST('two' AS varchar(5))) AND \"$like\"(R_STR_COL, \"$literal$\"(from_base64('DgAAAFZBUklBQkxFX1dJRFRIAQAAAAcAAAAABwAAAAIAAAB0JQA=')))",
filter("R_STR_COL IN ('three', CAST('two' AS varchar(5))) AND \"$like\"(R_STR_COL, \"$literal$\"(from_base64('DgAAAFZBUklBQkxFX1dJRFRIAQAAAAEAAAAHAAAAAAcAAAACAAAAdCUA')))",
tableScan("table_unpartitioned", Map.of("R_STR_COL", "str_col", "R_INT_COL", "int_col"))))))))));
}

Expand Down
2 changes: 1 addition & 1 deletion ...lttolerant-tests/src/test/java/io/trino/faulttolerant/BaseFaultTolerantExecutionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public void testExecutePreferredWritePartitioningSkewMitigation()
{
@Language("SQL") String createTableSql = """
CREATE TABLE test_execute_skew_mitigation WITH (%s = ARRAY['returnflag']) AS
SELECT orderkey, partkey, suppkey, linenumber, quantity, extendedprice, discount, tax, linestatus, shipdate, commitdate, receiptdate, shipinstruct, shipmode, comment, returnflag
SELECT orderkey, partkey, suppkey, linenumber, quantity, extendedprice, discount, tax, linestatus, shipdate, commitdate, receiptdate, shipinstruct, shipmode, returnflag
FROM tpch.sf1.lineitem
WHERE returnflag = 'N'
LIMIT 1000000""".formatted(partitioningTablePropertyName);
Expand Down