diff --git a/src/main/java/com/facebook/presto/BlockBuilder.java b/src/main/java/com/facebook/presto/BlockBuilder.java index 9bbc726bfbf82..9d7b1bc0ffa19 100644 --- a/src/main/java/com/facebook/presto/BlockBuilder.java +++ b/src/main/java/com/facebook/presto/BlockBuilder.java @@ -5,12 +5,8 @@ import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; -import java.util.ArrayList; -import java.util.List; - -import static com.facebook.presto.TupleInfo.Type.FIXED_INT_64; -import static com.facebook.presto.TupleInfo.Type.VARIABLE_BINARY; -import static com.google.common.base.Preconditions.*; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; public class BlockBuilder { @@ -22,10 +18,7 @@ public class BlockBuilder private final DynamicSliceOutput sliceOutput; private int count; - private final int fixedPartSize; - - private int currentField; - private final List variableLengthFields; + private TupleInfo.Builder tupleBuilder; public BlockBuilder(long startPosition, TupleInfo tupleInfo) { @@ -42,23 +35,7 @@ public BlockBuilder(long startPosition, TupleInfo tupleInfo, DataSize blockSize) maxBlockSize = (int) blockSize.toBytes(); sliceOutput = new DynamicSliceOutput((int) blockSize.toBytes()); - int fixedPartSize = 0; - int variableLengthFieldCount = 0; - for (TupleInfo.Type type : tupleInfo.getTypes()) { - if (!type.isFixedSize()) { - if (variableLengthFieldCount > 0) { - fixedPartSize += SizeOf.SIZE_OF_SHORT; - } - variableLengthFieldCount++; - } - else { - fixedPartSize += type.getSize(); - } - } - fixedPartSize += SizeOf.SIZE_OF_SHORT; // total tuple size - - this.fixedPartSize = fixedPartSize; - variableLengthFields = new ArrayList<>(variableLengthFieldCount); + tupleBuilder = tupleInfo.builder(sliceOutput); } public boolean isFull() @@ -70,90 +47,42 @@ public void append(long value) { flushTupleIfNecessary(); - checkState(tupleInfo.getTypes().get(currentField) == FIXED_INT_64, "Expected FIXED_INT_64"); - - sliceOutput.writeLong(value); - currentField++; + tupleBuilder.append(value); } public void append(byte[] value) { flushTupleIfNecessary(); - checkState(tupleInfo.getTypes().get(currentField) == VARIABLE_BINARY, "Expected VARIABLE_BINARY"); - - variableLengthFields.add(Slices.wrappedBuffer(value)); - currentField++; + tupleBuilder.append(Slices.wrappedBuffer(value)); } public void append(Slice value) { flushTupleIfNecessary(); - checkState(tupleInfo.getTypes().get(currentField) == VARIABLE_BINARY, "Expected VARIABLE_BINARY"); - - variableLengthFields.add(value); - currentField++; + tupleBuilder.append(value); } public void append(Tuple tuple) { - // TODO: optimization - single copy of block of fixed length fields - - int index = 0; - for (TupleInfo.Type type : tuple.getTupleInfo().getTypes()) { - switch (type) { - case FIXED_INT_64: - append(tuple.getLong(index)); - break; - case VARIABLE_BINARY: - append(tuple.getSlice(index)); - break; - default: - throw new IllegalStateException("Type not yet supported: " + type); - } - - index++; - } + flushTupleIfNecessary(); + + tupleBuilder.append(tuple); } private void flushTupleIfNecessary() { - if (currentField < tupleInfo.getTypes().size()) { - return; - } - - // write offsets - boolean isFirst = true; - int offset = fixedPartSize; - for (Slice field : variableLengthFields) { - if (!isFirst) { - sliceOutput.writeShort(offset); - } - offset += field.length(); - isFirst = false; - } - - if (!variableLengthFields.isEmpty()) { - sliceOutput.writeShort(offset); // total tuple length + if (tupleBuilder.isComplete()) { + tupleBuilder.finish(); + count++; } - - // write values - for (Slice field : variableLengthFields) { - sliceOutput.writeBytes(field); - } - - count++; - currentField = 0; - variableLengthFields.clear(); } public ValueBlock build() { flushTupleIfNecessary(); - Preconditions.checkState(currentField == 0, "Last tuple is incomplete"); - if (count == 0) { return EmptyValueBlock.INSTANCE; } diff --git a/src/main/java/com/facebook/presto/TupleInfo.java b/src/main/java/com/facebook/presto/TupleInfo.java index 976bdcc2b7dcb..58b90a5d86987 100644 --- a/src/main/java/com/facebook/presto/TupleInfo.java +++ b/src/main/java/com/facebook/presto/TupleInfo.java @@ -4,6 +4,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; +import java.util.ArrayList; import java.util.List; import static com.facebook.presto.SizeOf.SIZE_OF_SHORT; @@ -37,14 +38,18 @@ boolean isFixedSize() { return size != -1; } + } private final int size; + private final List types; private final List offsets; private final int firstVariableLengthField; private final int secondVariableLengthField; private final int fixedSizePart; + private final int variableLengthFieldCount; + private final int variablePartOffset; public TupleInfo(Type... types) { @@ -77,11 +82,13 @@ public TupleInfo(List types) int firstVariableLengthField = -1; int secondVariableLengthField = -1; + int variableLengthFieldCount = 0; // process variable length fields for (int i = 0; i < types.size(); i++) { Type type = types.get(i); if (!type.isFixedSize()) { + ++variableLengthFieldCount; offsets[i] = currentOffset; if (hasVariableLengthFields) { currentOffset += SIZE_OF_SHORT; // we use a short to encode the offset of a var length field @@ -115,8 +122,29 @@ public TupleInfo(List types) this.firstVariableLengthField = firstVariableLengthField; this.secondVariableLengthField = secondVariableLengthField; + this.variableLengthFieldCount = variableLengthFieldCount; this.offsets = ImmutableList.copyOf(Ints.asList(offsets)); + + + // compute offset of variable sized part + int variablePartOffset = 0; + boolean isFirst = true; + for (TupleInfo.Type type : getTypes()) { + if (!type.isFixedSize()) { + if (!isFirst) { // skip offset field for first variable length field + variablePartOffset += SizeOf.SIZE_OF_SHORT; + } + + isFirst = false; + } + else { + variablePartOffset += type.getSize(); + } + } + variablePartOffset += SizeOf.SIZE_OF_SHORT; // total tuple size field + + this.variablePartOffset = variablePartOffset; } public List getTypes() @@ -124,6 +152,11 @@ public List getTypes() return types; } + public int getFieldCount() + { + return types.size(); + } + public int size(Slice slice) { if (size != -1) { @@ -167,6 +200,16 @@ private int getOffset(int index) return offsets.get(index); } + public Builder builder(SliceOutput sliceOutput) + { + return new Builder(sliceOutput); + } + + public Builder builder() + { + return new Builder(new DynamicSliceOutput(0)); + } + @Override public boolean equals(Object o) { @@ -204,4 +247,100 @@ public String toString() ", fixedSizePart=" + fixedSizePart + '}'; } + + + public class Builder + { + private final SliceOutput sliceOutput; + private final List variableLengthFields; + + private int currentField; + + public Builder(SliceOutput sliceOutput) + { + this.sliceOutput = sliceOutput; + this.variableLengthFields = new ArrayList<>(variableLengthFieldCount); + } + + public Builder append(long value) + { + checkState(TupleInfo.this.getTypes().get(currentField) == FIXED_INT_64, "Cannot append long. Current field (%s) is of type %s", currentField, TupleInfo.this.getTypes().get(currentField)); + + sliceOutput.writeLong(value); + currentField++; + + return this; + } + + public Builder append(Slice value) + { + checkState(TupleInfo.this.getTypes().get(currentField) == VARIABLE_BINARY, "Cannot append binary. Current field (%s) is of type %s", currentField, TupleInfo.this.getTypes().get(currentField)); + + variableLengthFields.add(value); + currentField++; + + return this; + } + + public void append(Tuple tuple) + { + // TODO: optimization - single copy of block of fixed length fields + + int index = 0; + for (TupleInfo.Type type : tuple.getTupleInfo().getTypes()) { + switch (type) { + case FIXED_INT_64: + append(tuple.getLong(index)); + break; + case VARIABLE_BINARY: + append(tuple.getSlice(index)); + break; + default: + throw new IllegalStateException("Type not yet supported: " + type); + } + + index++; + } + } + + public boolean isComplete() + { + return currentField == types.size(); + } + + public void finish() + { + Preconditions.checkState(currentField == types.size(), "Tuple is incomplete"); + + // write offsets + boolean isFirst = true; + int offset = variablePartOffset; + for (Slice field : variableLengthFields) { + if (!isFirst) { + sliceOutput.writeShort(offset); + } + offset += field.length(); + isFirst = false; + } + + if (!variableLengthFields.isEmpty()) { + sliceOutput.writeShort(offset); // total tuple length + } + + // write values + for (Slice field : variableLengthFields) { + sliceOutput.writeBytes(field); + } + + currentField = 0; + variableLengthFields.clear(); + } + + public Tuple build() + { + finish(); + return new Tuple(sliceOutput.slice(), TupleInfo.this); + } + } + } diff --git a/src/test/java/com/facebook/presto/TestCsvFileScanner.java b/src/test/java/com/facebook/presto/TestCsvFileScanner.java index 7fb39c5fa52f0..74c281c8eb72b 100644 --- a/src/test/java/com/facebook/presto/TestCsvFileScanner.java +++ b/src/test/java/com/facebook/presto/TestCsvFileScanner.java @@ -1,6 +1,5 @@ package com.facebook.presto; -import com.google.common.base.Charsets; import com.google.common.collect.ImmutableList; import com.google.common.io.InputSupplier; import org.testng.Assert; @@ -12,7 +11,6 @@ import static com.facebook.presto.SizeOf.SIZE_OF_SHORT; import static com.facebook.presto.TupleInfo.Type.FIXED_INT_64; import static com.facebook.presto.TupleInfo.Type.VARIABLE_BINARY; -import static com.google.common.base.Charsets.*; import static com.google.common.base.Charsets.UTF_8; import static com.google.common.io.Resources.getResource; import static com.google.common.io.Resources.newReaderSupplier; @@ -54,20 +52,22 @@ public void testIterator() private Tuple createTuple(String value) { - byte[] bytes = value.getBytes(UTF_8); - Slice slice = Slices.allocate(bytes.length + SIZE_OF_SHORT); - slice.output() - .appendShort(bytes.length + 2) - .appendBytes(bytes); + TupleInfo tupleInfo = new TupleInfo(VARIABLE_BINARY); + Tuple tuple = tupleInfo.builder() + .append(Slices.wrappedBuffer(value.getBytes(UTF_8))) + .build(); - return new Tuple(slice, new TupleInfo(VARIABLE_BINARY)); + return tuple; } private Tuple createTuple(long value) { - Slice slice = Slices.allocate(SIZE_OF_LONG); - slice.setLong(0, value); - return new Tuple(slice, new TupleInfo(FIXED_INT_64)); + TupleInfo tupleInfo = new TupleInfo(FIXED_INT_64); + Tuple tuple = tupleInfo.builder() + .append(value) + .build(); + + return tuple; } diff --git a/src/test/java/com/facebook/presto/TestSumAggregation.java b/src/test/java/com/facebook/presto/TestSumAggregation.java index 461791073190f..1ef993744d18b 100644 --- a/src/test/java/com/facebook/presto/TestSumAggregation.java +++ b/src/test/java/com/facebook/presto/TestSumAggregation.java @@ -58,19 +58,6 @@ public AggregationFunction get() Assert.assertEquals(actual, expected); } - private Tuple createTuple(String key, long count) - { - byte[] bytes = key.getBytes(Charsets.UTF_8); - Slice slice = Slices.allocate(SIZE_OF_LONG + SIZE_OF_SHORT + bytes.length); - - slice.output() - .appendLong(count) - .appendShort(10 + bytes.length) - .appendBytes(bytes); - - return new Tuple(slice, new TupleInfo(VARIABLE_BINARY, FIXED_INT_64)); - } - @Test public void testHashAggregation() { @@ -157,4 +144,15 @@ private ValueBlock createBlock(long position, long... values) return builder.build(); } + + private Tuple createTuple(String key, long count) + { + TupleInfo tupleInfo = new TupleInfo(VARIABLE_BINARY, FIXED_INT_64); + Tuple tuple = tupleInfo.builder() + .append(Slices.wrappedBuffer(key.getBytes(UTF_8))) + .append(count) + .build(); + + return tuple; + } } diff --git a/src/test/java/com/facebook/presto/TestTupleInfo.java b/src/test/java/com/facebook/presto/TestTupleInfo.java index cbf4dade7cd8d..e66bb19b6f620 100644 --- a/src/test/java/com/facebook/presto/TestTupleInfo.java +++ b/src/test/java/com/facebook/presto/TestTupleInfo.java @@ -2,7 +2,6 @@ import org.testng.annotations.Test; -import static com.facebook.presto.SizeOf.SIZE_OF_LONG; import static com.facebook.presto.SizeOf.SIZE_OF_SHORT; import static com.facebook.presto.TupleInfo.Type.FIXED_INT_64; import static com.facebook.presto.TupleInfo.Type.VARIABLE_BINARY; @@ -11,26 +10,77 @@ public class TestTupleInfo { @Test - public void testBasic() + public void testOnlyFixedLength() { - Slice slice = new Slice(SIZE_OF_LONG + SIZE_OF_LONG + SIZE_OF_SHORT + SIZE_OF_SHORT + 10 + 15); + TupleInfo info = new TupleInfo(FIXED_INT_64, FIXED_INT_64); + + Tuple tuple = info.builder() + .append(42) + .append(67) + .build(); + + assertEquals(tuple.getLong(0), 42L); + assertEquals(tuple.getLong(1), 67L); + assertEquals(tuple.size(), 16); + } + + @Test + public void testSingleVariableLength() + { + TupleInfo info = new TupleInfo(VARIABLE_BINARY); + + Slice binary = Slices.wrappedBuffer(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + + Tuple tuple = info.builder() + .append(binary) + .build(); + + assertEquals(tuple.getSlice(0), binary); + assertEquals(tuple.size(), binary.length() + SIZE_OF_SHORT); + } + + @Test + public void testMultipleVariableLength() + { + TupleInfo info = new TupleInfo(VARIABLE_BINARY, VARIABLE_BINARY); Slice binary1 = Slices.wrappedBuffer(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); Slice binary2 = Slices.wrappedBuffer(new byte[] { 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 }); - slice.output() - .appendLong(42) // value of first long field - .appendLong(67) // value of second long field - .appendShort(30) // offset of second binary field - .appendShort(45) // tuple size - .appendBytes(binary1) - .appendBytes(binary2); + Tuple tuple = info.builder() + .append(binary1) + .append(binary2) + .build(); + + assertEquals(tuple.getSlice(0), binary1); + assertEquals(tuple.getSlice(1), binary2); + assertEquals(tuple.size(), binary1.length() + binary2.length() + SIZE_OF_SHORT + SIZE_OF_SHORT); + } + + + @Test + public void testMixed() + { + TupleInfo info = new TupleInfo(FIXED_INT_64, VARIABLE_BINARY, FIXED_INT_64, VARIABLE_BINARY, FIXED_INT_64, VARIABLE_BINARY); + + Slice binary1 = Slices.wrappedBuffer(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + Slice binary2 = Slices.wrappedBuffer(new byte[] { 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 }); + Slice binary3 = Slices.wrappedBuffer(new byte[] { 30, 31, 32, 33, 34, 35 }); + Tuple tuple = info.builder() + .append(42) + .append(binary1) + .append(67) + .append(binary2) + .append(90) + .append(binary3) + .build(); - TupleInfo info = new TupleInfo(FIXED_INT_64, VARIABLE_BINARY, FIXED_INT_64, VARIABLE_BINARY); - assertEquals(info.getLong(slice, 0), 42L); - assertEquals(info.getSlice(slice, 1), binary1); - assertEquals(info.getLong(slice, 2), 67L); - assertEquals(info.getSlice(slice, 3), binary2); + assertEquals(tuple.getLong(0), 42L); + assertEquals(tuple.getSlice(1), binary1); + assertEquals(tuple.getLong(2), 67L); + assertEquals(tuple.getSlice(3), binary2); + assertEquals(tuple.getLong(4), 90L); + assertEquals(tuple.getSlice(5), binary3); } }