Skip to content

ARROW-1474: [JAVA] Java ValueVector hierarchy Refactor (Implementation Phase 2) #1203

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
siddharthteotia wants to merge 3 commits into from
Closed

ARROW-1474: [JAVA] Java ValueVector hierarchy Refactor (Implementation Phase 2) #1203

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b83d874
ARROW-1474:[JAVA] ValueVector hierarchy (Implementation Phase 2)
siddharthteotia Oct 16, 2017
5e310ae
add some javadocs
siddharthteotia Oct 23, 2017
823f75d
review comments and complete javadocs, code comments
siddharthteotia Oct 23, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,8 @@ static void validateOutput(File testOutFile, BufferAllocator allocator) throws E
static void validateContent(int count, VectorSchemaRoot root) {
Assert.assertEquals(count, root.getRowCount());
for (int i = 0; i < count; i++) {
Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i));
Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
Assert.assertEquals(i, root.getVector("int").getObject(i));
Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i));
}
}

Expand Down
84 changes: 39 additions & 45 deletions java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,10 @@ private void testEchoServer(int serverPort,
for (int i = 0; i < batches; i++) {
vector.allocateNew(16);
for (int j = 0; j < 8; j++) {
vector.getMutator().set(j, j + i);
vector.getMutator().set(j + 8, 0, (byte) (j + i));
vector.set(j, j + i);
vector.set(j + 8, 0, (byte) (j + i));
}
vector.getMutator().setValueCount(16);
vector.setValueCount(16);
root.setRowCount(16);
writer.writeBatch();
}
Expand All @@ -120,10 +120,10 @@ private void testEchoServer(int serverPort,
for (int i = 0; i < batches; i++) {
Assert.assertTrue(reader.loadNextBatch());
assertEquals(16, reader.getVectorSchemaRoot().getRowCount());
assertEquals(16, readVector.getAccessor().getValueCount());
assertEquals(16, readVector.getValueCount());
for (int j = 0; j < 8; j++) {
assertEquals(j + i, readVector.getAccessor().get(j));
assertTrue(readVector.getAccessor().isNull(j + 8));
assertEquals(j + i, readVector.get(j));
assertTrue(readVector.isNull(j + 8));
}
}
Assert.assertFalse(reader.loadNextBatch());
Expand Down Expand Up @@ -169,20 +169,18 @@ public void testFlatDictionary() throws IOException {
FieldType.nullable(VARCHAR.getType()),
allocator)) {
writeVector.allocateNewSafe();
NullableIntVector.Mutator mutator = writeVector.getMutator();
mutator.set(0, 0);
mutator.set(1, 1);
mutator.set(3, 2);
mutator.set(4, 1);
mutator.set(5, 2);
mutator.setValueCount(6);
writeVector.set(0, 0);
writeVector.set(1, 1);
writeVector.set(3, 2);
writeVector.set(4, 1);
writeVector.set(5, 2);
writeVector.setValueCount(6);

writeDictionaryVector.allocateNewSafe();
NullableVarCharVector.Mutator dictionaryMutator = writeDictionaryVector.getMutator();
dictionaryMutator.set(0, "foo".getBytes(StandardCharsets.UTF_8));
dictionaryMutator.set(1, "bar".getBytes(StandardCharsets.UTF_8));
dictionaryMutator.set(2, "baz".getBytes(StandardCharsets.UTF_8));
dictionaryMutator.setValueCount(3);
writeDictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8));
writeDictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8));
writeDictionaryVector.set(2, "baz".getBytes(StandardCharsets.UTF_8));
writeDictionaryVector.setValueCount(3);

List<Field> fields = ImmutableList.of(writeVector.getField());
List<FieldVector> vectors = ImmutableList.of((FieldVector) writeVector);
Expand Down Expand Up @@ -210,23 +208,21 @@ public void testFlatDictionary() throws IOException {
Assert.assertNotNull(readEncoding);
Assert.assertEquals(1L, readEncoding.getId());

FieldVector.Accessor accessor = readVector.getAccessor();
Assert.assertEquals(6, accessor.getValueCount());
Assert.assertEquals(0, accessor.getObject(0));
Assert.assertEquals(1, accessor.getObject(1));
Assert.assertEquals(null, accessor.getObject(2));
Assert.assertEquals(2, accessor.getObject(3));
Assert.assertEquals(1, accessor.getObject(4));
Assert.assertEquals(2, accessor.getObject(5));
Assert.assertEquals(6, readVector.getValueCount());
Assert.assertEquals(0, readVector.getObject(0));
Assert.assertEquals(1, readVector.getObject(1));
Assert.assertEquals(null, readVector.getObject(2));
Assert.assertEquals(2, readVector.getObject(3));
Assert.assertEquals(1, readVector.getObject(4));
Assert.assertEquals(2, readVector.getObject(5));

Dictionary dictionary = reader.lookup(1L);
Assert.assertNotNull(dictionary);
NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary
.getVector()).getAccessor();
Assert.assertEquals(3, dictionaryAccessor.getValueCount());
Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2));
NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector());
Assert.assertEquals(3, dictionaryVector.getValueCount());
Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0));
Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1));
Assert.assertEquals(new Text("baz"), dictionaryVector.getObject(2));
}
}
}
Expand All @@ -243,9 +239,9 @@ public void testNestedDictionary() throws IOException {
// [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]

writeDictionaryVector.allocateNew();
writeDictionaryVector.getMutator().set(0, "foo".getBytes(StandardCharsets.UTF_8));
writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
writeDictionaryVector.getMutator().setValueCount(2);
writeDictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8));
writeDictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8));
writeDictionaryVector.setValueCount(2);

writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
writeVector.allocateNew();
Expand Down Expand Up @@ -297,19 +293,17 @@ public void testNestedDictionary() throws IOException {
Assert.assertEquals(2L, encoding.getId());
Assert.assertEquals(new Int(32, true), encoding.getIndexType());

ListVector.Accessor accessor = readVector.getAccessor();
Assert.assertEquals(3, accessor.getValueCount());
Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0));
Assert.assertEquals(Arrays.asList(0), accessor.getObject(1));
Assert.assertEquals(Arrays.asList(1), accessor.getObject(2));
Assert.assertEquals(3, readVector.getValueCount());
Assert.assertEquals(Arrays.asList(0, 1), readVector.getObject(0));
Assert.assertEquals(Arrays.asList(0), readVector.getObject(1));
Assert.assertEquals(Arrays.asList(1), readVector.getObject(2));

Dictionary readDictionary = reader.lookup(2L);
Assert.assertNotNull(readDictionary);
NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector)
readDictionary.getVector()).getAccessor();
Assert.assertEquals(2, dictionaryAccessor.getValueCount());
Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0));
Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1));
NullableVarCharVector dictionaryVector = ((NullableVarCharVector) readDictionary.getVector());
Assert.assertEquals(2, dictionaryVector.getValueCount());
Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0));
Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1));
}
}
}
Expand Down
24 changes: 4 additions & 20 deletions java/vector/src/main/codegen/templates/ComplexReaders.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,11 +70,7 @@ public Field getField(){

public boolean isSet(){
<#if nullMode == "Nullable">
<#if minor.class != "Int" && minor.class != "VarChar">
return !vector.getAccessor().isNull(idx());
<#else>
return !vector.isNull(idx());
</#if>
<#else>
return true;
</#if>
Expand All @@ -97,19 +93,11 @@ public void read(${minor.class?cap_first}Holder h){
</#if>

public void read(Nullable${minor.class?cap_first}Holder h){
<#if minor.class != "Int" && minor.class != "VarChar">
vector.getAccessor().get(idx(), h);
<#else>
vector.get(idx(), h);
</#if>
vector.get(idx(), h);
}

public ${friendlyType} read${safeType}(){
<#if minor.class == "Int" || minor.class == "VarChar">
return vector.getObject(idx());
<#else>
return vector.getAccessor().getObject(idx());
</#if>
return vector.getObject(idx());
}

<#if minor.class == "TimeStampSec" ||
Expand All @@ -118,7 +106,7 @@ public void read(Nullable${minor.class?cap_first}Holder h){
minor.class == "TimeStampNano">
@Override
public ${minor.boxedType} read${minor.boxedType}(){
return vector.getAccessor().get(idx());
return vector.get(idx());
}
</#if>

Expand All @@ -127,11 +115,7 @@ public void copyValue(FieldWriter w){
}

public Object readObject(){
<#if minor.class == "Int" || minor.class == "VarChar">
return (Object)vector.getObject(idx());
<#else>
return vector.getAccessor().getObject(idx());
</#if>
return (Object)vector.getObject(idx());
}
}
</#if>
Expand Down
51 changes: 12 additions & 39 deletions java/vector/src/main/codegen/templates/ComplexWriters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,9 @@
@SuppressWarnings("unused")
public class ${eName}WriterImpl extends AbstractFieldWriter {

<#if minor.class != "Int" && minor.class != "VarChar">
private final Nullable${name}Vector.Mutator mutator;
</#if>

final Nullable${name}Vector vector;

public ${eName}WriterImpl(Nullable${name}Vector vector) {
<#if minor.class != "Int" && minor.class != "VarChar">
this.mutator = vector.getMutator();
</#if>
this.vector = vector;
}

Expand Down Expand Up @@ -108,51 +101,31 @@ public void setPosition(int idx) {
<#else>

public void write(${minor.class}Holder h) {
<#if minor.class != "Int" && minor.class != "VarChar">
mutator.setSafe(idx(), h);
vector.getMutator().setValueCount(idx()+1);
<#else>
vector.setSafe(idx(), h);
vector.setValueCount(idx()+1);
</#if>
vector.setSafe(idx(), h);
vector.setValueCount(idx()+1);
}

public void write(Nullable${minor.class}Holder h) {
<#if minor.class != "Int" && minor.class != "VarChar">
mutator.setSafe(idx(), h);
vector.getMutator().setValueCount(idx()+1);
<#else>
vector.setSafe(idx(), h);
vector.setValueCount(idx()+1);
</#if>
vector.setSafe(idx(), h);
vector.setValueCount(idx()+1);
}

public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
<#if minor.class != "Int" && minor.class != "VarChar">
mutator.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
vector.getMutator().setValueCount(idx()+1);
<#else>
vector.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
vector.setValueCount(idx()+1);
</#if>
vector.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
vector.setValueCount(idx()+1);
}
<#if minor.class == "Decimal">

<#if minor.class == "Decimal">
public void write${minor.class}(${friendlyType} value) {
mutator.setSafe(idx(), value);
vector.getMutator().setValueCount(idx()+1);
vector.setSafe(idx(), value);
vector.setValueCount(idx()+1);
}
</#if>
<#if mode == "Nullable">

<#if mode == "Nullable">
public void writeNull() {
<#if minor.class != "Int" && minor.class != "VarChar">
mutator.setNull(idx());
vector.getMutator().setValueCount(idx()+1);
<#else>
vector.setNull(idx());
vector.setValueCount(idx()+1);
</#if>
vector.setNull(idx());
vector.setValueCount(idx()+1);
}
</#if>
</#if>
Expand Down
Loading