diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java index 6d9a6c1323e..c56a5a3303b 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java @@ -78,8 +78,8 @@ static void validateOutput(File testOutFile, BufferAllocator allocator) throws E static void validateContent(int count, VectorSchemaRoot root) { Assert.assertEquals(count, root.getRowCount()); for (int i = 0; i < count; i++) { - Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i)); - Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i)); + Assert.assertEquals(i, root.getVector("int").getObject(i)); + Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); } } diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java index 467965aff95..db7eb54e631 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java @@ -104,10 +104,10 @@ private void testEchoServer(int serverPort, for (int i = 0; i < batches; i++) { vector.allocateNew(16); for (int j = 0; j < 8; j++) { - vector.getMutator().set(j, j + i); - vector.getMutator().set(j + 8, 0, (byte) (j + i)); + vector.set(j, j + i); + vector.set(j + 8, 0, (byte) (j + i)); } - vector.getMutator().setValueCount(16); + vector.setValueCount(16); root.setRowCount(16); writer.writeBatch(); } @@ -120,10 +120,10 @@ private void testEchoServer(int serverPort, for (int i = 0; i < batches; i++) { Assert.assertTrue(reader.loadNextBatch()); assertEquals(16, reader.getVectorSchemaRoot().getRowCount()); - assertEquals(16, readVector.getAccessor().getValueCount()); + assertEquals(16, readVector.getValueCount()); for (int j = 0; j < 8; j++) { - assertEquals(j + i, readVector.getAccessor().get(j)); - assertTrue(readVector.getAccessor().isNull(j + 8)); + assertEquals(j + i, readVector.get(j)); + assertTrue(readVector.isNull(j + 8)); } } Assert.assertFalse(reader.loadNextBatch()); @@ -169,20 +169,18 @@ public void testFlatDictionary() throws IOException { FieldType.nullable(VARCHAR.getType()), allocator)) { writeVector.allocateNewSafe(); - NullableIntVector.Mutator mutator = writeVector.getMutator(); - mutator.set(0, 0); - mutator.set(1, 1); - mutator.set(3, 2); - mutator.set(4, 1); - mutator.set(5, 2); - mutator.setValueCount(6); + writeVector.set(0, 0); + writeVector.set(1, 1); + writeVector.set(3, 2); + writeVector.set(4, 1); + writeVector.set(5, 2); + writeVector.setValueCount(6); writeDictionaryVector.allocateNewSafe(); - NullableVarCharVector.Mutator dictionaryMutator = writeDictionaryVector.getMutator(); - dictionaryMutator.set(0, "foo".getBytes(StandardCharsets.UTF_8)); - dictionaryMutator.set(1, "bar".getBytes(StandardCharsets.UTF_8)); - dictionaryMutator.set(2, "baz".getBytes(StandardCharsets.UTF_8)); - dictionaryMutator.setValueCount(3); + writeDictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.set(2, "baz".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.setValueCount(3); List fields = ImmutableList.of(writeVector.getField()); List vectors = ImmutableList.of((FieldVector) writeVector); @@ -210,23 +208,21 @@ public void testFlatDictionary() throws IOException { Assert.assertNotNull(readEncoding); Assert.assertEquals(1L, readEncoding.getId()); - FieldVector.Accessor accessor = readVector.getAccessor(); - Assert.assertEquals(6, accessor.getValueCount()); - Assert.assertEquals(0, accessor.getObject(0)); - Assert.assertEquals(1, accessor.getObject(1)); - Assert.assertEquals(null, accessor.getObject(2)); - Assert.assertEquals(2, accessor.getObject(3)); - Assert.assertEquals(1, accessor.getObject(4)); - Assert.assertEquals(2, accessor.getObject(5)); + Assert.assertEquals(6, readVector.getValueCount()); + Assert.assertEquals(0, readVector.getObject(0)); + Assert.assertEquals(1, readVector.getObject(1)); + Assert.assertEquals(null, readVector.getObject(2)); + Assert.assertEquals(2, readVector.getObject(3)); + Assert.assertEquals(1, readVector.getObject(4)); + Assert.assertEquals(2, readVector.getObject(5)); Dictionary dictionary = reader.lookup(1L); Assert.assertNotNull(dictionary); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary - .getVector()).getAccessor(); - Assert.assertEquals(3, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); - Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2)); + NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector()); + Assert.assertEquals(3, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); + Assert.assertEquals(new Text("baz"), dictionaryVector.getObject(2)); } } } @@ -243,9 +239,9 @@ public void testNestedDictionary() throws IOException { // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]] writeDictionaryVector.allocateNew(); - writeDictionaryVector.getMutator().set(0, "foo".getBytes(StandardCharsets.UTF_8)); - writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8)); - writeDictionaryVector.getMutator().setValueCount(2); + writeDictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.setValueCount(2); writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null)); writeVector.allocateNew(); @@ -297,19 +293,17 @@ public void testNestedDictionary() throws IOException { Assert.assertEquals(2L, encoding.getId()); Assert.assertEquals(new Int(32, true), encoding.getIndexType()); - ListVector.Accessor accessor = readVector.getAccessor(); - Assert.assertEquals(3, accessor.getValueCount()); - Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0)); - Assert.assertEquals(Arrays.asList(0), accessor.getObject(1)); - Assert.assertEquals(Arrays.asList(1), accessor.getObject(2)); + Assert.assertEquals(3, readVector.getValueCount()); + Assert.assertEquals(Arrays.asList(0, 1), readVector.getObject(0)); + Assert.assertEquals(Arrays.asList(0), readVector.getObject(1)); + Assert.assertEquals(Arrays.asList(1), readVector.getObject(2)); Dictionary readDictionary = reader.lookup(2L); Assert.assertNotNull(readDictionary); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) - readDictionary.getVector()).getAccessor(); - Assert.assertEquals(2, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); + NullableVarCharVector dictionaryVector = ((NullableVarCharVector) readDictionary.getVector()); + Assert.assertEquals(2, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); } } } diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java index 79106490515..60347a2fcb8 100644 --- a/java/vector/src/main/codegen/templates/ComplexReaders.java +++ b/java/vector/src/main/codegen/templates/ComplexReaders.java @@ -70,11 +70,7 @@ public Field getField(){ public boolean isSet(){ <#if nullMode == "Nullable"> - <#if minor.class != "Int" && minor.class != "VarChar"> - return !vector.getAccessor().isNull(idx()); - <#else> return !vector.isNull(idx()); - <#else> return true; @@ -97,19 +93,11 @@ public void read(${minor.class?cap_first}Holder h){ public void read(Nullable${minor.class?cap_first}Holder h){ - <#if minor.class != "Int" && minor.class != "VarChar"> - vector.getAccessor().get(idx(), h); - <#else> - vector.get(idx(), h); - + vector.get(idx(), h); } public ${friendlyType} read${safeType}(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - return vector.getObject(idx()); - <#else> - return vector.getAccessor().getObject(idx()); - + return vector.getObject(idx()); } <#if minor.class == "TimeStampSec" || @@ -118,7 +106,7 @@ public void read(Nullable${minor.class?cap_first}Holder h){ minor.class == "TimeStampNano"> @Override public ${minor.boxedType} read${minor.boxedType}(){ - return vector.getAccessor().get(idx()); + return vector.get(idx()); } @@ -127,11 +115,7 @@ public void copyValue(FieldWriter w){ } public Object readObject(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - return (Object)vector.getObject(idx()); - <#else> - return vector.getAccessor().getObject(idx()); - + return (Object)vector.getObject(idx()); } } diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index 77f6594a078..406bbb39c7f 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -39,16 +39,9 @@ @SuppressWarnings("unused") public class ${eName}WriterImpl extends AbstractFieldWriter { - <#if minor.class != "Int" && minor.class != "VarChar"> - private final Nullable${name}Vector.Mutator mutator; - - final Nullable${name}Vector vector; public ${eName}WriterImpl(Nullable${name}Vector vector) { - <#if minor.class != "Int" && minor.class != "VarChar"> - this.mutator = vector.getMutator(); - this.vector = vector; } @@ -108,51 +101,31 @@ public void setPosition(int idx) { <#else> public void write(${minor.class}Holder h) { - <#if minor.class != "Int" && minor.class != "VarChar"> - mutator.setSafe(idx(), h); - vector.getMutator().setValueCount(idx()+1); - <#else> - vector.setSafe(idx(), h); - vector.setValueCount(idx()+1); - + vector.setSafe(idx(), h); + vector.setValueCount(idx()+1); } public void write(Nullable${minor.class}Holder h) { - <#if minor.class != "Int" && minor.class != "VarChar"> - mutator.setSafe(idx(), h); - vector.getMutator().setValueCount(idx()+1); - <#else> - vector.setSafe(idx(), h); - vector.setValueCount(idx()+1); - + vector.setSafe(idx(), h); + vector.setValueCount(idx()+1); } public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { - <#if minor.class != "Int" && minor.class != "VarChar"> - mutator.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); - vector.getMutator().setValueCount(idx()+1); - <#else> - vector.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); - vector.setValueCount(idx()+1); - + vector.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); + vector.setValueCount(idx()+1); } - <#if minor.class == "Decimal"> + <#if minor.class == "Decimal"> public void write${minor.class}(${friendlyType} value) { - mutator.setSafe(idx(), value); - vector.getMutator().setValueCount(idx()+1); + vector.setSafe(idx(), value); + vector.setValueCount(idx()+1); } - <#if mode == "Nullable"> + <#if mode == "Nullable"> public void writeNull() { - <#if minor.class != "Int" && minor.class != "VarChar"> - mutator.setNull(idx()); - vector.getMutator().setValueCount(idx()+1); - <#else> - vector.setNull(idx()); - vector.setValueCount(idx()+1); - + vector.setNull(idx()); + vector.setValueCount(idx()+1); } diff --git a/java/vector/src/main/codegen/templates/LegacyUnionVector.java b/java/vector/src/main/codegen/templates/LegacyUnionVector.java new file mode 100644 index 00000000000..b9cb6dfdc24 --- /dev/null +++ b/java/vector/src/main/codegen/templates/LegacyUnionVector.java @@ -0,0 +1,356 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +<@pp.dropOutputFile /> +<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/LegacyUnionVector.java" /> + + +<#include "/@includes/license.ftl" /> + + package org.apache.arrow.vector.complex; + +<#include "/@includes/vv_imports.ftl" /> + import com.google.common.collect.ImmutableList; + import java.util.ArrayList; + import java.util.Collections; + import java.util.Iterator; + import org.apache.arrow.vector.BaseDataValueVector; + import org.apache.arrow.vector.complex.impl.ComplexCopier; + import org.apache.arrow.vector.util.CallBack; + import org.apache.arrow.vector.schema.ArrowFieldNode; + + import static org.apache.arrow.vector.types.UnionMode.Sparse; + + + +/* + * This class is generated using freemarker and the ${.template_name} template. + */ +@SuppressWarnings("unused") + + +/** + * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each + * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods, + * as well as the addOrGet method. + * + * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup + * each time the vector is accessed. + * Source code generated using FreeMarker template ${.template_name} + */ +public class LegacyUnionVector implements FieldVector { + + private Accessor accessor = new Accessor(); + private Mutator mutator = new Mutator(); + private final UnionVector unionVector; + + public LegacyUnionVector(String name, BufferAllocator allocator, CallBack callBack) { + unionVector = new UnionVector(name, allocator, callBack); + } + + public BufferAllocator getAllocator() { + return unionVector.getAllocator(); + } + + @Override + public MinorType getMinorType() { + return MinorType.UNION; + } + + @Override + public void initializeChildrenFromFields(List children) { + unionVector.initializeChildrenFromFields(children); + } + + @Override + public List getChildrenFromFields() { + return unionVector.getChildrenFromFields(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + unionVector.loadFieldBuffers(fieldNode, ownBuffers); + } + + @Override + public List getFieldBuffers() { + return unionVector.getFieldBuffers(); + } + + @Override + public List getFieldInnerVectors() { + return unionVector.getFieldInnerVectors(); + } + + @Override + public long getValidityBufferAddress() { + return unionVector.getValidityBufferAddress(); + } + + @Override + public long getDataBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getValidityBuffer() { + return unionVector.getValidityBuffer(); + } + + @Override + public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); } + + @Override + public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } + + public NullableMapVector getMap() { + return unionVector.getMap(); + } + <#list vv.types as type> + <#list type.minor as minor> + <#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#assign lowerCaseName = name?lower_case/> + <#if !minor.typeParams?? > + + private Nullable${name}Vector ${uncappedName}Vector; + + public Nullable${name}Vector get${name}Vector() { + return unionVector.get${name}Vector(); + } + + + + + public ListVector getList() { + return unionVector.getList(); + } + + public int getTypeValue(int index) { + return unionVector.getTypeValue(index); + } + + @Override + public void allocateNew() throws OutOfMemoryException { + unionVector.allocateNew(); + } + + @Override + public boolean allocateNewSafe() { + return unionVector.allocateNewSafe(); + } + + @Override + public void reAlloc() { + unionVector.reAlloc(); + } + + @Override + public void setInitialCapacity(int numRecords) { + } + + @Override + public int getValueCapacity() { + return unionVector.getValueCapacity(); + } + + @Override + public void close() { + unionVector.close(); + } + + @Override + public void clear() { + unionVector.clear(); + } + + @Override + public Field getField() { + return unionVector.getField(); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return unionVector.getTransferPair(allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return unionVector.getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return unionVector.getTransferPair(ref, allocator, callBack); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return unionVector.makeTransferPair(((LegacyUnionVector)target).unionVector); + } + + public void copyFrom(int inIndex, int outIndex, UnionVector from) { + unionVector.copyFrom(inIndex, outIndex, from); + } + + public void copyFromSafe(int inIndex, int outIndex, UnionVector from) { + unionVector.copyFromSafe(inIndex, outIndex, from); + } + + public FieldVector addVector(FieldVector v) { + return unionVector.addVector(v); + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + @Override + public FieldReader getReader() { + return unionVector.getReader(); + } + + public FieldWriter getWriter() { + return unionVector.getWriter(); + } + + @Override + public int getBufferSize() { + return unionVector.getBufferSize(); + } + + @Override + public int getBufferSizeFor(final int valueCount) { + return unionVector.getBufferSizeFor(valueCount); + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + return unionVector.getBuffers(clear); + } + + @Override + public Iterator iterator() { + return unionVector.iterator(); + } + + public class Accessor extends BaseValueVector.BaseAccessor { + + @Override + public Object getObject(int index) { + return unionVector.getObject(index); + } + + public byte[] get(int index) { + return unionVector.get(index); + } + + public void get(int index, ComplexHolder holder) { + } + + public void get(int index, UnionHolder holder) { + unionVector.get(index, holder); + } + + public int getNullCount() { + return unionVector.getNullCount(); + } + + @Override + public int getValueCount() { + return unionVector.getValueCount(); + } + + @Override + public boolean isNull(int index) { + return unionVector.isNull(index); + } + + public int isSet(int index) { + return unionVector.isSet(index); + } + } + + public class Mutator extends BaseValueVector.BaseMutator { + + UnionWriter writer; + + @Override + public void setValueCount(int valueCount) { + unionVector.setValueCount(valueCount); + } + + public void setSafe(int index, UnionHolder holder) { + unionVector.setSafe(index, holder); + } + <#list vv.types as type> + <#list type.minor as minor> + <#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.typeParams?? > + public void setSafe(int index, Nullable${name}Holder holder) { + unionVector.setSafe(index, holder); + } + + + + + + public void setType(int index, MinorType type) { + unionVector.setType(index, type); + } + + @Override + public void reset() { } + + @Override + public void generateTestData(int values) { } + } + + @Override + @Deprecated + public int getValueCount() { return getAccessor().getValueCount(); } + + @Override + @Deprecated + public void setValueCount(int valueCount) { getMutator().setValueCount(valueCount);} + + @Override + @Deprecated + public Object getObject(int index) { return getAccessor().getObject(index); } + + @Override + @Deprecated + public int getNullCount() { return getAccessor().getNullCount(); } + + @Override + @Deprecated + public boolean isNull(int index) { return getAccessor().isNull(index); } +} diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java index b89f91457e8..6afd508e688 100644 --- a/java/vector/src/main/codegen/templates/MapWriters.java +++ b/java/vector/src/main/codegen/templates/MapWriters.java @@ -184,7 +184,7 @@ public ListWriter list(String name) { } public void setValueCount(int count) { - container.getMutator().setValueCount(count); + container.setValueCount(count); } @Override @@ -199,7 +199,7 @@ public void setPosition(int index) { public void start() { <#if mode == "Single"> <#else> - container.getMutator().setIndexDefined(idx()); + container.setIndexDefined(idx()); } diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java index 5d1f5a33b90..8e5b10b649b 100644 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java @@ -19,14 +19,8 @@ <#list vv.types as type> <#list type.minor as minor> -<#if minor.class == "Int" || minor.class == "VarChar"> <#assign className = "LegacyNullable${minor.class}Vector" /> <#assign valuesName = "Nullable${minor.class}Vector" /> -<#else> -<#assign className = "Nullable${minor.class}Vector" /> -<#assign valuesName = "${minor.class}Vector" /> - - <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> <@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" /> @@ -50,35 +44,21 @@ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ @SuppressWarnings("unused") -<#if minor.class == "Int" || minor.class == "VarChar"> @Deprecated - public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidthVector, FieldVector { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); protected final static byte[] emptyByteArray = new byte[]{}; - <#if minor.class != "Int" && minor.class != "VarChar"> - private final FieldReader reader = new ${minor.class}ReaderImpl(${className}.this); - - private final String bitsField = "$bits$"; private final String valuesField = "$values$"; - <#if minor.class != "Int" && minor.class != "VarChar"> - private final Field field; - - final BitVector bits = new BitVector(bitsField, allocator); final ${valuesName} values; private final Mutator mutator; private final Accessor accessor; - <#if minor.class != "Int" && minor.class != "VarChar"> - private final List innerVectors; - - <#if minor.typeParams??> <#assign typeParams = minor.typeParams?reverse> <#list typeParams as typeParam> @@ -122,33 +102,12 @@ public final class ${className} extends BaseValueVector implements <#if type.maj this.mutator = new Mutator(); this.accessor = new Accessor(); - <#if minor.class != "Int" && minor.class != "VarChar"> - this.field = new Field(name, fieldType, null); - innerVectors = Collections.unmodifiableList(Arrays.asList( - bits, - <#if type.major = "VarLen"> - values.offsetVector, - - values - )); - } - <#if minor.class != "Int" && minor.class != "VarChar"> - /* not needed for new vectors */ - public BitVector getValidityVector() { - return bits; - } - - @Override public List getFieldInnerVectors() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getFieldInnerVectors(); - <#else> - return innerVectors; - + /* DELEGATE TO NEW VECTOR */ + return values.getFieldInnerVectors(); } @Override @@ -165,123 +124,61 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.loadFieldBuffers(fieldNode, ownBuffers); - <#else> - <#if type.major = "VarLen"> - // variable width values: truncate offset vector buffer to size (#1) - org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, - values.offsetVector.getBufferSizeFor( - fieldNode.getLength() == 0? 0 : fieldNode.getLength() + 1)); - mutator.lastSet = fieldNode.getLength() - 1; - <#else> - // fixed width values truncate value vector to size (#1) - org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, values.getBufferSizeFor(fieldNode.getLength())); - - org.apache.arrow.vector.BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); - bits.valueCount = fieldNode.getLength(); - + /* DELEGATE TO NEW VECTOR */ + values.loadFieldBuffers(fieldNode, ownBuffers); } public List getFieldBuffers() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getFieldBuffers(); - <#else> - return org.apache.arrow.vector.BaseDataValueVector.unload(getFieldInnerVectors()); - + /* DELEGATE TO NEW VECTOR */ + return values.getFieldBuffers(); } @Override public Field getField() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getField(); - <#else> - return field; - + /* DELEGATE TO NEW VECTOR */ + return values.getField(); } @Override public MinorType getMinorType() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getMinorType(); - <#else> - return MinorType.${minor.class?upper_case}; - + /* DELEGATE TO NEW VECTOR */ + return values.getMinorType(); } @Override public FieldReader getReader(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getReader(); - <#else> - return reader; - + /* DELEGATE TO NEW VECTOR */ + return values.getReader(); } @Override public int getValueCapacity(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getValueCapacity(); - <#else> - return Math.min(bits.getValueCapacity(), values.getValueCapacity()); - + /* DELEGATE TO NEW VECTOR */ + return values.getValueCapacity(); } @Override public ArrowBuf[] getBuffers(boolean clear) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getBuffers(clear); - <#else> - final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), values.getBuffers(false), ArrowBuf.class); - if (clear) { - for (final ArrowBuf buffer:buffers) { - buffer.retain(1); - } - clear(); - } - return buffers; - + /* DELEGATE TO NEW VECTOR */ + return values.getBuffers(clear); } @Override public void close() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.close(); - <#else> - bits.close(); - values.close(); - super.close(); - + /* DELEGATE TO NEW VECTOR */ + values.close(); } @Override public void clear() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.clear(); - <#else> - bits.clear(); - values.clear(); - super.clear(); - + /* DELEGATE TO NEW VECTOR */ + values.clear(); } @Override public int getBufferSize(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getBufferSize(); - <#else> - return values.getBufferSize() + bits.getBufferSize(); - + /* DELEGATE TO NEW VECTOR */ + return values.getBufferSize(); } @Override @@ -289,14 +186,8 @@ public int getBufferSizeFor(final int valueCount) { if (valueCount == 0) { return 0; } - - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getBufferSizeFor(valueCount); - <#else> - return values.getBufferSizeFor(valueCount) - + bits.getBufferSizeFor(valueCount); - + /* DELEGATE TO NEW VECTOR */ + return values.getBufferSizeFor(valueCount); } public ArrowBuf getBuffer() { @@ -309,93 +200,38 @@ public ArrowBuf getBuffer() { @Override public void setInitialCapacity(int numRecords) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.setInitialCapacity(numRecords); - <#else> - bits.setInitialCapacity(numRecords); - values.setInitialCapacity(numRecords); - + /* DELEGATE TO NEW VECTOR */ + values.setInitialCapacity(numRecords); } @Override public void allocateNew() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(); - <#else> - if(!allocateNewSafe()){ - throw new OutOfMemoryException("Failure while allocating buffer."); - } - + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(); } @Override public boolean allocateNewSafe() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.allocateNewSafe(); - <#else> - /* Boolean to keep track if all the memory allocations were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; - try { - success = values.allocateNewSafe() && bits.allocateNewSafe(); - } finally { - if (!success) { - clear(); - } - } - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - return success; - + /* DELEGATE TO NEW VECTOR */ + return values.allocateNewSafe(); } @Override public void reAlloc() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.reAlloc(); - <#else> - bits.reAlloc(); - values.reAlloc(); - + /* DELEGATE TO NEW VECTOR */ + values.reAlloc(); } public void reset() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.reset(); - <#else> - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - + /* DELEGATE TO NEW VECTOR */ + values.reset(); } <#if type.major == "VarLen"> @Override public void allocateNew(int totalBytes, int valueCount) { - <#if minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(totalBytes, valueCount); - <#else> - try { - values.allocateNew(totalBytes, valueCount); - bits.allocateNew(valueCount); - } catch(RuntimeException e) { - clear(); - throw e; - } - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(totalBytes, valueCount); } @Override @@ -411,21 +247,8 @@ public int getCurrentSizeInBytes(){ <#else> @Override public void allocateNew(int valueCount) { - <#if minor.class == "Int"> - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(valueCount); - <#else> - try { - values.allocateNew(valueCount); - bits.allocateNew(valueCount); - } catch(OutOfMemoryException e) { - clear(); - throw e; - } - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(valueCount); } /** @@ -433,13 +256,8 @@ public void allocateNew(int valueCount) { */ @Override public void zeroVector() { - <#if minor.class == "Int"> - /* DELEGATE TO NEW VECTOR */ - values.zeroVector(); - <#else> - bits.zeroVector(); - values.zeroVector(); - + /* DELEGATE TO NEW VECTOR */ + values.zeroVector(); } @@ -447,53 +265,36 @@ public void zeroVector() { @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(ref, allocator, callBack); - <#else> - return getTransferPair(ref, allocator); - + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(ref, allocator, callBack); } @Override public TransferPair getTransferPair(BufferAllocator allocator){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(allocator); - <#else> - return new TransferImpl(name, allocator); - + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(allocator); } @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(ref, allocator); - <#else> - return new TransferImpl(ref, allocator); - + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(ref, allocator); } @Override public TransferPair makeTransferPair(ValueVector to) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.makeTransferPair(to); - <#else> - return new TransferImpl((${className}) to); - + /* DELEGATE TO NEW VECTOR */ + return values.makeTransferPair(to); } - <#if minor.class == "Int" || minor.class == "VarChar"> public void transferTo(${valuesName} target) { /* DELEGATE TO NEW VECTOR */ <#if type.major == "VarLen"> @@ -512,61 +313,6 @@ public void splitAndTransferTo(int startIndex, int length, ${valuesName} target) } - <#else> - public void transferTo(${className} target){ - bits.transferTo(target.bits); - values.transferTo(target.values); - <#if type.major == "VarLen"> - target.mutator.lastSet = mutator.lastSet; - - clear(); - } - - public void splitAndTransferTo(int startIndex, int length, ${className} target) { - bits.splitAndTransferTo(startIndex, length, target.bits); - values.splitAndTransferTo(startIndex, length, target.values); - <#if type.major == "VarLen"> - target.mutator.lastSet = length - 1; - - } - - - - - <#if minor.class != "Int" && minor.class != "VarChar"> - private class TransferImpl implements TransferPair { - ${className} to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new ${className}(ref, field.getFieldType(), allocator); - } - - public TransferImpl(${className} to){ - this.to = to; - } - - @Override - public ${className} getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, ${className}.this); - } - } - - @Override @@ -580,8 +326,6 @@ public Mutator getMutator(){ } - - <#if minor.class == "Int" || minor.class == "VarChar"> public void copyFrom(int fromIndex, int thisIndex, ${valuesName} from) { /* DELEGATE TO NEW VECTOR */ values.copyFrom(fromIndex, thisIndex, from); @@ -591,107 +335,43 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from) { /* DELEGATE TO NEW VECTOR */ values.copyFromSafe(fromIndex, thisIndex, from); } - <#else> - public void copyFrom(int fromIndex, int thisIndex, ${className} from) { - final Accessor fromAccessor = from.getAccessor(); - if (!fromAccessor.isNull(fromIndex)) { - mutator.set(thisIndex, fromAccessor.get(fromIndex)); - } - <#if type.major == "VarLen">mutator.lastSet = thisIndex; - } - - public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from){ - <#if type.major == "VarLen"> - mutator.fillEmpties(thisIndex); - - values.copyFromSafe(fromIndex, thisIndex, from); - bits.getMutator().setSafeToOne(thisIndex); - <#if type.major == "VarLen">mutator.lastSet = thisIndex; - } - - public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){ - <#if type.major == "VarLen"> - mutator.fillEmpties(thisIndex); - - bits.copyFromSafe(fromIndex, thisIndex, from.bits); - values.copyFromSafe(fromIndex, thisIndex, from.values); - <#if type.major == "VarLen">mutator.lastSet = thisIndex; - } - @Override public long getValidityBufferAddress() { - /* address of the databuffer associated with the bitVector */ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getValidityBufferAddress(); - <#else> - return (bits.getDataBuffer().memoryAddress()); - + /* DELEGATE TO NEW VECTOR */ + return values.getValidityBufferAddress(); } @Override public long getDataBufferAddress() { - /* address of the dataBuffer associated with the valueVector */ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getDataBufferAddress(); - <#else> - return (bits.getDataBuffer().memoryAddress()); - + /* DELEGATE TO NEW VECTOR */ + return values.getDataBufferAddress(); } @Override public long getOffsetBufferAddress() { - /* address of the dataBuffer associated with the offsetVector - * this operation is not supported for fixed-width vector types. - */ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getOffsetBufferAddress(); - <#else> - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - return (values.getOffsetAddr()); - - + /* DELEGATE TO NEW VECTOR */ + return values.getOffsetBufferAddress(); } @Override public ArrowBuf getValidityBuffer() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getValidityBuffer(); - <#else> - return (bits.getDataBuffer()); - + /* DELEGATE TO NEW VECTOR */ + return values.getValidityBuffer(); } @Override public ArrowBuf getDataBuffer() { - /* dataBuffer associated with the valueVector */ return (values.getDataBuffer()); } @Override public ArrowBuf getOffsetBuffer() { - /* dataBuffer associated with the offsetVector of the valueVector */ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getOffsetBuffer(); - <#else> - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - return (values.getOffsetBuffer()); - - + /* DELEGATE TO NEW VECTOR */ + return values.getOffsetBuffer(); } public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor { - final BitVector.Accessor bAccessor = bits.getAccessor(); - final ${valuesName}.Accessor vAccessor = values.getAccessor(); /** * Get the element at the specified position. @@ -699,119 +379,67 @@ public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.m * @param index position of the value * @return value of the element, if not null */ - <#if minor.class == "Int" || minor.class == "VarChar"> - public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.get(index); - } - <#else> - - public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { - if (isNull(index)) { - throw new IllegalStateException("Can't get a null value"); - } - return vAccessor.get(index); - } - + public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { + /* DELEGATE TO NEW VECTOR */ + return values.get(index); + } @Override public boolean isNull(int index) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.isNull(index); - <#else> - return isSet(index) == 0; - + /* DELEGATE TO NEW VECTOR */ + return values.isNull(index); } public int isSet(int index){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.isSet(index); - <#else> - return bAccessor.get(index); - + /* DELEGATE TO NEW VECTOR */ + return values.isSet(index); } <#if type.major == "VarLen"> public long getStartEnd(int index){ - <#if minor.class == "VarChar"> /* DELEGATE TO NEW VECTOR */ return values.getStartEnd(index); - <#else> - return vAccessor.getStartEnd(index); - } @Override public int getValueLength(int index) { - <#if minor.class == "VarChar"> /* DELEGATE TO NEW VECTOR */ return values.getValueLength(index); - <#else> - return values.getAccessor().getValueLength(index); - } - <#if minor.class == "Int" || minor.class == "VarChar"> public void get(int index, Nullable${minor.class}Holder holder){ /* DELEGATE TO NEW VECTOR */ values.get(index, holder); } - <#else> - public void get(int index, Nullable${minor.class}Holder holder){ - vAccessor.get(index, holder); - holder.isSet = bAccessor.get(index); - } - - <#if minor.class == "Int" || minor.class == "VarChar"> @Override public ${friendlyType} getObject(int index) { /* DELEGATE TO NEW VECTOR */ return values.getObject(index); } - <#else> - @Override - public ${friendlyType} getObject(int index) { - if (isNull(index)) { - return null; - }else{ - return vAccessor.getObject(index); - } - } - <#if minor.class == "IntervalYear" || minor.class == "IntervalDay"> public StringBuilder getAsStringBuilder(int index) { - if (isNull(index)) { - return null; - }else{ - return vAccessor.getAsStringBuilder(index); - } + /* DELEGATE TO NEW VECTOR */ + return values.getAsStringBuilder(index); } @Override public int getValueCount(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getValueCount(); - <#else> - return bits.getAccessor().getValueCount(); - + /* DELEGATE TO NEW VECTOR */ + return values.getValueCount(); } - public void reset(){} + public void reset() { } } public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator { private int setCount; <#if type.major = "VarLen"> private int lastSet = -1; - private Mutator(){ - } + private Mutator() { } public ${valuesName} getVectorWithValues() { return values; @@ -819,12 +447,9 @@ private Mutator(){ @Override - public void setIndexDefined(int index){ - <#if minor.class == "Int" || minor.class == "VarChar"> + public void setIndexDefined(int index) { + /* DELEGATE TO NEW VECTOR */ values.setIndexDefined(index); - <#else> - bits.getMutator().setToOne(index); - } @@ -835,32 +460,14 @@ public void setIndexDefined(int index){ * @param index position of the bit to set * @param value array of bytes (or int if smaller than 4 bytes) to write */ - - <#if minor.class == "Int" || minor.class == "VarChar"> public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType} value) { /* DELEGATE TO NEW VECTOR */ values.set(index, value); } - <#else> - public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType} value) { - setCount++; - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - final BitVector.Mutator bitsMutator = bits.getMutator(); - <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.set(i, emptyByteArray); - } - - bitsMutator.setToOne(index); - valuesMutator.set(index, value); - <#if type.major == "VarLen">lastSet = index; - } - <#if type.major == "VarLen"> - <#if minor.class == "VarChar"> public void fillEmpties(int index) { /* DELEGATE TO NEW VECTOR */ values.fillEmpties(index); @@ -871,203 +478,55 @@ public void setValueLengthSafe(int index, int length) { /* DELEGATE TO NEW VECTOR */ values.setValueLengthSafe(index, length); } - - <#else> - public void fillEmpties(int index){ - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.setSafe(i, emptyByteArray); - } - while(index > bits.getValueCapacity()) { - bits.reAlloc(); - } - lastSet = index - 1; - } - - @Override - public void setValueLengthSafe(int index, int length) { - values.getMutator().setValueLengthSafe(index, length); - lastSet = index; - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void setSafe(int index, byte[] value, int start, int length) { /* DELEGATE TO NEW VECTOR */ values.setSafe(index, value, start, length); } - <#else> - public void setSafe(int index, byte[] value, int start, int length) { - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - fillEmpties(index); - - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value, start, length); - setCount++; - <#if type.major == "VarLen">lastSet = index; - - } - - - <#if minor.class == "VarChar"> public void setSafe(int index, ByteBuffer value, int start, int length) { /* DELEGATE TO NEW VECTOR */ values.setSafe(index, value, start, length); } - <#else> - public void setSafe(int index, ByteBuffer value, int start, int length) { - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - fillEmpties(index); - - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value, start, length); - setCount++; - <#if type.major == "VarLen">lastSet = index; - - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void setNull(int index) { /* DELEGATE TO NEW VECTOR */ values.setNull(index); } - <#else> - public void setNull(int index){ - bits.getMutator().setSafe(index, 0); - } - - - - - <#if minor.class != "Int" && minor.class != "VarChar"> - /* these methods are probably not needed */ - public void setSkipNull(int index, ${minor.class}Holder holder){ - values.getMutator().set(index, holder); - } - - public void setSkipNull(int index, Nullable${minor.class}Holder holder){ - values.getMutator().set(index, holder); - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void set(int index, Nullable${minor.class}Holder holder) { /* DELEGATE TO NEW VECTOR */ values.set(index, holder); } - <#else> - public void set(int index, Nullable${minor.class}Holder holder) { - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.set(i, emptyByteArray); - } - - bits.getMutator().set(index, holder.isSet); - valuesMutator.set(index, holder); - <#if type.major == "VarLen">lastSet = index; - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void set(int index, ${minor.class}Holder holder) { /* DELEGATE TO NEW VECTOR */ values.set(index, holder); } - <#else> - public void set(int index, ${minor.class}Holder holder) { - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.set(i, emptyByteArray); - } - - bits.getMutator().setToOne(index); - valuesMutator.set(index, holder); - <#if type.major == "VarLen">lastSet = index; - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public boolean isSafe(int outIndex) { /* DELEGATE TO NEW VECTOR */ return values.isSafe(outIndex); } - <#else> - public boolean isSafe(int outIndex) { - return outIndex < ${className}.this.getValueCapacity(); - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> - <#if minor.class == "Int"> - public void set(int index, int isSet, int valueField) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, isSet, valueField); - } - public void setSafe(int index, int isSet, int valueField) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, isSet, valueField); - } - - <#if minor.class == "VarChar"> - public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, isSet, startField, endField, bufferField); - } - public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, isSet, startField, endField, bufferField); - } - - <#else> <#assign fields = minor.fields!type.fields /> public void set(int index, int isSet<#list fields as field>, ${field.type} ${field.name}Field ){ - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.set(i, emptyByteArray); - } - - bits.getMutator().set(index, isSet); - valuesMutator.set(index<#list fields as field><#if field.include!true >, ${field.name}Field); - <#if type.major == "VarLen">lastSet = index; + values.set(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field); } public void setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field ) { - <#if type.major == "VarLen"> - fillEmpties(index); - - bits.getMutator().setSafe(index, isSet); - values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field); - setCount++; - <#if type.major == "VarLen">lastSet = index; + values.setSafe(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field); } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void setSafe(int index, Nullable${minor.class}Holder value) { /* DELEGATE TO NEW VECTOR */ values.setSafe(index, value); @@ -1077,40 +536,12 @@ public void setSafe(int index, ${minor.class}Holder value) { /* DELEGATE TO NEW VECTOR */ values.setSafe(index, value); } - <#else> - public void setSafe(int index, Nullable${minor.class}Holder value) { - <#if type.major == "VarLen"> - fillEmpties(index); - - bits.getMutator().setSafe(index, value.isSet); - values.getMutator().setSafe(index, value); - setCount++; - <#if type.major == "VarLen">lastSet = index; - } - - public void setSafe(int index, ${minor.class}Holder value) { - <#if type.major == "VarLen"> - fillEmpties(index); - - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value); - setCount++; - <#if type.major == "VarLen">lastSet = index; - } - - <#if !(type.major == "VarLen" || minor.class == "IntervalDay")> public void setSafe(int index, ${minor.javaType!type.javaType} value) { - <#if minor.class == "Int"> - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value); - <#else> - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value); - setCount++; - + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value); } @@ -1118,93 +549,45 @@ public void setSafe(int index, ${minor.javaType!type.javaType} value) { <#if minor.class == "Decimal"> public void set(int index, ${friendlyType} value) { - bits.getMutator().setToOne(index); - values.getMutator().set(index, value); + /* DELEGATE TO NEW VECTOR */ + values.set(index, value); } public void setSafe(int index, ${friendlyType} value) { - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value); - setCount++; + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value); } - - <#if minor.class == "Int" || minor.class == "VarChar"> @Override public void setValueCount(int valueCount) { /* DELEGATE TO NEW VECTOR */ values.setValueCount(valueCount); } - <#else> - @Override - public void setValueCount(int valueCount) { - assert valueCount >= 0; - <#if type.major == "VarLen"> - fillEmpties(valueCount); - - values.getMutator().setValueCount(valueCount); - bits.getMutator().setValueCount(valueCount); - } - - - <#if minor.class != "Int" && minor.class != "VarChar"> /* THIS METHOD IS PROBABLY NOT NEEDED FOR NEW VECTORS */ @Override - public void generateTestData(int valueCount){ - bits.getMutator().generateTestDataAlt(valueCount); - values.getMutator().generateTestData(valueCount); - <#if type.major = "VarLen">lastSet = valueCount; - setValueCount(valueCount); - } - + public void generateTestData(int valueCount) { } - - <#if minor.class != "Int" && minor.class != "VarChar"> /* MUTATOR RESET IS NOT NEEDED FOR NEW VECTORS */ @Override - public void reset(){ - setCount = 0; - <#if type.major = "VarLen">lastSet = -1; - } - - + public void reset() { } - <#if minor.class == "VarChar"> + <#if type.major == "VarLen"> public void setLastSet(int value) { /* DELEGATE TO NEW VECTOR */ values.setLastSet(value); } - <#else> - public void setLastSet(int value) { - <#if type.major = "VarLen"> - lastSet = value; - <#else> - throw new UnsupportedOperationException(); - - } - - - <#if minor.class == "VarChar"> public int getLastSet() { /* DELEGATE TO NEW VECTOR */ return values.getLastSet(); } - <#else> - public int getLastSet() { - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - return lastSet; - - } } } diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java index 9fe41d0d963..8ac23fe46f7 100644 --- a/java/vector/src/main/codegen/templates/UnionListWriter.java +++ b/java/vector/src/main/codegen/templates/UnionListWriter.java @@ -36,11 +36,11 @@ public class UnionListWriter extends AbstractFieldWriter { private ListVector vector; - private UInt4Vector offsets; private PromotableWriter writer; private boolean inMap = false; private String mapName; private int lastIndex = 0; + private static final int OFFSET_WIDTH = 4; public UnionListWriter(ListVector vector) { this(vector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance()); @@ -49,7 +49,6 @@ public UnionListWriter(ListVector vector) { public UnionListWriter(ListVector vector, NullableMapWriterFactory nullableMapWriterFactory) { this.vector = vector; this.writer = new PromotableWriter(vector.getDataVector(), vector, nullableMapWriterFactory); - this.offsets = vector.getOffsetVector(); } public UnionListWriter(ListVector vector, AbstractFieldWriter parent) { @@ -72,7 +71,7 @@ public Field getField() { } public void setValueCount(int count) { - vector.getMutator().setValueCount(count); + vector.setValueCount(count); } @Override @@ -133,13 +132,13 @@ public MapWriter map(String name) { @Override public void startList() { - vector.getMutator().startNewValue(idx()); - writer.setPosition(offsets.getAccessor().get(idx() + 1)); + vector.startNewValue(idx()); + writer.setPosition(vector.getOffsetBuffer().getInt((idx() + 1) * OFFSET_WIDTH)); } @Override public void endList() { - offsets.getMutator().set(idx() + 1, writer.idx()); + vector.getOffsetBuffer().setInt((idx() + 1) * OFFSET_WIDTH, writer.idx()); setPosition(idx() + 1); } diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 3c7ed019019..3da383fe6f3 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -25,6 +25,7 @@ <#include "/@includes/vv_imports.ftl" /> import com.google.common.collect.ImmutableList; +import io.netty.buffer.ArrowBuf; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -32,6 +33,9 @@ import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.memory.BaseAllocator; +import org.apache.arrow.vector.BaseValueVector; +import org.apache.arrow.vector.util.OversizedAllocationException; import static org.apache.arrow.vector.types.UnionMode.Sparse; @@ -56,12 +60,10 @@ public class UnionVector implements FieldVector { private String name; private BufferAllocator allocator; - private Accessor accessor = new Accessor(); - private Mutator mutator = new Mutator(); int valueCount; MapVector internalMap; - UInt1Vector typeVector; + ArrowBuf typeBuffer; private NullableMapVector mapVector; private ListVector listVector; @@ -71,16 +73,17 @@ public class UnionVector implements FieldVector { private int singleType = 0; private ValueVector singleVector; + private static final byte TYPE_WIDTH = 1; private final CallBack callBack; - private final List innerVectors; + private int typeBufferAllocationSizeInBytes; public UnionVector(String name, BufferAllocator allocator, CallBack callBack) { this.name = name; this.allocator = allocator; this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); - this.typeVector = new UInt1Vector("types", allocator); + this.typeBuffer = allocator.getEmpty(); this.callBack = callBack; - this.innerVectors = Collections.unmodifiableList(Arrays.asList(typeVector)); + this.typeBufferAllocationSizeInBytes = BaseValueVector.INITIAL_VALUE_ALLOCATION * TYPE_WIDTH; } public BufferAllocator getAllocator() { @@ -104,20 +107,33 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - // truncate types vector buffer to size (#0) - org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 0, typeVector.getBufferSizeFor(fieldNode.getLength())); - BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); + if (ownBuffers.size() != 1) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + } + + ArrowBuf buffer = ownBuffers.get(0); + typeBuffer.release(); + typeBuffer = buffer.retain(allocator); + typeBufferAllocationSizeInBytes = typeBuffer.capacity(); this.valueCount = fieldNode.getLength(); } @Override public List getFieldBuffers() { - return BaseDataValueVector.unload(getFieldInnerVectors()); + List result = new ArrayList<>(1); + + typeBuffer.readerIndex(0); + typeBuffer.writerIndex(valueCount * TYPE_WIDTH); + + result.add(typeBuffer); + + return result; } @Override + @Deprecated public List getFieldInnerVectors() { - return this.innerVectors; + throw new UnsupportedOperationException("There are no inner vectors. Use geFieldBuffers"); } private String fieldName(MinorType type) { @@ -134,7 +150,7 @@ private T addOrGet(MinorType minorType, Class c) { @Override public long getValidityBufferAddress() { - return typeVector.getDataBuffer().memoryAddress(); + return typeBuffer.memoryAddress(); } @Override @@ -148,7 +164,7 @@ public long getOffsetBufferAddress() { } @Override - public ArrowBuf getValidityBuffer() { return typeVector.getDataBuffer(); } + public ArrowBuf getValidityBuffer() { return typeBuffer; } @Override public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); } @@ -211,47 +227,76 @@ public ListVector getList() { } public int getTypeValue(int index) { - return typeVector.getAccessor().get(index); - } - - public UInt1Vector getTypeVector() { - return typeVector; + return typeBuffer.getByte(index * TYPE_WIDTH); } @Override public void allocateNew() throws OutOfMemoryException { internalMap.allocateNew(); - typeVector.allocateNew(); - if (typeVector != null) { - typeVector.zeroVector(); + try { + allocateTypeBuffer(); + } catch (Exception e) { + clear(); + throw e; } } @Override public boolean allocateNewSafe() { boolean safe = internalMap.allocateNewSafe(); - safe = safe && typeVector.allocateNewSafe(); - if (safe) { - if (typeVector != null) { - typeVector.zeroVector(); - } + if (!safe) { return false; } + try { + allocateTypeBuffer(); + } catch (Exception e) { + clear(); + return false; } - return safe; + + return true; + } + + private void allocateTypeBuffer() { + typeBuffer = allocator.buffer(typeBufferAllocationSizeInBytes); + typeBuffer.readerIndex(0); + typeBuffer.setZero(0, typeBuffer.capacity()); } @Override public void reAlloc() { internalMap.reAlloc(); - typeVector.reAlloc(); + reallocTypeBuffer(); } - @Override - public void setInitialCapacity(int numRecords) { + private void reallocTypeBuffer() { + final int currentBufferCapacity = typeBuffer.capacity(); + long baseSize = typeBufferAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, typeBuffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + typeBuffer.release(1); + typeBuffer = newBuf; + typeBufferAllocationSizeInBytes = (int)newAllocationSize; } + @Override + public void setInitialCapacity(int numRecords) { } + @Override public int getValueCapacity() { - return Math.min(typeVector.getValueCapacity(), internalMap.getValueCapacity()); + return Math.min(getTypeBufferValueCapacity(), internalMap.getValueCapacity()); } @Override @@ -261,7 +306,9 @@ public void close() { @Override public void clear() { - typeVector.clear(); + valueCount = 0; + typeBuffer.release(); + typeBuffer = allocator.getEmpty(); internalMap.clear(); } @@ -300,7 +347,7 @@ public TransferPair makeTransferPair(ValueVector target) { public void copyFrom(int inIndex, int outIndex, UnionVector from) { from.getReader().setPosition(inIndex); getWriter().setPosition(outIndex); - ComplexCopier.copy(from.reader, mutator.writer); + ComplexCopier.copy(from.reader, writer); } public void copyFromSafe(int inIndex, int outIndex, UnionVector from) { @@ -321,33 +368,34 @@ public FieldVector addVector(FieldVector v) { private class TransferImpl implements TransferPair { private final TransferPair internalMapVectorTransferPair; - private final TransferPair typeVectorTransferPair; private final UnionVector to; public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { to = new UnionVector(name, allocator, callBack); internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap); - typeVectorTransferPair = typeVector.makeTransferPair(to.typeVector); } public TransferImpl(UnionVector to) { this.to = to; internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap); - typeVectorTransferPair = typeVector.makeTransferPair(to.typeVector); } @Override public void transfer() { + to.clear(); + to.typeBuffer = typeBuffer.transferOwnership(to.allocator).buffer; internalMapVectorTransferPair.transfer(); - typeVectorTransferPair.transfer(); to.valueCount = valueCount; } @Override public void splitAndTransfer(int startIndex, int length) { + to.clear(); internalMapVectorTransferPair.splitAndTransfer(startIndex, length); - typeVectorTransferPair.splitAndTransfer(startIndex, length); - to.getMutator().setValueCount(length); + final int startPoint = startIndex * TYPE_WIDTH; + final int sliceLength = length * TYPE_WIDTH; + to.typeBuffer = typeBuffer.slice(startPoint, sliceLength).transferOwnership(to.allocator).buffer; + to.setValueCount(length); } @Override @@ -362,13 +410,15 @@ public void copyValueSafe(int from, int to) { } @Override + @Deprecated public Accessor getAccessor() { - return accessor; + throw new UnsupportedOperationException("Accessor is not supported for reading from UNION"); } @Override + @Deprecated public Mutator getMutator() { - return mutator; + throw new UnsupportedOperationException("Mutator is not supported for writing to UNION"); } @Override @@ -380,15 +430,17 @@ public FieldReader getReader() { } public FieldWriter getWriter() { - if (mutator.writer == null) { - mutator.writer = new UnionWriter(this); + if (writer == null) { + writer = new UnionWriter(this); } - return mutator.writer; + return writer; } @Override public int getBufferSize() { - return typeVector.getBufferSize() + internalMap.getBufferSize(); + if (valueCount == 0) { return 0; } + + return (valueCount * TYPE_WIDTH) + internalMap.getBufferSize(); } @Override @@ -402,13 +454,13 @@ public int getBufferSizeFor(final int valueCount) { bufferSize += v.getBufferSizeFor(valueCount); } - return (int) bufferSize; + return (int) bufferSize + (valueCount * TYPE_WIDTH); } @Override public ArrowBuf[] getBuffers(boolean clear) { ImmutableList.Builder builder = ImmutableList.builder(); - builder.add(typeVector.getBuffers(clear)); + builder.add(typeBuffer); builder.add(internalMap.getBuffers(clear)); List list = builder.build(); return list.toArray(new ArrowBuf[list.size()]); @@ -417,15 +469,12 @@ public ArrowBuf[] getBuffers(boolean clear) { @Override public Iterator iterator() { List vectors = Lists.newArrayList(internalMap.iterator()); - vectors.add(typeVector); return vectors.iterator(); } - public class Accessor extends BaseValueVector.BaseAccessor { - @Override public Object getObject(int index) { - int type = typeVector.getAccessor().get(index); + int type = typeBuffer.getByte(index * TYPE_WIDTH); switch (MinorType.values()[type]) { case NULL: return null; @@ -436,18 +485,14 @@ public Object getObject(int index) { <#assign uncappedName = name?uncap_first/> <#if !minor.typeParams?? > case ${name?upper_case}: - <#if minor.class != "Int" && minor.class != "VarChar"> - return get${name}Vector().getAccessor().getObject(index); - <#else> return get${name}Vector().getObject(index); - case MAP: - return getMap().getAccessor().getObject(index); + return getMap().getObject(index); case LIST: - return getList().getAccessor().getObject(index); + return getList().getObject(index); default: throw new UnsupportedOperationException("Cannot support type: " + MinorType.values()[type]); } @@ -466,30 +511,37 @@ public void get(int index, UnionHolder holder) { holder.reader = reader; } - @Override public int getValueCount() { return valueCount; } - @Override public boolean isNull(int index) { - return typeVector.getAccessor().get(index) == 0; + return (typeBuffer.getByte(index * TYPE_WIDTH) == 0); + } + + @Override + public int getNullCount() { + int nullCount = 0; + for (int i = 0; i < getValueCount(); i++) { + if (isNull(i)) { + nullCount++; + } + } + return nullCount; } public int isSet(int index) { return isNull(index) ? 0 : 1; } - } - - public class Mutator extends BaseValueVector.BaseMutator { UnionWriter writer; - @Override public void setValueCount(int valueCount) { - UnionVector.this.valueCount = valueCount; - typeVector.getMutator().setValueCount(valueCount); - internalMap.getMutator().setValueCount(valueCount); + this.valueCount = valueCount; + while (valueCount > getTypeBufferValueCapacity()) { + reallocTypeBuffer(); + } + internalMap.setValueCount(valueCount); } public void setSafe(int index, UnionHolder holder) { @@ -534,11 +586,7 @@ public void setSafe(int index, UnionHolder holder) { <#if !minor.typeParams?? > public void setSafe(int index, Nullable${name}Holder holder) { setType(index, MinorType.${name?upper_case}); - <#if minor.class != "Int" && minor.class != "VarChar"> - get${name}Vector().getMutator().setSafe(index, holder); - <#else> - get${name}Vector().setSafe(index, holder); - + get${name}Vector().setSafe(index, holder); } @@ -546,19 +594,13 @@ public void setSafe(int index, Nullable${name}Holder holder) { public void setType(int index, MinorType type) { - typeVector.getMutator().setSafe(index, (byte) type.ordinal()); + while (index >= getTypeBufferValueCapacity()) { + reallocTypeBuffer(); + } + typeBuffer.setByte(index * TYPE_WIDTH , (byte) type.ordinal()); } - @Override - public void reset() { } - - @Override - public void generateTestData(int values) { } - } - - public int getValueCount() { return 0; } - - public void setValueCount(int valueCount) { } - - public Object getObject(int index) { return null; } + private int getTypeBufferValueCapacity() { + return (int) ((typeBuffer.capacity() * 1.0) / TYPE_WIDTH); + } } diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java index f892bac9133..526708a4c32 100644 --- a/java/vector/src/main/codegen/templates/UnionWriter.java +++ b/java/vector/src/main/codegen/templates/UnionWriter.java @@ -62,7 +62,7 @@ public void setPosition(int index) { @Override public void start() { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); getMapWriter().start(); } @@ -74,7 +74,7 @@ public void end() { @Override public void startList() { getListWriter().startList(); - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); } @Override @@ -92,7 +92,7 @@ private MapWriter getMapWriter() { } public MapWriter asMap() { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); return getMapWriter(); } @@ -106,7 +106,7 @@ private ListWriter getListWriter() { } public ListWriter asList() { - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); return getListWriter(); } @@ -150,19 +150,19 @@ BaseWriter getWriter(MinorType minorType) { } public ${name}Writer as${name}() { - data.getMutator().setType(idx(), MinorType.${name?upper_case}); + data.setType(idx(), MinorType.${name?upper_case}); return get${name}Writer(); } @Override public void write(${name}Holder holder) { - data.getMutator().setType(idx(), MinorType.${name?upper_case}); + data.setType(idx(), MinorType.${name?upper_case}); get${name}Writer().setPosition(idx()); get${name}Writer().write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, ); } public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { - data.getMutator().setType(idx(), MinorType.${name?upper_case}); + data.setType(idx(), MinorType.${name?upper_case}); get${name}Writer().setPosition(idx()); get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); } @@ -175,28 +175,28 @@ public void writeNull() { @Override public MapWriter map() { - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); getListWriter().setPosition(idx()); return getListWriter().map(); } @Override public ListWriter list() { - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); getListWriter().setPosition(idx()); return getListWriter().list(); } @Override public ListWriter list(String name) { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); getMapWriter().setPosition(idx()); return getMapWriter().list(name); } @Override public MapWriter map(String name) { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); getMapWriter().setPosition(idx()); return getMapWriter().map(name); } @@ -209,14 +209,14 @@ public MapWriter map(String name) { <#if !minor.typeParams?? > @Override public ${capName}Writer ${lowerName}(String name) { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); getMapWriter().setPosition(idx()); return getMapWriter().${lowerName}(name); } @Override public ${capName}Writer ${lowerName}() { - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); getListWriter().setPosition(idx()); return getListWriter().${lowerName}(); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java index c5f781000c9..83dc409a74f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java @@ -35,12 +35,17 @@ import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; +/** + * BaseNullableFixedWidthVector provides an abstract interface for + * implementing vectors of fixed width values. The vectors are nullable + * implying that zero or more elements in the vector could be NULL. + */ public abstract class BaseNullableFixedWidthVector extends BaseValueVector implements FixedWidthVector, FieldVector { private final byte typeWidth; - private int valueAllocationSizeInBytes; - private int validityAllocationSizeInBytes; + protected int valueAllocationSizeInBytes; + protected int validityAllocationSizeInBytes; protected final Field field; private int allocationMonitor; @@ -52,23 +57,21 @@ public BaseNullableFixedWidthVector(final String name, final BufferAllocator all FieldType fieldType, final byte typeWidth) { super(name, allocator); this.typeWidth = typeWidth; - valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; - validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); field = new Field(name, fieldType, null); valueCount = 0; allocationMonitor = 0; validityBuffer = allocator.getEmpty(); valueBuffer = allocator.getEmpty(); + if (typeWidth > 0) { + valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + } else { + /* specialized handling for NullableBitVector */ + valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + validityAllocationSizeInBytes = valueAllocationSizeInBytes; + } } - /* TODO: - * Determine how writerIndex and readerIndex need to be used. Right now we - * are setting the writerIndex and readerIndex in the call to getFieldBuffers - * using the valueCount -- this assumes that the caller of getFieldBuffers - * on the vector has already invoked setValueCount. - * - * Do we need to set them during vector transfer and splitAndTransfer? - */ /* TODO: * @@ -84,57 +87,91 @@ public BaseNullableFixedWidthVector(final String name, final BufferAllocator all * abstracted out to top level base class. * * Right now BaseValueVector is the top level base class for other - * vector types in ValueVector hierarchy and those vectors have not - * yet been refactored so moving things to the top class as of now - * is not a good idea. + * vector types in ValueVector hierarchy (non-nullable) and those + * vectors have not yet been refactored/removed so moving things to + * the top class as of now is not a good idea. */ - /* TODO: - * See if we need logger -- unnecessary object probably - */ - - protected abstract org.slf4j.Logger getLogger(); @Override + @Deprecated public Mutator getMutator() { - throw new UnsupportedOperationException("Mutator is not needed to write into vector"); + throw new UnsupportedOperationException("Mutator is not supported for writing to vector"); } @Override + @Deprecated public Accessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not needed to read from vector"); + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); } + /** + * Get the memory address of buffer that manages the validity + * (NULL or NON-NULL nature) of elements in the vector. + * @return starting address of the buffer + */ @Override public long getValidityBufferAddress() { return (validityBuffer.memoryAddress()); } + /** + * Get the memory address of buffer that stores the data for elements + * in the vector. + * @return starting address of the buffer + */ @Override public long getDataBufferAddress() { return (valueBuffer.memoryAddress()); } + /** + * Get the memory address of buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return starting address of the buffer + * @throws UnsupportedOperationException for fixed width vectors + */ @Override public long getOffsetBufferAddress() { throw new UnsupportedOperationException("not supported for fixed-width vectors"); } + /** + * Get buffer that manages the validity (NULL or NON-NULL nature) of + * elements in the vector. Consider it as a buffer for internal bit vector + * data structure. + * @return buffer + */ @Override public ArrowBuf getValidityBuffer() { return validityBuffer; } + /** + * Get the buffer that stores the data for elements in the vector. + * @return buffer + */ @Override public ArrowBuf getDataBuffer() { return valueBuffer; } + /** + * buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return buffer + * @throws UnsupportedOperationException for fixed width vectors + */ @Override public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException("not supported for fixed-width vectors"); } + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ @Override public void setInitialCapacity(int valueCount) { final long size = (long)valueCount * typeWidth; @@ -142,52 +179,65 @@ public void setInitialCapacity(int valueCount) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } valueAllocationSizeInBytes = (int)size; - validityAllocationSizeInBytes = getSizeFromCount(valueCount); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); } + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ @Override public int getValueCapacity(){ return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity()); } - /* for test purposes */ private int getValueBufferValueCapacity() { return (int)((valueBuffer.capacity() * 1.0)/typeWidth); } - /* for test purposes */ private int getValidityBufferValueCapacity() { return (int)(validityBuffer.capacity() * 8L); } - /* number of bytes for the validity buffer for the given valueCount */ - protected int getSizeFromCount(int valueCount) { - return (int) Math.ceil(valueCount / 8.0); - } - + /** + * zero out the vector and the data in associated buffers. + */ @Override public void zeroVector() { initValidityBuffer(); initValueBuffer(); } + /* zero out the validity buffer */ private void initValidityBuffer() { validityBuffer.setZero(0, validityBuffer.capacity()); } + /* zero out the data buffer */ private void initValueBuffer() { valueBuffer.setZero(0, valueBuffer.capacity()); } + /** + * Reset the vector to initial state. Same as {@link #zeroVector()}. + * Note that this method doesn't release any memory. + */ public void reset() { zeroVector(); } + /** + * Close the vector and release the associated buffers. + */ @Override public void close() { clear(); } + /** + * Same as {@link #close()} + */ @Override public void clear() { + valueCount = 0; validityBuffer = releaseBuffer(validityBuffer); valueBuffer = releaseBuffer(valueBuffer); } @@ -208,6 +258,9 @@ protected void decrementAllocationMonitor() { allocationMonitor--; } + /** + * Same as {@link #allocateNewSafe()}. + */ @Override public void allocateNew() { if(!allocateNewSafe()){ @@ -215,6 +268,15 @@ public void allocateNew() { } } + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. See {@link #allocateNew(int)} for allocating memory for specific + * number of elements in the vector. + * + * @return false if memory allocation fails, true otherwise. + */ + @Override public boolean allocateNewSafe() { long curAllocationSizeValue = valueAllocationSizeInBytes; long curAllocationSizeValidity = validityAllocationSizeInBytes; @@ -229,8 +291,7 @@ public boolean allocateNewSafe() { try{ allocateBytes(curAllocationSizeValue, curAllocationSizeValidity); } catch (Exception e) { - getLogger().error("ERROR: Failure in allocateNewSafe"); - getLogger().error(e.getMessage()); + e.printStackTrace(); clear(); return false; } @@ -238,9 +299,20 @@ public boolean allocateNewSafe() { return true; } + /** + * Allocate memory for the vector to support storing at least the provided number of + * elements in the vector. This method must be called prior to using the ValueVector. + * + * @param valueCount the desired number of elements in the vector + * @throws org.apache.arrow.memory.OutOfMemoryException + */ public void allocateNew(int valueCount) { long valueBufferSize = valueCount * typeWidth; - long validityBufferSize = getSizeFromCount(valueCount); + long validityBufferSize = getValidityBufferSizeFromCount(valueCount); + if (typeWidth == 0) { + /* specialized handling for NullableBitVector */ + valueBufferSize = validityBufferSize; + } if (allocationMonitor > 10) { /* step down the default memory allocation since we have observed @@ -266,8 +338,7 @@ public void allocateNew(int valueCount) { try { allocateBytes(valueBufferSize, validityBufferSize); } catch(Exception e) { - getLogger().error("ERROR: Failure in allocateNew"); - getLogger().error(e.getMessage()); + e.printStackTrace(); clear(); throw e; } @@ -294,8 +365,8 @@ private void allocateBytes(final long valueBufferSize, final long validityBuffer initValidityBuffer(); } - /* - * during splitAndTransfer, if we splitting from a random position within a byte, + /** + * During splitAndTransfer, if we splitting from a random position within a byte, * we can't just slice the source buffer so we have to explicitly allocate the * validityBuffer of the target vector. This is unlike the databuffer which we can * always slice for the target vector. @@ -307,23 +378,49 @@ private void allocateValidityBuffer(final int validityBufferSize) { initValidityBuffer(); } + /** + * Get the potential buffer size for a particular number of records. + * @param count desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ @Override public int getBufferSizeFor(final int count) { if (count == 0) { return 0; } - return (count * typeWidth) + getSizeFromCount(count); + return (count * typeWidth) + getValidityBufferSizeFromCount(count); } + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ @Override public int getBufferSize() { if (valueCount == 0) { return 0; } - return (valueCount * typeWidth) + getSizeFromCount(valueCount); + return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount); } + /** + * Get information about how this field is materialized. + * @return the field corresponding to this vector + */ @Override public Field getField() { return field; } + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ @Override public ArrowBuf[] getBuffers(boolean clear) { final ArrowBuf[] buffers = new ArrowBuf[2]; @@ -338,12 +435,20 @@ public ArrowBuf[] getBuffers(boolean clear) { return buffers; } + /** + * Resize the vector to increase the capacity. The internal behavior is to + * double the current value capacity. + */ @Override public void reAlloc() { valueBuffer = reallocBufferHelper(valueBuffer, true); validityBuffer = reallocBufferHelper(validityBuffer, false); } + /** + * Helper method for reallocating a particular internal buffer + * Returns the new buffer. + */ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) { final int currentBufferCapacity = buffer.capacity(); long baseSize = (dataBuffer ? valueAllocationSizeInBytes @@ -360,10 +465,6 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) throw new OversizedAllocationException("Unable to expand the buffer"); } - getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", - name, (dataBuffer ? valueAllocationSizeInBytes : validityAllocationSizeInBytes), - newAllocationSize); - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); newBuf.setBytes(0, buffer, 0, currentBufferCapacity); final int halfNewCapacity = newBuf.capacity() / 2; @@ -381,8 +482,17 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) } @Override - public List getFieldInnerVectors() { throw new UnsupportedOperationException(); } + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + } + /** + * Initialize the children in schema for this Field. This operation is a + * NO-OP for scalar types since they don't have any children. + * @param children the schema + * @throws IllegalArgumentException if children is a non-empty list for scalar types. + */ @Override public void initializeChildrenFromFields(List children) { if (!children.isEmpty()) { @@ -390,11 +500,23 @@ public void initializeChildrenFromFields(List children) { } } + /** + * Get the inner child vectors. + * @return list of child vectors for complex types, empty list for scalar vector + * types + */ @Override public List getChildrenFromFields() { return Collections.emptyList(); } + /** + * Load the buffers of this vector with provided source buffers. + * The caller manages the source buffers and populates them before invoking + * this method. + * @param fieldNode the fieldNode indicating the value count + * @param ownBuffers the buffers for this Field (own buffers only, children not included) + */ @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { if (ownBuffers.size() != 2) { @@ -415,13 +537,22 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers validityAllocationSizeInBytes = validityBuffer.capacity(); } + /** + * Get the buffers belonging to this vector + * @return the inner buffers. + */ public List getFieldBuffers() { List result = new ArrayList<>(2); validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getSizeFromCount(valueCount)); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); valueBuffer.readerIndex(0); - valueBuffer.writerIndex(valueCount * typeWidth); + if (typeWidth == 0) { + /* specialized handling for NullableBitVector */ + valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + } else { + valueBuffer.writerIndex(valueCount * typeWidth); + } result.add(validityBuffer); result.add(valueBuffer); @@ -429,18 +560,42 @@ public List getFieldBuffers() { return result; } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @param callBack + * @return TransferPair + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return getTransferPair(ref, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param allocator allocator for the target vector + * @return TransferPair + */ @Override public TransferPair getTransferPair(BufferAllocator allocator){ return getTransferPair(name, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return TransferPair + */ public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + /** + * Transfer this vector'data to another vector. The memory associated + * with this vector is transferred to the allocator of target vector + * for accounting and management purposes. + * @param target destination vector for transfer + */ public void transferTo(BaseNullableFixedWidthVector target){ compareTypes(target, "transferTo"); target.clear(); @@ -450,6 +605,13 @@ public void transferTo(BaseNullableFixedWidthVector target){ clear(); } + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ public void splitAndTransferTo(int startIndex, int length, BaseNullableFixedWidthVector target) { compareTypes(target, "splitAndTransferTo"); @@ -459,6 +621,9 @@ public void splitAndTransferTo(int startIndex, int length, target.setValueCount(length); } + /** + * Data buffer can always be split and transferred using slicing. + */ private void splitAndTransferValueBuffer(int startIndex, int length, BaseNullableFixedWidthVector target) { final int startPoint = startIndex * typeWidth; @@ -466,17 +631,21 @@ private void splitAndTransferValueBuffer(int startIndex, int length, target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; } + /** + * Validity buffer has multiple cases of split and transfer depending on + * the starting position of the source index. + */ private void splitAndTransferValidityBuffer(int startIndex, int length, BaseNullableFixedWidthVector target) { assert startIndex + length <= valueCount; int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getSizeFromCount(length); + int byteSizeTarget = getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { if (offset == 0) { - // slice + /* slice */ if (target.validityBuffer != null) { target.validityBuffer.release(); } @@ -493,8 +662,10 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, target.allocateValidityBuffer(byteSizeTarget); for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); - byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, + firstByteSource + i + 1, offset); target.validityBuffer.setByte(i, (b1 + b2)); } @@ -509,15 +680,15 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, * by shifting data from the current byte. */ if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = getBitsFromNextByte(this.validityBuffer, + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + byteSizeTarget, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); } else { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1); } @@ -525,14 +696,6 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, } } - private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) & 0xFF) >>> offset); - } - - private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) << (8 - offset))); - } - /****************************************************************** * * @@ -547,27 +710,9 @@ private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { * @return the number of null elements. */ public int getNullCount() { - int count = 0; - final int sizeInBytes = getSizeFromCount(valueCount); - - for (int i = 0; i < sizeInBytes; ++i) { - final byte byteValue = validityBuffer.getByte(i); - /* Java uses two's complement binary representation, hence 11111111_b which is -1 - * when converted to Int will have 32bits set to 1. Masking the MSB and then - * adding it back solves the issue. - */ - count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); - } - int nullCount = (sizeInBytes * 8) - count; - /* if the valueCount is not a multiple of 8, - * the bits on the right were counted as null bits. - */ - int remainder = valueCount % 8; - nullCount -= remainder == 0 ? 0 : 8 - remainder; - return nullCount; + return BitVectorHelper.getNullCount(validityBuffer, valueCount); } - /** * Get the value count of vector. This will always be zero unless * {@link #setValueCount(int)} has been called prior to calling this. @@ -578,7 +723,6 @@ public int getValueCount(){ return valueCount; } - /** * Set value count for the vector. * @@ -625,7 +769,6 @@ public void setValueCount(int valueCount) { } } - /** * Check if the given index is within the current value capacity * of the vector @@ -637,7 +780,6 @@ public boolean isSafe(int index) { return index < getValueCapacity(); } - /** * Check if element at given index is null. * @@ -648,7 +790,6 @@ public boolean isNull(int index) { return (isSet(index) == 0); } - /** * Same as {@link #isNull(int)}. * @@ -662,6 +803,11 @@ public int isSet(int index) { return Long.bitCount(b & (1L << bitIndex)); } + /** + * Mark the particular position in the vector as non-null. + * + * @param index position of the element. + */ public void setIndexDefined(int index) { handleSafe(index); BitVectorHelper.setValidityBitToOne(validityBuffer, index); @@ -691,7 +837,6 @@ public void setSafe(int index, ByteBuffer value, int start, int length) { ******************************************************************/ - protected void handleSafe(int index) { while (index >= getValueCapacity()) { decrementAllocationMonitor(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java index a79709da4dd..6587cde633e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -45,7 +45,7 @@ public abstract class BaseNullableVariableWidthVector extends BaseValueVector private int offsetAllocationSizeInBytes; /* protected members */ - protected static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ + public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ protected static final byte[] emptyByteArray = new byte[]{}; protected ArrowBuf validityBuffer; protected ArrowBuf valueBuffer; @@ -59,7 +59,7 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator FieldType fieldType) { super(name, allocator); valueAllocationSizeInBytes = INITIAL_BYTE_COUNT; - validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH; field = new Field(name, fieldType, null); valueCount = 0; @@ -70,21 +70,12 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator cleared = false; } - /* TODO: - * Determine how writerIndex and readerIndex need to be used. Right now we - * are setting the writerIndex and readerIndex in the call to getFieldBuffers - * using the valueCount -- this assumes that the caller of getFieldBuffers - * on the vector has already invoked setValueCount. - * - * Do we need to set them during vector transfer and splitAndTransfer? - */ - /* TODO: * * see if getNullCount() can be made faster -- O(1) */ - /* TODO: + /* TODO: * Once the entire hierarchy has been refactored, move common functions * like getNullCount(), splitAndTransferValidityBuffer to top level * base class BaseValueVector. @@ -93,59 +84,89 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator * abstracted out to top level base class. * * Right now BaseValueVector is the top level base class for other - * vector types in ValueVector hierarchy and those vectors have not - * yet been refactored so moving things to the top class as of now - * is not a good idea. - */ - - /* TODO: - * See if we need logger -- unnecessary object probably + * vector types in ValueVector hierarchy (non-nullable) and those + * vectors have not yet been refactored/removed so moving things to + * the top class as of now is not a good idea. */ - /* TODO: - * Implement getBufferSize(), getCurrentSizeInBytes(). - */ - - protected abstract org.slf4j.Logger getLogger(); + @Override + @Deprecated public VariableWidthMutator getMutator() { - throw new UnsupportedOperationException("Mutator is not needed to write into vector"); + throw new UnsupportedOperationException("Mutator is not supported for writing into vector"); } + @Override + @Deprecated public VariableWidthAccessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not needed to read from vector"); + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); } + /** + * Get buffer that manages the validity (NULL or NON-NULL nature) of + * elements in the vector. Consider it as a buffer for internal bit vector + * data structure. + * @return buffer + */ @Override public ArrowBuf getValidityBuffer() { return validityBuffer; } + /** + * Get the buffer that stores the data for elements in the vector. + * @return buffer + */ @Override public ArrowBuf getDataBuffer() { return valueBuffer; } + /** + * buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return buffer + */ @Override public ArrowBuf getOffsetBuffer() { return offsetBuffer; } + /** + * Get the memory address of buffer that stores the offsets for elements + * in the vector. + * @return starting address of the buffer + */ @Override public long getOffsetBufferAddress() { return offsetBuffer.memoryAddress(); } + /** + * Get the memory address of buffer that manages the validity + * (NULL or NON-NULL nature) of elements in the vector. + * @return starting address of the buffer + */ @Override public long getValidityBufferAddress() { return validityBuffer.memoryAddress(); } + /** + * Get the memory address of buffer that stores the data for elements + * in the vector. + * @return starting address of the buffer + */ @Override public long getDataBufferAddress() { return valueBuffer.memoryAddress(); } + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ @Override public void setInitialCapacity(int valueCount) { final long size = (long)valueCount * DEFAULT_RECORD_BYTE_COUNT; @@ -153,57 +174,69 @@ public void setInitialCapacity(int valueCount) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } valueAllocationSizeInBytes = (int)size; - validityAllocationSizeInBytes = getSizeFromCount(valueCount); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); /* to track the end offset of last data element in vector, we need * an additional slot in offset buffer. */ offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH; } + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ @Override public int getValueCapacity(){ final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); return Math.min(offsetValueCapacity, getValidityBufferValueCapacity()); } - /* for test purposes */ private int getValidityBufferValueCapacity() { return (int)(validityBuffer.capacity() * 8L); } - /* for test purposes */ private int getOffsetBufferValueCapacity() { return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); } - /* number of bytes for the validity buffer for a given valueCount */ - protected int getSizeFromCount(int valueCount) { - return (int) Math.ceil(valueCount / 8.0); - } - + /** + * zero out the vector and the data in associated buffers. + */ public void zeroVector() { initValidityBuffer(); initOffsetBuffer(); } + /* zero out the validity buffer */ private void initValidityBuffer() { validityBuffer.setZero(0, validityBuffer.capacity()); } + /* zero out the offset buffer */ private void initOffsetBuffer() { offsetBuffer.setZero(0, offsetBuffer.capacity()); } + /** + * Reset the vector to initial state. Same as {@link #zeroVector()}. + * Note that this method doesn't release any memory. + */ public void reset() { zeroVector(); lastSet = -1; } + /** + * Close the vector and release the associated buffers. + */ @Override public void close() { clear(); } + /** + * Same as {@link #close()} + */ @Override public void clear() { validityBuffer = releaseBuffer(validityBuffer); @@ -215,8 +248,17 @@ public void clear() { } @Override - public List getFieldInnerVectors() { throw new UnsupportedOperationException(); } + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + } + /** + * Initialize the children in schema for this Field. This operation is a + * NO-OP for scalar types since they don't have any children. + * @param children the schema + * @throws IllegalArgumentException if children is a non-empty list for scalar types. + */ @Override public void initializeChildrenFromFields(List children) { if (!children.isEmpty()) { @@ -224,11 +266,24 @@ public void initializeChildrenFromFields(List children) { } } + /** + * Get the inner child vectors. + * @return list of child vectors for complex types, empty list for scalar vector + * types + */ @Override public List getChildrenFromFields() { return Collections.emptyList(); } + + /** + * Load the buffers of this vector with provided source buffers. + * The caller manages the source buffers and populates them before invoking + * this method. + * @param fieldNode the fieldNode indicating the value count + * @param ownBuffers the buffers for this Field (own buffers only, children not included) + */ @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { ArrowBuf bitBuffer = ownBuffers.get(0); @@ -246,11 +301,15 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers valueCount = fieldNode.getLength(); } + /** + * Get the buffers belonging to this vector + * @return the inner buffers. + */ public List getFieldBuffers() { List result = new ArrayList<>(3); final int lastDataOffset = getstartOffset(valueCount); validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getSizeFromCount(valueCount)); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); offsetBuffer.readerIndex(0); offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); valueBuffer.readerIndex(0); @@ -263,6 +322,9 @@ public List getFieldBuffers() { return result; } + /** + * Same as {@link #allocateNewSafe()}. + */ @Override public void allocateNew() { if(!allocateNewSafe()){ @@ -270,6 +332,14 @@ public void allocateNew() { } } + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. See {@link #allocateNew(int, int)} for allocating memory for specific + * number of elements in the vector. + * + * @return false if memory allocation fails, true otherwise. + */ @Override public boolean allocateNewSafe() { long curAllocationSizeValue = valueAllocationSizeInBytes; @@ -287,8 +357,7 @@ public boolean allocateNewSafe() { try { allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset); } catch (Exception e) { - getLogger().error("ERROR: Failure in allocateNewSafe"); - getLogger().error(e.getMessage()); + e.printStackTrace(); clear(); return false; } @@ -296,11 +365,19 @@ public boolean allocateNewSafe() { return true; } + /** + * Allocate memory for the vector to support storing at least the provided number of + * elements in the vector. This method must be called prior to using the ValueVector. + * + * @param totalBytes desired total memory capacity + * @param valueCount the desired number of elements in the vector + * @throws org.apache.arrow.memory.OutOfMemoryException + */ @Override public void allocateNew(int totalBytes, int valueCount) { assert totalBytes >= 0; final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; - final int validityBufferSize = getSizeFromCount(valueCount); + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); if (totalBytes > MAX_ALLOCATION_SIZE || offsetBufferSize > MAX_ALLOCATION_SIZE) { @@ -313,12 +390,12 @@ public void allocateNew(int totalBytes, int valueCount) { try { allocateBytes(totalBytes, validityBufferSize, offsetBufferSize); } catch (Exception e) { - getLogger().error("ERROR: Failure in allocateNewSafe"); - getLogger().error(e.getMessage()); + e.printStackTrace(); clear(); } } + /* allocate the inner buffers */ private void allocateBytes(final long valueBufferSize, final long validityBufferSize, final long offsetBufferSize) { /* allocate data buffer */ @@ -330,6 +407,7 @@ private void allocateBytes(final long valueBufferSize, final long validityBuffer allocateOffsetBuffer(offsetBufferSize); } + /* allocate offset buffer */ private void allocateOffsetBuffer(final long size) { final int curSize = (int)size; offsetBuffer = allocator.buffer(curSize); @@ -338,6 +416,7 @@ private void allocateOffsetBuffer(final long size) { initOffsetBuffer(); } + /* allocate validity buffer */ private void allocateValidityBuffer(final long size) { final int curSize = (int)size; validityBuffer = allocator.buffer(curSize); @@ -346,6 +425,10 @@ private void allocateValidityBuffer(final long size) { initValidityBuffer(); } + /** + * Resize the vector to increase the capacity. The internal behavior is to + * double the current value capacity. + */ public void reAlloc() { reallocValueBuffer(); reallocValidityAndOffsetBuffers(); @@ -378,10 +461,7 @@ protected void reallocValidityAndOffsetBuffers() { validityBuffer = reallocBufferHelper(validityBuffer, false); } - /* need to refactor this to keep the logic in an single place and make callers - * more intelligent. see handleSafe() for more comments on realloc - */ - + /* helper method to realloc a particular buffer. returns the allocated buffer */ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) { final int currentBufferCapacity = buffer.capacity(); long baseSize = (offsetBuffer ? offsetAllocationSizeInBytes @@ -398,10 +478,6 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer throw new OversizedAllocationException("Unable to expand the buffer"); } - getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", - name, (offsetBuffer ? offsetAllocationSizeInBytes : validityAllocationSizeInBytes), - newAllocationSize); - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); newBuf.setBytes(0, buffer, 0, currentBufferCapacity); final int halfNewCapacity = newBuf.capacity() / 2; @@ -418,6 +494,10 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer return buffer; } + /** + * Get the size (number of bytes) of underlying data buffer. + * @return + */ @Override public int getByteCapacity(){ return valueBuffer.capacity(); @@ -429,30 +509,55 @@ public int getCurrentSizeInBytes(){ return 0; } + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ @Override public int getBufferSize() { - /* TODO */ - return 0; + return getBufferSizeFor(this.valueCount); } + /** + * Get the potential buffer size for a particular number of records. + * @param valueCount desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ @Override public int getBufferSizeFor(final int valueCount) { if (valueCount == 0) { return 0; } - final int validityBufferSize = getSizeFromCount(valueCount); + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; /* get the end offset for this valueCount */ final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH); return validityBufferSize + offsetBufferSize + dataBufferSize; } + /** + * Get information about how this field is materialized. + * @return the field corresponding to this vector + */ @Override public Field getField() { return field; } + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ @Override public ArrowBuf[] getBuffers(boolean clear) { final ArrowBuf[] buffers = new ArrowBuf[3]; @@ -468,18 +573,42 @@ public ArrowBuf[] getBuffers(boolean clear) { return buffers; } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @param callBack + * @return TransferPair + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return getTransferPair(ref, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param allocator allocator for the target vector + * @return TransferPair + */ @Override public TransferPair getTransferPair(BufferAllocator allocator){ return getTransferPair(name, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return TransferPair + */ public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + /** + * Transfer this vector'data to another vector. The memory associated + * with this vector is transferred to the allocator of target vector + * for accounting and management purposes. + * @param target destination vector for transfer + */ public void transferTo(BaseNullableVariableWidthVector target){ compareTypes(target, "transferTo"); target.clear(); @@ -491,6 +620,13 @@ public void transferTo(BaseNullableVariableWidthVector target){ clear(); } + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ public void splitAndTransferTo(int startIndex, int length, BaseNullableVariableWidthVector target) { compareTypes(target, "splitAndTransferTo"); @@ -502,7 +638,12 @@ public void splitAndTransferTo(int startIndex, int length, } /* - * transfer the offsets along with data + * Transfer the offsets along with data. Unlike the data buffer, we cannot simply + * slice the offset buffer for split and transfer. The reason is that offsets + * in the target vector have to be adjusted and made relative to the staring + * offset in source vector from the start index of split. This is why, we + * need to explicitly allocate the offset buffer and set the adjusted offsets + * in the target vector. */ private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); @@ -517,14 +658,14 @@ private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullab } /* - * transfer the validity. + * Transfer the validity. */ private void splitAndTransferValidityBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { assert startIndex + length <= valueCount; int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getSizeFromCount(length); + int byteSizeTarget = getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -546,8 +687,8 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, target.allocateValidityBuffer(byteSizeTarget); for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); - byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); target.validityBuffer.setByte(i, (b1 + b2)); } @@ -562,15 +703,15 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, * by shifting data from the current byte. */ if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = getBitsFromNextByte(this.validityBuffer, + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + byteSizeTarget, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); } else { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1); } @@ -578,14 +719,6 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, } } - private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) & 0xFF) >>> offset); - } - - private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) << (8 - offset))); - } - /****************************************************************** * * @@ -600,24 +733,7 @@ private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { * @return the number of null elements. */ public int getNullCount() { - int count = 0; - final int sizeInBytes = getSizeFromCount(valueCount); - - for (int i = 0; i < sizeInBytes; ++i) { - final byte byteValue = validityBuffer.getByte(i); - /* Java uses two's complement binary representation, hence 11111111_b which is -1 - * when converted to Int will have 32bits set to 1. Masking the MSB and then - * adding it back solves the issue. - */ - count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); - } - int nullCount = (sizeInBytes * 8) - count; - /* if the valueCount is not a multiple of 8, - * the bits on the right were counted as null bits. - */ - int remainder = valueCount % 8; - nullCount -= remainder == 0 ? 0 : 8 - remainder; - return nullCount; + return BitVectorHelper.getNullCount(validityBuffer, valueCount); } /** @@ -679,24 +795,55 @@ public void setValueCount(int valueCount) { lastSet = valueCount - 1; } + /** + * Create holes in the vector upto the given index (exclusive). + * Holes will be created from the current last set position in + * the vector. + * + * @param index target index + */ public void fillEmpties(int index) { handleSafe(index, emptyByteArray.length); fillHoles(index); lastSet = index - 1; } + /** + * Set the index of last non-null element in the vector. + * It is important to call this method with appropriate value + * before calling {@link #setValueCount(int)}. + * + * @param value desired index of last non-null element. + */ public void setLastSet(int value) { lastSet = value; } + /** + * Get the index of last non-null element in the vector. + * + * @return index of the last non-null element + */ public int getLastSet() { return lastSet; } + /** + * Get the starting position (offset) in the data stream for a given + * element in the vector. + * + * @param index position of the element in the vector + * @return starting offset for the element + */ public long getStartEnd(int index) { return (long)offsetBuffer.getInt(index * OFFSET_WIDTH); } + /** + * Mark the particular position in the vector as non-null. + * + * @param index position of the element. + */ public void setIndexDefined(int index) { handleSafe(index, 0); BitVectorHelper.setValidityBitToOne(validityBuffer, index); @@ -761,4 +908,97 @@ protected final void handleSafe(int index, int dataLength) { reallocValueBuffer(); } } + + + /****************************************************************** + * * + * helper methods currently * + * used by JsonFileReader and * + * JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Method used by Json Reader to explicitly set the data of the variable + * width vector elements. The method takes care of allocating the memory + * for the vector if caller hasn't done so. + * + * This method should not be used externally. + * + * @param data ArrowBuf for storing variable width elements in the vector + * @param offset offset of the element + * @param allocator memory allocator + * @param index position of the element in the vector + * @param value array of bytes for the element + * @param valueCount number of elements in the vector + * @return buffer holding the variable width data. + */ + public static ArrowBuf set(ArrowBuf data, ArrowBuf offset, + BufferAllocator allocator, int index, byte[] value, + int valueCount) { + if (data == null) { + data = allocator.buffer(INITIAL_BYTE_COUNT); + } + final int currentBufferCapacity = data.capacity(); + final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH); + while (currentBufferCapacity < currentStartOffset + value.length) { + final ArrowBuf newBuf = allocator.buffer(currentBufferCapacity * 2); + newBuf.setBytes(0, data, 0, currentBufferCapacity); + data.release(); + data = newBuf; + } + data.setBytes(currentStartOffset, value, 0, value.length); + if (index == (valueCount - 1)) { + data.writerIndex(offset.getInt(valueCount * OFFSET_WIDTH)); + } + return data; + } + + /** + * Method used by Json Writer to read a variable width element from + * the variable width vector and write to Json. + * + * This method should not be used externally. + * + * @param data buffer storing the variable width vector elements + * @param offset buffer storing the offsets of variable width vector elements + * @param index position of the element in the vector + * @return array of bytes + */ + public static byte[] get(final ArrowBuf data, final ArrowBuf offset, int index) { + final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH); + final int dataLength = + offset.getInt((index + 1) * OFFSET_WIDTH) - currentStartOffset; + final byte[] result = new byte[dataLength]; + data.getBytes(currentStartOffset, result, 0, dataLength); + return result; + } + + /** + * Method used by Json Reader to explicitly set the offsets of the variable + * width vector data. The method takes care of allocating the memory for + * offsets if the caller hasn't done so. + * + * This method should not be used externally. + * + * @param buffer ArrowBuf to store offsets for variable width elements + * @param allocator memory allocator + * @param valueCount number of elements + * @param index position of the element + * @param value offset of the element + * @return buffer holding the offsets + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * OFFSET_WIDTH); + } + buffer.setInt(index * OFFSET_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * OFFSET_WIDTH); + } + + return buffer; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 1db96246485..1b96782681f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -56,7 +56,6 @@ public String toString() { @Override public void clear() { - getMutator().reset(); } @Override @@ -137,10 +136,36 @@ protected ArrowBuf releaseBuffer(ArrowBuf buffer) { return buffer; } - public int getValueCount() { return 0; } + @Override + public int getValueCount() { + return getAccessor().getValueCount(); + } + + @Override + public void setValueCount(int valueCount) { + getMutator().setValueCount(valueCount); + } + + @Override + + public Object getObject(int index) { + return getAccessor().getObject(index); + } + + @Override + + public int getNullCount() { + return getAccessor().getNullCount(); + } - public void setValueCount(int valueCount) { } + @Override + public boolean isNull(int index) { + return getAccessor().isNull(index); + } - public Object getObject(int index) { return null; } + /* number of bytes for the validity buffer for the given valueCount */ + protected static int getValidityBufferSizeFromCount(final int valueCount) { + return (int) Math.ceil(valueCount / 8.0); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index 2439bd26eab..69325b5ea06 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -19,24 +19,35 @@ package org.apache.arrow.vector; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; -class BitVectorHelper { +/** + * Helper class for performing generic operations on a bit vector buffer. + * External use of this class is not recommended. + */ +public class BitVectorHelper { /** * Get the index of byte corresponding to bit index in validity buffer */ - protected static int byteIndex(int absoluteBitIndex) { + public static int byteIndex(int absoluteBitIndex) { return absoluteBitIndex >> 3; } /** * Get the relative index of bit within the byte in validity buffer */ - private static int bitIndex(int absoluteBitIndex) { + public static int bitIndex(int absoluteBitIndex) { return absoluteBitIndex & 7; } - protected static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { + /** + * Set the bit at provided index to 1. + * + * @param validityBuffer + * @param index + */ + public static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { final int byteIndex = byteIndex(index); final int bitIndex = bitIndex(index); byte currentByte = validityBuffer.getByte(byteIndex); @@ -45,7 +56,14 @@ protected static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { validityBuffer.setByte(byteIndex, currentByte); } - protected static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { + /** + * Set the bit at a given index to provided value (1 or 0) + * + * @param validityBuffer + * @param index + * @param value + */ + public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { final int byteIndex = byteIndex(index); final int bitIndex = bitIndex(index); byte currentByte = validityBuffer.getByte(byteIndex); @@ -57,4 +75,91 @@ protected static void setValidityBit(ArrowBuf validityBuffer, int index, int val } validityBuffer.setByte(byteIndex, currentByte); } + + /** + * Set the bit at a given index to provided value (1 or 0). Internally + * takes care of allocating the buffer if the caller didn't do so. + * + * @param validityBuffer + * @param allocator + * @param valueCount + * @param index + * @param value + * @return ArrowBuf + */ + public static ArrowBuf setValidityBit(ArrowBuf validityBuffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (validityBuffer == null) { + validityBuffer = allocator.buffer(getValidityBufferSize(valueCount)); + } + setValidityBit(validityBuffer, index, value); + if (index == (valueCount - 1)) { + validityBuffer.writerIndex(getValidityBufferSize(valueCount)); + } + + return validityBuffer; + } + + /** + * Check if a bit at a given index is set or not. + * + * @param buffer + * @param index + * @return 1 if bit is set, 0 otherwise. + */ + public static int get(final ArrowBuf buffer, int index) { + final int byteIndex = index >> 3; + final byte b = buffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Compute the size of validity buffer required to manage a given number + * of elements in a vector. + * + * @param valueCount + * @return buffer size + */ + public static int getValidityBufferSize(int valueCount) { + return ((int) Math.ceil(valueCount / 8.0)); + } + + /** + * Given a validity buffer, find the number of bits that are not set. + * This is used to compute the number of null elements in a nullable vector. + * + * @param validityBuffer + * @param valueCount + * @return number of bits not set. + */ + public static int getNullCount(final ArrowBuf validityBuffer, final int valueCount) { + if (valueCount == 0) { return 0; } + int count = 0; + final int sizeInBytes = getValidityBufferSize(valueCount); + + for (int i = 0; i < sizeInBytes; ++i) { + final byte byteValue = validityBuffer.getByte(i); + /* Java uses two's complement binary representation, hence 11111111_b which is -1 + * when converted to Int will have 32bits set to 1. Masking the MSB and then + * adding it back solves the issue. + */ + count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); + } + int nullCount = (sizeInBytes * 8) - count; + /* if the valueCount is not a multiple of 8, + * the bits on the right were counted as null bits. + */ + int remainder = valueCount % 8; + nullCount -= remainder == 0 ? 0 : 8 - remainder; + return nullCount; + } + + public static byte getBitsFromCurrentByte(final ArrowBuf data, final int index, final int offset) { + return (byte)((data.getByte(index) & 0xFF) >>> offset); + } + + public static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { + return (byte)((data.getByte(index) << (8 - offset))); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java new file mode 100644 index 00000000000..ee40d708cc1 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java @@ -0,0 +1,403 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.BigIntReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.BigIntHolder; +import org.apache.arrow.vector.holders.NullableBigIntHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableBigIntVector implements a fixed width vector (8 bytes) of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableBigIntVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableBigIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableBigIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.BIGINT.getType()), + allocator); + } + + /** + * Instantiate a NullableBigIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableBigIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new BigIntReaderImpl(NullableBigIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.BIGINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableBigIntHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableBigIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableBigIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableBigIntHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, BigIntHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableBigIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableBigIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, BigIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, BigIntHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableBigIntVector)to); + } + + private class TransferImpl implements TransferPair { + NullableBigIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableBigIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableBigIntVector to){ + this.to = to; + } + + @Override + public NullableBigIntVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableBigIntVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java new file mode 100644 index 00000000000..eceaa61b57f --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java @@ -0,0 +1,502 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.BitReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.BitHolder; +import org.apache.arrow.vector.holders.NullableBitHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.OversizedAllocationException; +import org.apache.arrow.vector.util.TransferPair; + +import java.util.ArrayList; +import java.util.List; + +/** + * NullableBitVector implements a fixed width (1 bit) vector of + * boolean values which could be null. Each value in the vector corresponds + * to a single bit in the underlying data stream backing the vector. + */ +public class NullableBitVector extends BaseNullableFixedWidthVector { + private final FieldReader reader; + + /** + * Instantiate a NullableBitVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableBitVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.BIT.getType()), + allocator); + } + + /** + * Instantiate a NullableBitVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableBitVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, (byte)0); + reader = new BitReaderImpl(NullableBitVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.BIT; + } + + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ + @Override + public void setInitialCapacity(int valueCount) { + final int size = getValidityBufferSizeFromCount(valueCount); + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = size; + validityAllocationSizeInBytes = size; + } + + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ + @Override + public int getValueCapacity(){ + return (int)(validityBuffer.capacity() * 8L); + } + + /** + * Get the potential buffer size for a particular number of records. + * @param count desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ + @Override + public int getBufferSizeFor(final int count) { + if (count == 0) { return 0; } + return 2 * getValidityBufferSizeFromCount(count); + } + + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ + @Override + public int getBufferSize() { + return getBufferSizeFor(valueCount); + } + + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ + public void splitAndTransferTo(int startIndex, int length, + BaseNullableFixedWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + target.validityBuffer = splitAndTransferBuffer(startIndex, length, target, + validityBuffer, target.validityBuffer); + target.valueBuffer = splitAndTransferBuffer(startIndex, length, target, + valueBuffer, target.valueBuffer); + + target.setValueCount(length); + } + + private ArrowBuf splitAndTransferBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target, + ArrowBuf sourceBuffer, ArrowBuf destBuffer) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + /* slice */ + if (destBuffer != null) { + destBuffer.release(); + } + destBuffer = destBuffer.slice(firstByteSource, byteSizeTarget); + destBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + destBuffer = allocator.buffer(byteSizeTarget); + destBuffer.readerIndex(0); + destBuffer.setZero(0, destBuffer.capacity()); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset); + + destBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, + firstByteSource + byteSizeTarget, offset); + + destBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, + firstByteSource + byteSizeTarget - 1, offset); + destBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + + return destBuffer; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + private int getBit(int index) { + final int byteIndex = index >> 3; + final byte b = valueBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return getBit(index); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableBitHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = getBit(index); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Boolean getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return new Boolean (getBit(index) != 0); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableBitVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableBitVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableBitHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (holder.value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, BitHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (holder.value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableBitHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, BitHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, BitHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableBitVector)to); + } + + private class TransferImpl implements TransferPair { + NullableBitVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableBitVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableBitVector to){ + this.to = to; + } + + @Override + public NullableBitVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableBitVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java new file mode 100644 index 00000000000..949287ecafe --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java @@ -0,0 +1,404 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.DateDayReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.DateDayHolder; +import org.apache.arrow.vector.holders.NullableDateDayHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableDateDayVector implements a fixed width (4 bytes) vector of + * date values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableDateDayVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableDateDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDateDayVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.DATEDAY.getType()), + allocator); + } + + /** + * Instantiate a NullableDateDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDateDayVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new DateDayReaderImpl(NullableDateDayVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DATEDAY; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDateDayHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDateDayVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDateDayVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDateDayHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DateDayHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDateDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDateDayHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DateDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DateDayHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setInt(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDateDayVector)to); + } + + private class TransferImpl implements TransferPair { + NullableDateDayVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableDateDayVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableDateDayVector to){ + this.to = to; + } + + @Override + public NullableDateDayVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDateDayVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java new file mode 100644 index 00000000000..a0bdccedea4 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java @@ -0,0 +1,408 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.DateMilliReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.DateMilliHolder; +import org.apache.arrow.vector.holders.NullableDateMilliHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; +import org.joda.time.LocalDateTimes; +import org.slf4j.Logger; + +/** + * NullableDateMilliVector implements a fixed width vector (8 bytes) of + * date values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableDateMilliVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDateMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.DATEMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new DateMilliReaderImpl(NullableDateMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DATEMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDateMilliHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long millis = get(index); + final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDateMilliVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDateMilliVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DateMilliHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDateMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DateMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DateMilliHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDateMilliVector)to); + } + + private class TransferImpl implements TransferPair { + NullableDateMilliVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableDateMilliVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableDateMilliVector to){ + this.to = to; + } + + @Override + public NullableDateMilliVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDateMilliVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java new file mode 100644 index 00000000000..8320f90830a --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java @@ -0,0 +1,454 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.DecimalReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.DecimalHolder; +import org.apache.arrow.vector.holders.NullableDecimalHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.DecimalUtility; +import org.apache.arrow.vector.util.TransferPair; + +import java.math.BigDecimal; + +/** + * NullableDecimalVector implements a fixed width vector (16 bytes) of + * decimal values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableDecimalVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 16; + private final FieldReader reader; + + private final int precision; + private final int scale; + + /** + * Instantiate a NullableDecimalVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDecimalVector(String name, BufferAllocator allocator, + int precision, int scale) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)), + allocator); + } + + /** + * Instantiate a NullableDecimalVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + org.apache.arrow.vector.types.pojo.ArrowType.Decimal arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Decimal)fieldType.getType(); + reader = new DecimalReaderImpl(NullableDecimalVector.this); + this.precision = arrowType.getPrecision(); + this.scale = arrowType.getScale(); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DECIMAL; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public ArrowBuf get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDecimalHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.buffer = valueBuffer; + holder.precision = precision; + holder.scale = scale; + holder.start = index * TYPE_WIDTH; + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public BigDecimal getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) { + if (from.isSet(fromIndex) != 0) { + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer, + thisIndex * TYPE_WIDTH, TYPE_WIDTH); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDecimalVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param buffer ArrowBuf containing decimal value. + */ + public void set(int index, ArrowBuf buffer) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param start start index of data in the buffer + * @param buffer ArrowBuf containing decimal value. + */ + public void set(int index, int start, ArrowBuf buffer) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, buffer, start, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value BigDecimal containing decimal value. + */ + public void set(int index, BigDecimal value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + DecimalUtility.checkPrecisionAndScale(value, precision, scale); + DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DecimalHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); + } + + /** + * Same as {@link #set(int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param buffer ArrowBuf containing decimal value. + */ + public void setSafe(int index, ArrowBuf buffer) { + handleSafe(index); + set(index, buffer); + } + + /** + * Same as {@link #set(int, int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param start start index of data in the buffer + * @param buffer ArrowBuf containing decimal value. + */ + public void setSafe(int index, int start, ArrowBuf buffer) { + handleSafe(index); + set(index, start, buffer); + } + + /** + * Same as {@link #set(int, BigDecimal)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value BigDecimal containing decimal value. + */ + public void setSafe(int index, BigDecimal value){ + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDecimalHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDecimalHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DecimalHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DecimalHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of the value in the buffer + * @param buffer buffer containing the value to be stored in the vector + */ + public void set(int index, int isSet, int start, ArrowBuf buffer) { + if (isSet > 0) { + set(index, start, buffer); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #setSafe(int, int, int, ArrowBuf)} except that it handles + * the case when the position of new value is beyond the current value + * capacity of the vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of the value in the buffer + * @param buffer buffer containing the value to be stored in the vector + */ + public void setSafe(int index, int isSet, int start, ArrowBuf buffer) { + handleSafe(index); + set(index, isSet, start, buffer); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value as array of bytes + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, byte[] value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + DecimalUtility.writeByteArrayToArrowBuf(value, buffer, index); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDecimalVector)to); + } + + private class TransferImpl implements TransferPair { + NullableDecimalVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableDecimalVector(ref, allocator, NullableDecimalVector.this.precision, + NullableDecimalVector.this.scale); + } + + public TransferImpl(NullableDecimalVector to){ + this.to = to; + } + + @Override + public NullableDecimalVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDecimalVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java new file mode 100644 index 00000000000..5b28065c402 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java @@ -0,0 +1,404 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.Float4ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.Float4Holder; +import org.apache.arrow.vector.holders.NullableFloat4Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableFloat4Vector implements a fixed width vector (4 bytes) of + * float values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableFloat4Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableFloat4Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.FLOAT4.getType()), + allocator); + } + + /** + * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableFloat4Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new Float4ReaderImpl(NullableFloat4Vector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.FLOAT4; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public float get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getFloat(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableFloat4Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getFloat(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Float getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableFloat4Vector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat4Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, float value) { + valueBuffer.setFloat(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, float value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableFloat4Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, Float4Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, float)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, float value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableFloat4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableFloat4Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, Float4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, Float4Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, float value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, float)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, float value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, float value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setFloat(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static float get(final ArrowBuf buffer, final int index) { + return buffer.getFloat(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableFloat4Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableFloat4Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableFloat4Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableFloat4Vector to){ + this.to = to; + } + + @Override + public NullableFloat4Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableFloat4Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java new file mode 100644 index 00000000000..624abf2f272 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java @@ -0,0 +1,404 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.Float8ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.Float8Holder; +import org.apache.arrow.vector.holders.NullableFloat8Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableFloat8Vector implements a fixed width vector (8 bytes) of + * double values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableFloat8Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableFloat8Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.FLOAT8.getType()), + allocator); + } + + /** + * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableFloat8Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new Float8ReaderImpl(NullableFloat8Vector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.FLOAT8; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public double get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getDouble(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableFloat8Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getDouble(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Double getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableFloat8Vector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat8Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, double value) { + valueBuffer.setDouble(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, double value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableFloat8Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, Float8Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, double)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, double value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableFloat8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableFloat8Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, Float8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, Float8Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, double value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, double)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, double value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, double value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setDouble(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static double get(final ArrowBuf buffer, final int index) { + return buffer.getDouble(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableFloat8Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableFloat8Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableFloat8Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableFloat8Vector to){ + this.to = to; + } + + @Override + public NullableFloat8Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableFloat8Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java index 26b19fa7bd5..6311daf4f5c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,10 +14,11 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.impl.IntReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -27,39 +27,53 @@ import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; -import org.slf4j.Logger; /** - * NullableIntVector implements a fixed width vector of values which could - * be null. A validity buffer (bit vector) is maintained to track which - * elements in the vector are null. + * NullableIntVector implements a fixed width (4 bytes) vector of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. */ public class NullableIntVector extends BaseNullableFixedWidthVector { - private static final org.slf4j.Logger logger = - org.slf4j.LoggerFactory.getLogger(NullableIntVector.class); private static final byte TYPE_WIDTH = 4; private final FieldReader reader; + /** + * Instantiate a NullableIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ public NullableIntVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()), allocator); } + /** + * Instantiate a NullableIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); reader = new IntReaderImpl(NullableIntVector.this); } - @Override - protected org.slf4j.Logger getLogger() { - return logger; - } - + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ @Override public FieldReader getReader(){ return reader; } + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ @Override public Types.MinorType getMinorType() { return Types.MinorType.INT; @@ -116,12 +130,27 @@ public Integer getObject(int index) { } } + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); } } + /** + * Same as {@link #copyFrom(int, int, NullableIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); @@ -234,17 +263,80 @@ public void setNull(int index){ BitVectorHelper.setValidityBit(validityBuffer, index, 0); } - public void set(int index, int isSet, int valueField ) { + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { if (isSet > 0) { - set(index, valueField); + set(index, value); } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); } } - public void setSafe(int index, int isSet, int valueField ) { + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { handleSafe(index); - set(index, isSet, valueField); + set(index, isSet, value); + } + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setInt(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); } @@ -255,11 +347,23 @@ public void setSafe(int index, int isSet, int valueField ) { ******************************************************************/ + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator){ return new TransferImpl(ref, allocator); } + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ @Override public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((NullableIntVector)to); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java new file mode 100644 index 00000000000..174b722b043 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java @@ -0,0 +1,429 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.IntervalDayReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.IntervalDayHolder; +import org.apache.arrow.vector.holders.NullableIntervalDayHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.Period; + +/** + * NullableIntervalDayVector implements a fixed width vector (8 bytes) of + * interval (days and milliseconds) values which could be null. + * A validity buffer (bit vector) is maintained to track which elements in the + * vector are null. + */ +public class NullableIntervalDayVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private static final byte MILLISECOND_OFFSET = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableIntervalDayVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.INTERVALDAY.getType()), + allocator); + } + + /** + * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableIntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntervalDayReaderImpl(NullableIntervalDayVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INTERVALDAY; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public ArrowBuf get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + return null; + } + return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntervalDayHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + final int startIndex = index * TYPE_WIDTH; + holder.isSet = 1; + holder.days = valueBuffer.getInt(startIndex); + holder.milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Period getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final int startIndex = index * TYPE_WIDTH; + final int days = valueBuffer.getInt(startIndex); + final int milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + final Period p = new Period(); + return p.plusDays(days).plusMillis(milliseconds); + } + } + + /** + * Get the Interval value at a given index as a {@link StringBuilder} object + * @param index position of the element + * @return String Builder object with Interval value as + * [days, hours, minutes, seconds, millis] + */ + public StringBuilder getAsStringBuilder(int index) { + if (isSet(index) == 0) { + return null; + }else{ + return getAsStringBuilderHelper(index); + } + } + + private StringBuilder getAsStringBuilderHelper(int index) { + final int startIndex = index * TYPE_WIDTH; + + final int days = valueBuffer.getInt(startIndex); + int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + + final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis); + + final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis); + + final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis); + + final String dayString = (Math.abs(days) == 1) ? " day " : " days "; + + return(new StringBuilder(). + append(days).append(dayString). + append(hours).append(":"). + append(minutes).append(":"). + append(seconds).append("."). + append(millis)); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) { + if (from.isSet(fromIndex) != 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer, + thisIndex * TYPE_WIDTH, TYPE_WIDTH); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableIntervalDayVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalDayVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, ArrowBuf value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param days days for the interval + * @param milliseconds milliseconds for the interval + */ + public void set(int index, int days, int milliseconds){ + final int offsetIndex = index * TYPE_WIDTH; + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setInt(offsetIndex, days); + valueBuffer.setInt((offsetIndex + MILLISECOND_OFFSET), milliseconds); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + set(index, holder.days, holder.milliseconds); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntervalDayHolder holder){ + set(index, holder.days, holder.milliseconds); + } + + /** + * Same as {@link #set(int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, ArrowBuf value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param days days for the interval + * @param milliseconds milliseconds for the interval + */ + public void setSafe(int index, int days, int milliseconds) { + handleSafe(index); + set(index, days, milliseconds); + } + + /** + * Same as {@link #set(int, NullableIntervalDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntervalDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntervalDayHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param days days component of interval + * @param milliseconds millisecond component of interval + */ + public void set(int index, int isSet, int days, int milliseconds) { + if (isSet > 0) { + set(index, days, milliseconds); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param days days component of interval + * @param milliseconds millisecond component of interval + */ + public void setSafe(int index, int isSet, int days, int milliseconds) { + handleSafe(index); + set(index, isSet, days, milliseconds); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntervalDayVector)to); + } + + private class TransferImpl implements TransferPair { + NullableIntervalDayVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableIntervalDayVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntervalDayVector to){ + this.to = to; + } + + @Override + public NullableIntervalDayVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntervalDayVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java new file mode 100644 index 00000000000..303286d0ed3 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java @@ -0,0 +1,387 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.IntervalYearReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.IntervalYearHolder; +import org.apache.arrow.vector.holders.NullableIntervalYearHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.Period; + +/** + * NullableIntervalYearVector implements a fixed width (4 bytes) vector of + * interval (years and months) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableIntervalYearVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableIntervalYearVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.INTERVALYEAR.getType()), + allocator); + } + + /** + * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableIntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntervalYearReaderImpl(NullableIntervalYearVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INTERVALYEAR; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntervalYearHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Period getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final int interval = get(index); + final int years = (interval / org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final int months = (interval % org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final Period p = new Period(); + return p.plusYears(years).plusMonths(months); + } + } + + /** + * Get the Interval value at a given index as a {@link StringBuilder} object + * @param index position of the element + * @return String Builder object with Interval value as + * [years, months] + */ + public StringBuilder getAsStringBuilder(int index) { + if (isSet(index) == 0) { + return null; + }else{ + return getAsStringBuilderHelper(index); + } + } + + private StringBuilder getAsStringBuilderHelper(int index) { + int value = valueBuffer.getInt(index * TYPE_WIDTH); + + final int years = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths); + + final String yearString = (Math.abs(years) == 1) ? " year " : " years "; + final String monthString = (Math.abs(months) == 1) ? " month " : " months "; + + return(new StringBuilder(). + append(years).append(yearString). + append(months).append(monthString)); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableIntervalYearVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalYearVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntervalYearHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableIntervalYearHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntervalYearHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntervalYearHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntervalYearVector)to); + } + + private class TransferImpl implements TransferPair { + NullableIntervalYearVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableIntervalYearVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntervalYearVector to){ + this.to = to; + } + + @Override + public NullableIntervalYearVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntervalYearVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java new file mode 100644 index 00000000000..c45a8d5f5bc --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java @@ -0,0 +1,431 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.SmallIntReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.SmallIntHolder; +import org.apache.arrow.vector.holders.NullableSmallIntHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableSmallIntVector implements a fixed width (2 bytes) vector of + * short values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableSmallIntVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 2; + private final FieldReader reader; + + /** + * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableSmallIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.SMALLINT.getType()), + allocator); + } + + /** + * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableSmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new SmallIntReaderImpl(NullableSmallIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.SMALLINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public short get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getShort(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableSmallIntHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getShort(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Short getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableSmallIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableSmallIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setShort(index * TYPE_WIDTH, value); + } + + private void setValue(int index, short value) { + valueBuffer.setShort(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, short value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, SmallIntHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, short)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, short value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableSmallIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, SmallIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, SmallIntHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, short value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, short)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, short value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, short value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setShort(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static short get(final ArrowBuf buffer, final int index) { + return buffer.getShort(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableSmallIntVector)to); + } + + private class TransferImpl implements TransferPair { + NullableSmallIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableSmallIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableSmallIntVector to){ + this.to = to; + } + + @Override + public NullableSmallIntVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableSmallIntVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java new file mode 100644 index 00000000000..454a4ac41a6 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java @@ -0,0 +1,404 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeMicroReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeMicroHolder; +import org.apache.arrow.vector.holders.NullableTimeMicroHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableTimeMicroVector implements a fixed width vector (8 bytes) of + * time (microsecond resolution) values which could be null. + * A validity buffer (bit vector) is maintained to track which elements in the + * vector are null. + */ +public class NullableTimeMicroVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeMicroVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMEMICRO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeMicroReaderImpl(NullableTimeMicroVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMEMICRO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeMicroHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeMicroVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMicroVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeMicroHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeMicroHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeMicroVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTimeMicroVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTimeMicroVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeMicroVector to){ + this.to = to; + } + + @Override + public NullableTimeMicroVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeMicroVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java new file mode 100644 index 00000000000..8540d169242 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java @@ -0,0 +1,405 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeMilliReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeMilliHolder; +import org.apache.arrow.vector.holders.NullableTimeMilliHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; +import org.slf4j.Logger; + +/** + * NullableTimeMilliVector implements a fixed width (4 bytes) vector of + * time (millisecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeMilliVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMEMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeMilliReaderImpl(NullableTimeMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMEMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeMilliHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } + org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index), + org.joda.time.DateTimeZone.UTC); + return ldt; + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeMilliVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMilliVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeMilliHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeMilliHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setInt(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeMilliVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTimeMilliVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTimeMilliVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeMilliVector to){ + this.to = to; + } + + @Override + public NullableTimeMilliVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeMilliVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java new file mode 100644 index 00000000000..015226da223 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java @@ -0,0 +1,402 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeNanoReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeNanoHolder; +import org.apache.arrow.vector.holders.NullableTimeNanoHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeNanoVector implements a fixed width vector (8 bytes) of + * time (nanosecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeNanoVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeNanoVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMENANO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeNanoReaderImpl(NullableTimeNanoVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMENANO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeNanoHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeNanoVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeNanoVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeNanoHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeNanoHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer,final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeNanoVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTimeNanoVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTimeNanoVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeNanoVector to){ + this.to = to; + } + + @Override + public NullableTimeNanoVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeNanoVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java new file mode 100644 index 00000000000..2b2375e92f1 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java @@ -0,0 +1,403 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeSecReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeSecHolder; +import org.apache.arrow.vector.holders.NullableTimeSecHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeSecVector implements a fixed width (4 bytes) vector of + * time (seconds resolution) values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableTimeSecVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeSecVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESEC.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeSecReaderImpl(NullableTimeSecVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESEC; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeSecHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeSecVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeSecVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeSecHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeSecHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setInt(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeSecVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTimeSecVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTimeSecVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeSecVector to){ + this.to = to; + } + + @Override + public NullableTimeSecVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeSecVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java new file mode 100644 index 00000000000..0d25eff4e02 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampMicroTZReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampMicroTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMicroTZHolder; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampMicroTZVector implements a fixed width vector (8 bytes) of + * timestamp (microsecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampMicroTZVector extends NullableTimeStampVector { + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampMicroTZReaderImpl(NullableTimeStampMicroTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMICROTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMicroTZHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMicroTZHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMicroTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMicroTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMicroTZHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMicroTZVector to = new NullableTimeStampMicroTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMicroTZVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java new file mode 100644 index 00000000000..41f7a1112f2 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampMicroReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampMicroHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMicroHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; + +/** + * NullableTimeStampMicroVector implements a fixed width vector (8 bytes) of + * timestamp (microsecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampMicroVector extends NullableTimeStampVector { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMICRO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampMicroReaderImpl(NullableTimeStampMicroVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMICRO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMicroHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + /* value is truncated when converting microseconds to milliseconds in order to use DateTime type */ + final long micros = get(index); + final long millis = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(micros); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMicroHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMicroHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMicroVector to = new NullableTimeStampMicroVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMicroVector )to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java new file mode 100644 index 00000000000..6376b085e64 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampMilliTZReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampMilliTZVector implements a fixed width vector (8 bytes) of + * timestamp (millisecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampMilliTZVector extends NullableTimeStampVector { + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MILLISECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampMilliTZReaderImpl(NullableTimeStampMilliTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMILLITZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMilliTZHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMilliTZHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMilliTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMilliTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMilliTZHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMilliTZVector to = new NullableTimeStampMilliTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMilliTZVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java new file mode 100644 index 00000000000..ff581d2ea2f --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampMilliReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampMilliHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; + +/** + * NullableTimeStampMilliVector implements a fixed width vector (8 bytes) of + * timestamp (millisecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampMilliVector extends NullableTimeStampVector { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampMilliReaderImpl(NullableTimeStampMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMilliHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long millis = get(index); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMilliHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMilliHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMilliVector to = new NullableTimeStampMilliVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMilliVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java new file mode 100644 index 00000000000..6c4445d500b --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampNanoTZReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampNanoTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampNanoTZVector implements a fixed width vector (8 bytes) of + * timestamp (nanosecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampNanoTZVector extends NullableTimeStampVector { + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampNanoTZReaderImpl(NullableTimeStampNanoTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPNANOTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampNanoTZHolder holder) { + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampNanoTZHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampNanoTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampNanoTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampNanoTZHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampNanoTZVector to = new NullableTimeStampNanoTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampNanoTZVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java new file mode 100644 index 00000000000..096913ebf1c --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampNanoReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampNanoHolder; +import org.apache.arrow.vector.holders.NullableTimeStampNanoHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; + +/** + * NullableTimeStampNanoVector implements a fixed width vector (8 bytes) of + * timestamp (nanosecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampNanoVector extends NullableTimeStampVector { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPNANO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampNanoReaderImpl(NullableTimeStampNanoVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPNANO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampNanoHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long nanos = get(index); + final long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(nanos); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampNanoHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampNanoHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampNanoVector to = new NullableTimeStampNanoVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampNanoVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java new file mode 100644 index 00000000000..8e5ba72ae1f --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java @@ -0,0 +1,217 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampSecTZReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampSecTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampSecTZHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampSecTZVector implements a fixed width vector (8 bytes) of + * timestamp (seconds resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampSecTZVector extends NullableTimeStampVector { + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampSecTZReaderImpl(NullableTimeStampSecTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPSECTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampSecTZHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampSecTZHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampSecTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampSecTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampSecTZHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampSecTZVector to = new NullableTimeStampSecTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampSecTZVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java new file mode 100644 index 00000000000..bc800e0e4f0 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampSecReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampSecHolder; +import org.apache.arrow.vector.holders.NullableTimeStampSecHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; + +/** + * NullableTimeStampSecVector implements a fixed width vector (8 bytes) of + * timestamp (seconds resolution) values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableTimeStampSecVector extends NullableTimeStampVector { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPSEC.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampSecReaderImpl(NullableTimeStampSecVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPSEC; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampSecHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long secs = get(index); + final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampSecHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampSecHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampSecVector to = new NullableTimeStampSecVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampSecVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java new file mode 100644 index 00000000000..b2a58bd4568 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampVector is an abstract interface for fixed width vector (8 bytes) + * of timestamp values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public abstract class NullableTimeStampVector extends BaseNullableFixedWidthVector { + protected static final byte TYPE_WIDTH = 8; + + /** + * Instantiate a NullableTimeStampVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFromSafe(int, int, NullableTimeStampVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeStampVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + protected void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and * + * JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + public class TransferImpl implements TransferPair { + NullableTimeStampVector to; + + public TransferImpl(NullableTimeStampVector to) { + this.to = to; + } + + @Override + public NullableTimeStampVector getTo(){ + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeStampVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java new file mode 100644 index 00000000000..15100530d0e --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java @@ -0,0 +1,432 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TinyIntReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TinyIntHolder; +import org.apache.arrow.vector.holders.NullableTinyIntHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTinyIntVector implements a fixed width (1 bytes) vector of + * byte values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableTinyIntVector extends BaseNullableFixedWidthVector { + public static final byte TYPE_WIDTH = 1; + private final FieldReader reader; + + /** + * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTinyIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TINYINT.getType()), + allocator); + } + + /** + * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TinyIntReaderImpl(NullableTinyIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TINYINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public byte get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTinyIntHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Byte getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTinyIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTinyIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + private void setValue(int index, int value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + private void setValue(int index, byte value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, byte value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TinyIntHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, byte)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, byte value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTinyIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TinyIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TinyIntHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, byte value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, byte)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, byte value) { + handleSafe(index); + set(index, isSet, value); + } + + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, byte value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setByte(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static byte get(final ArrowBuf buffer, final int index) { + return buffer.getByte(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTinyIntVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTinyIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTinyIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTinyIntVector to){ + this.to = to; + } + + @Override + public NullableTinyIntVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTinyIntVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java new file mode 100644 index 00000000000..4d20f800ba1 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java @@ -0,0 +1,318 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.UInt1ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.UInt1Holder; +import org.apache.arrow.vector.holders.NullableUInt1Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableUInt1Vector implements a fixed width (1 bytes) vector of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableUInt1Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 1; + private final FieldReader reader; + + public NullableUInt1Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT1.getType()), + allocator); + } + + public NullableUInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt1ReaderImpl(NullableUInt1Vector.this); + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT1; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public byte get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt1Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Byte getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + private void setValue(int index, byte value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, byte value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt1Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt1Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, byte)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, byte value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt1Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt1Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt1Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt1Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, byte value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, byte value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt1Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableUInt1Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableUInt1Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt1Vector to){ + this.to = to; + } + + @Override + public NullableUInt1Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt1Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java new file mode 100644 index 00000000000..b140b9f5cbc --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.UInt2ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.UInt2Holder; +import org.apache.arrow.vector.holders.NullableUInt2Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableUInt2Vector implements a fixed width (2 bytes) vector of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableUInt2Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 2; + private final FieldReader reader; + + public NullableUInt2Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT2.getType()), + allocator); + } + + public NullableUInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt2ReaderImpl(NullableUInt2Vector.this); + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT2; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public char get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getChar(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt2Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getChar(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Character getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setChar(index * TYPE_WIDTH, value); + } + + private void setValue(int index, char value) { + valueBuffer.setChar(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, char value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt2Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt2Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, char)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, char value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt2Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt2Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt2Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt2Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, char value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, char value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt2Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableUInt2Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableUInt2Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt2Vector to){ + this.to = to; + } + + @Override + public NullableUInt2Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt2Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java new file mode 100644 index 00000000000..fc8cb639581 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.UInt4ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.UInt4Holder; +import org.apache.arrow.vector.holders.NullableUInt4Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableUInt4Vector implements a fixed width (4 bytes) vector of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableUInt4Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + public NullableUInt4Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT4.getType()), + allocator); + } + + public NullableUInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt4ReaderImpl(NullableUInt4Vector.this); + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT4; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt4Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt4Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt4Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt4Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt4Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt4Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableUInt4Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableUInt4Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt4Vector to){ + this.to = to; + } + + @Override + public NullableUInt4Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt4Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java new file mode 100644 index 00000000000..2110cf03b9f --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.UInt8ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.UInt8Holder; +import org.apache.arrow.vector.holders.NullableUInt8Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableUInt8Vector implements a fixed width vector (8 bytes) of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableUInt8Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + public NullableUInt8Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.UINT8.getType()), + allocator); + } + + public NullableUInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt8ReaderImpl(NullableUInt8Vector.this); + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT8; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt8Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt8Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt8Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt8Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt8Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt8Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableUInt8Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableUInt8Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt8Vector to){ + this.to = to; + } + + @Override + public NullableUInt8Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt8Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java new file mode 100644 index 00000000000..330e06710fc --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java @@ -0,0 +1,514 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.VarBinaryReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.VarBinaryHolder; +import org.apache.arrow.vector.holders.NullableVarBinaryHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +import java.nio.ByteBuffer; + +/** + * NullableVarBinaryVector implements a variable width vector of binary + * values which could be NULL. A validity buffer (bit vector) is maintained + * to track which elements in the vector are null. + */ +public class NullableVarBinaryVector extends BaseNullableVariableWidthVector { + private final FieldReader reader; + + /** + * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableVarBinaryVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.VARBINARY.getType()), allocator); + } + + /** + * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableVarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType); + reader = new VarBinaryReaderImpl(NullableVarBinaryVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.VARBINARY; + } + + + /****************************************************************** + * * + * vector value getter methods * + * * + ******************************************************************/ + + + /** + * Get the variable length element at specified index as byte array. + * + * @param index position of element to get + * @return array of bytes for non-null element, null otherwise + */ + public byte[] get(int index) { + assert index >= 0; + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + final byte[] result = new byte[dataLength]; + valueBuffer.getBytes(startOffset, result, 0, dataLength); + return result; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return byte array for non-null element, null otherwise + */ + public byte[] getObject(int index) { + byte[] b; + try { + b = get(index); + } catch (IllegalStateException e) { + return null; + } + return b; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return greater than 0 length for non-null element, 0 otherwise + */ + public int getValueLength(int index) { + assert index >= 0; + if(isSet(index) == 0) { + return 0; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + return dataLength; + } + + /** + * Get the variable length element at specified index and sets the state + * in provided holder. + * + * @param index position of element to get + * @param holder data holder to be populated by this function + */ + public void get(int index, NullableVarBinaryHolder holder){ + assert index >= 0; + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + holder.isSet = 1; + holder.start = startOffset; + holder.end = dataLength; + holder.buffer = valueBuffer; + } + + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + fillHoles(thisIndex); + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + lastSet = thisIndex; + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableVarBinaryVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + fillEmpties(thisIndex); + if (from.isSet(fromIndex) != 0) { + setSafe(thisIndex, from.get(fromIndex)); + lastSet = thisIndex; + } + } + + + /** + * Set the variable length element at the specified index to the supplied + * byte array. This is same as using {@link #set(int, byte[], int, int)} + * with start as 0 and length as value.length + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void set(int index, byte[] value) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[])} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void setSafe(int index, byte[] value) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, value.length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the supplied + * byte array. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void set(int index, byte[] value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[], int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void setSafe(int index, byte[] value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the + * content in supplied ByteBuffer + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void set(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void setSafe(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, VarBinaryHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, VarBinaryHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, VarBinaryHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, NullableVarBinaryHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, NullableVarBinaryHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, NullableVarBinaryHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Sets the value length for an element. + * + * @param index position of the element to set + * @param length length of the element + */ + public void setValueLengthSafe(int index, int length) { + assert index >= 0; + handleSafe(index, length); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index, 0); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, isSet); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end); + final ArrowBuf bb = buffer.slice(start, end); + valueBuffer.setBytes(startOffset, bb); + lastSet = index; + } + + /** + * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { + assert index >= 0; + handleSafe(index, end); + set(index, isSet, start, end, buffer); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableVarBinaryVector)to); + } + + private class TransferImpl implements TransferPair { + NullableVarBinaryVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableVarBinaryVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableVarBinaryVector to){ + this.to = to; + } + + @Override + public NullableVarBinaryVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableVarBinaryVector.this); + } + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java index b9021544b0e..0b03164f0f3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,7 +14,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector; @@ -32,30 +31,50 @@ import java.nio.ByteBuffer; +/** + * NullableVarCharVector implements a variable width vector of VARCHAR + * values which could be NULL. A validity buffer (bit vector) is maintained + * to track which elements in the vector are null. + */ public class NullableVarCharVector extends BaseNullableVariableWidthVector { - private static final org.slf4j.Logger logger = - org.slf4j.LoggerFactory.getLogger(NullableIntVector.class); private final FieldReader reader; + /** + * Instantiate a NullableVarCharVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ public NullableVarCharVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator); } + /** + * Instantiate a NullableVarCharVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType); reader = new VarCharReaderImpl(NullableVarCharVector.this); } - @Override - protected org.slf4j.Logger getLogger() { - return logger; - } - + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ @Override public FieldReader getReader(){ return reader; } + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ @Override public Types.MinorType getMinorType() { return Types.MinorType.VARCHAR; @@ -154,7 +173,13 @@ public void get(int index, NullableVarCharHolder holder){ ******************************************************************/ - + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { fillHoles(thisIndex); if (from.isSet(fromIndex) != 0) { @@ -163,6 +188,14 @@ public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { } } + /** + * Same as {@link #copyFrom(int, int, NullableVarCharVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) { fillEmpties(thisIndex); if (from.isSet(fromIndex) != 0) { @@ -383,21 +416,42 @@ public void setNull(int index){ BitVectorHelper.setValidityBit(validityBuffer, index, 0); } - public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { assert index >= 0; fillHoles(index); BitVectorHelper.setValidityBit(validityBuffer, index, isSet); final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + endField); - final ArrowBuf bb = bufferField.slice(startField, endField); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end); + final ArrowBuf bb = buffer.slice(start, end); valueBuffer.setBytes(startOffset, bb); lastSet = index; } - public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + /** + * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { assert index >= 0; - handleSafe(index, endField); - set(index, isSet, startField, endField, bufferField); + handleSafe(index, end); + set(index, isSet, start, end, buffer); } @@ -407,11 +461,23 @@ public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf * * ******************************************************************/ + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator){ return new TransferImpl(ref, allocator); } + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ @Override public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((NullableVarCharVector)to); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index e6048b4b517..dc74afaa2e4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -256,9 +256,17 @@ interface Mutator { */ public ArrowBuf getOffsetBuffer(); + /* temporarily add these methods here until we remove other vectors + * (non-nullable) which are under ValueVector hierarchy and still + * use the mutator/accessor interfaces. + */ public int getValueCount(); public void setValueCount(int valueCount); public Object getObject(int index); + + public int getNullCount(); + + public boolean isNull(int index); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java index 0c8868cad55..196393fb958 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java @@ -38,7 +38,7 @@ public class VectorSchemaRoot implements AutoCloseable { private final Map fieldVectorsMap = new HashMap<>(); public VectorSchemaRoot(FieldVector parent) { - this(parent.getField().getChildren(), parent.getChildrenFromFields(), parent.getAccessor().getValueCount()); + this(parent.getField().getChildren(), parent.getChildrenFromFields(), parent.getValueCount()); } public VectorSchemaRoot(List fields, List fieldVectors, int rowCount) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java index 96243eb5876..288afc6bbec 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java @@ -55,17 +55,7 @@ public ArrowRecordBatch getRecordBatch() { } private void appendNodes(FieldVector vector, List nodes, List buffers) { - Accessor accessor = null; - if (vector instanceof NullableIntVector) { - nodes.add(new ArrowFieldNode(((NullableIntVector)vector).getValueCount(), - includeNullCount ? ((NullableIntVector)vector).getNullCount() : -1)); - } else if (vector instanceof NullableVarCharVector) { - nodes.add(new ArrowFieldNode(((NullableVarCharVector)vector).getValueCount(), - includeNullCount ? ((NullableVarCharVector)vector).getNullCount() : -1)); - } else { - accessor = vector.getAccessor(); - nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1)); - } + nodes.add(new ArrowFieldNode(vector.getValueCount(), includeNullCount ? vector.getNullCount() : -1)); List fieldBuffers = vector.getFieldBuffers(); List expectedBuffers = vector.getField().getTypeLayout().getVectorTypes(); if (fieldBuffers.size() != expectedBuffers.size()) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index 804e001746e..8a77065ca89 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -268,4 +268,10 @@ public void setValueCount(int valueCount) { } @Override public Object getObject(int index) { return null; } + + @Override + public int getNullCount() { return 0; } + + @Override + public boolean isNull(int index) { return false; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java index 26b0f90581f..1b20b09d81b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java @@ -141,8 +141,8 @@ public T addOrGet(String childName, FieldType fieldType, } private boolean nullFilled(ValueVector vector) { - for (int r = 0; r < vector.getAccessor().getValueCount(); r++) { - if (!vector.getAccessor().isNull(r)) { + for (int r = 0; r < vector.getValueCount(); r++) { + if (!vector.isNull(r)) { return false; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index 8e2877f892a..3eab30a32d2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.Iterator; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.AddOrGetResult; import org.apache.arrow.vector.BaseValueVector; @@ -31,6 +32,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.SchemaChangeRuntimeException; import com.google.common.base.Preconditions; @@ -41,12 +43,14 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector { public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE; - public final static String OFFSETS_VECTOR_NAME = "$offsets$"; public final static String DATA_VECTOR_NAME = "$data$"; - protected final UInt4Vector offsets; + public final static byte OFFSET_WIDTH = 4; + protected ArrowBuf offsetBuffer; protected FieldVector vector; protected final CallBack callBack; + protected int valueCount; + protected int offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; protected BaseRepeatedValueVector(String name, BufferAllocator allocator, CallBack callBack) { this(name, allocator, DEFAULT_DATA_VECTOR, callBack); @@ -54,42 +58,73 @@ protected BaseRepeatedValueVector(String name, BufferAllocator allocator, CallBa protected BaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector, CallBack callBack) { super(name, allocator); - this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator); + this.offsetBuffer = allocator.getEmpty(); this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null"); this.callBack = callBack; + this.valueCount = 0; } @Override public boolean allocateNewSafe() { - /* boolean to keep track if all the memory allocation were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; + boolean dataAlloc = false; try { - if (!offsets.allocateNewSafe()) { - return false; - } - success = vector.allocateNewSafe(); - } finally { - if (!success) { + allocateOffsetBuffer(offsetAllocationSizeInBytes); + dataAlloc = vector.allocateNewSafe(); + } catch (Exception e) { + e.printStackTrace(); + clear(); + return false; + } + finally { + if (!dataAlloc) { clear(); } } - offsets.zeroVector(); - return success; + return dataAlloc; + } + + protected void allocateOffsetBuffer(final long size) { + final int curSize = (int)size; + offsetBuffer = allocator.buffer(curSize); + offsetBuffer.readerIndex(0); + offsetAllocationSizeInBytes = curSize; + offsetBuffer.setZero(0, offsetBuffer.capacity()); } @Override public void reAlloc() { - offsets.reAlloc(); + reallocOffsetBuffer(); vector.reAlloc(); } + protected void reallocOffsetBuffer() { + final int currentBufferCapacity = offsetBuffer.capacity(); + long baseSize = offsetAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, offsetBuffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + offsetBuffer.release(1); + offsetBuffer = newBuf; + offsetAllocationSizeInBytes = (int)newAllocationSize; + } + @Override + @Deprecated public UInt4Vector getOffsetVector() { - return offsets; + throw new UnsupportedOperationException("There is no inner offset vector"); } @Override @@ -99,25 +134,29 @@ public FieldVector getDataVector() { @Override public void setInitialCapacity(int numRecords) { - offsets.setInitialCapacity(numRecords + 1); + offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH; vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); } @Override public int getValueCapacity() { - final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0); + final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); if (vector == DEFAULT_DATA_VECTOR) { return offsetValueCapacity; } return Math.min(vector.getValueCapacity(), offsetValueCapacity); } + private int getOffsetBufferValueCapacity() { + return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); + } + @Override public int getBufferSize() { - if (getAccessor().getValueCount() == 0) { + if (getValueCount() == 0) { return 0; } - return offsets.getBufferSize() + vector.getBufferSize(); + return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSize(); } @Override @@ -126,7 +165,7 @@ public int getBufferSizeFor(int valueCount) { return 0; } - return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount); + return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSizeFor(valueCount); } @Override @@ -136,14 +175,16 @@ public Iterator iterator() { @Override public void clear() { - offsets.clear(); + offsetBuffer = releaseBuffer(offsetBuffer); vector.clear(); + valueCount = 0; super.clear(); } @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class); + final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, + vector.getBuffers(false), ArrowBuf.class); if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); @@ -187,54 +228,61 @@ protected void replaceDataVector(FieldVector v) { vector = v; } - public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor { - @Override - public int getValueCount() { - return Math.max(offsets.getAccessor().getValueCount() - 1, 0); - } + @Override + public int getValueCount() { + return valueCount; + } - @Override - public int getInnerValueCount() { - return vector.getAccessor().getValueCount(); - } + /* returns the value count for inner data vector for this list vector */ + public int getInnerValueCount() { + return vector.getValueCount(); + } - @Override - public int getInnerValueCountAt(int index) { - return offsets.getAccessor().get(index + 1) - offsets.getAccessor().get(index); - } - @Override - public boolean isNull(int index) { - return false; - } + /* returns the value count for inner data vector at a particular index */ + public int getInnerValueCountAt(int index) { + return offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - + offsetBuffer.getInt(index * OFFSET_WIDTH); + } - @Override - public boolean isEmpty(int index) { - return false; - } + public boolean isNull(int index) { + return false; } - public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator { + public boolean isEmpty(int index) { + return false; + } - @Override - public int startNewValue(int index) { - while (offsets.getValueCapacity() <= index) { - offsets.reAlloc(); - } - int offset = offsets.getAccessor().get(index); - offsets.getMutator().setSafe(index + 1, offset); - setValueCount(index + 1); - return offset; + public int startNewValue(int index) { + while (index >= getOffsetBufferValueCapacity()) { + reallocOffsetBuffer(); } + int offset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, offset); + setValueCount(index + 1); + return offset; + } - @Override - public void setValueCount(int valueCount) { - // TODO: populate offset end points - offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount + 1); - final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); - vector.getMutator().setValueCount(childValueCount); + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + while (valueCount > getOffsetBufferValueCapacity()) { + reallocOffsetBuffer(); } + final int childValueCount = valueCount == 0 ? 0 : + offsetBuffer.getInt(valueCount * OFFSET_WIDTH); + vector.setValueCount(childValueCount); + } + + @Override + @Deprecated + public RepeatedAccessor getAccessor() { + throw new UnsupportedOperationException("Accessor is not supported for reading from LIST."); } + @Override + @Deprecated + public RepeatedMutator getMutator() { + throw new UnsupportedOperationException("Mutator is not supported for writing to LIST"); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index e95442acf6d..bb091f68c18 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -22,16 +22,14 @@ import static java.util.Collections.singletonList; import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; +import java.util.*; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ObjectArrays; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.vector.*; @@ -42,10 +40,7 @@ import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.JsonStringArrayList; -import org.apache.arrow.vector.util.SchemaChangeRuntimeException; -import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.*; public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector { @@ -55,15 +50,13 @@ public static FixedSizeListVector empty(String name, int size, BufferAllocator a } private FieldVector vector; - private final BitVector bits; + private ArrowBuf validityBuffer; private final int listSize; private final FieldType fieldType; - private final List innerVectors; private UnionFixedSizeListReader reader; - - private Mutator mutator = new Mutator(); - private Accessor accessor = new Accessor(); + private int valueCount; + private int validityAllocationSizeInBytes; // deprecated, use FieldType or static constructor instead @Deprecated @@ -80,13 +73,14 @@ public FixedSizeListVector(String name, FieldType fieldType, CallBack schemaChangeCallback) { super(name, allocator); - this.bits = new BitVector("$bits$", allocator); + this.validityBuffer = allocator.getEmpty(); this.vector = ZeroVector.INSTANCE; this.fieldType = fieldType; this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize(); Preconditions.checkArgument(listSize > 0, "list size must be positive"); - this.innerVectors = Collections.singletonList((BufferBacked) bits); this.reader = new UnionFixedSizeListReader(this); + this.valueCount = 0; + this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } @Override @@ -124,27 +118,47 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - BaseDataValueVector.load(fieldNode, innerVectors, ownBuffers); + if (ownBuffers.size() != 1) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + valueCount = fieldNode.getLength(); + + validityAllocationSizeInBytes = validityBuffer.capacity(); } @Override public List getFieldBuffers() { - return BaseDataValueVector.unload(innerVectors); + List result = new ArrayList<>(1); + + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + + result.add(validityBuffer); + + return result; } @Override + @Deprecated public List getFieldInnerVectors() { - return innerVectors; + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); } @Override + @Deprecated public Accessor getAccessor() { - return accessor; + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); } @Override + @Deprecated public Mutator getMutator() { - return mutator; + throw new UnsupportedOperationException("Mutator is not supported for writing to vector"); } @Override @@ -154,7 +168,9 @@ public UnionFixedSizeListReader getReader() { @Override public void allocateNew() throws OutOfMemoryException { - allocateNewSafe(); + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory"); + } } @Override @@ -166,31 +182,63 @@ public boolean allocateNewSafe() { */ boolean success = false; try { - success = bits.allocateNewSafe() && vector.allocateNewSafe(); + /* allocate validity buffer */ + allocateValidityBuffer(validityAllocationSizeInBytes); + success = vector.allocateNewSafe(); } finally { if (!success) { clear(); + return false; } } - if (success) { - bits.zeroVector(); - } - return success; + + return true; + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int)size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + validityBuffer.setZero(0, validityBuffer.capacity()); } @Override public void reAlloc() { - bits.reAlloc(); + reallocValidityBuffer(); vector.reAlloc(); } + private void reallocValidityBuffer() { + final int currentBufferCapacity = validityBuffer.capacity(); + long baseSize = validityAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setZero(0, newBuf.capacity()); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + validityBuffer.release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int)newAllocationSize; + } + public FieldVector getDataVector() { return vector; } @Override public void setInitialCapacity(int numRecords) { - bits.setInitialCapacity(numRecords); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); vector.setInitialCapacity(numRecords * listSize); } @@ -199,15 +247,15 @@ public int getValueCapacity() { if (vector == ZeroVector.INSTANCE) { return 0; } - return vector.getValueCapacity() / listSize; + return Math.min(vector.getValueCapacity() / listSize, getValidityBufferValueCapacity()); } @Override public int getBufferSize() { - if (accessor.getValueCount() == 0) { + if (getValueCount() == 0) { return 0; } - return bits.getBufferSize() + vector.getBufferSize(); + return getValidityBufferSizeFromCount(valueCount) + vector.getBufferSize(); } @Override @@ -215,7 +263,8 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - return bits.getBufferSizeFor(valueCount) + vector.getBufferSizeFor(valueCount * listSize); + return getValidityBufferSizeFromCount(valueCount) + + vector.getBufferSizeFor(valueCount * listSize); } @Override @@ -225,14 +274,15 @@ public Iterator iterator() { @Override public void clear() { - bits.clear(); + validityBuffer = releaseBuffer(validityBuffer); vector.clear(); + valueCount = 0; super.clear(); } @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), vector.getBuffers(false), ArrowBuf.class); + final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), ArrowBuf.class); if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); @@ -288,7 +338,7 @@ public UnionVector promoteToUnion() { @Override public long getValidityBufferAddress() { - return (bits.getDataBuffer().memoryAddress()); + return validityBuffer.memoryAddress(); } @Override @@ -303,7 +353,7 @@ public long getOffsetBufferAddress() { @Override public ArrowBuf getValidityBuffer() { - return (bits.getDataBuffer()); + return validityBuffer; } @Override @@ -316,62 +366,64 @@ public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } - public class Accessor extends BaseValueVector.BaseAccessor { - - @Override - public Object getObject(int index) { - if (isNull(index)) { - return null; - } - final List vals = new JsonStringArrayList<>(listSize); - if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { - for (int i = 0; i < listSize; i++) { - vals.add(vector.getObject(index * listSize + i)); - } - } else { - final ValueVector.Accessor valuesAccessor = vector.getAccessor(); - for (int i = 0; i < listSize; i++) { - vals.add(valuesAccessor.getObject(index * listSize + i)); - } - } - return vals; + @Override + public Object getObject(int index) { + if (isSet(index) == 0) { + return null; } - - @Override - public boolean isNull(int index) { - return bits.getAccessor().get(index) == 0; + final List vals = new JsonStringArrayList<>(listSize); + for (int i = 0; i < listSize; i++) { + vals.add(vector.getObject(index * listSize + i)); } + return vals; + } - @Override - public int getNullCount() { - return bits.getAccessor().getNullCount(); - } + public boolean isNull(int index) { + return (isSet(index) == 0); + } - @Override - public int getValueCount() { - return bits.getAccessor().getValueCount(); - } + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); } - public class Mutator extends BaseValueVector.BaseMutator { + @Override + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } - public void setNull(int index) { - bits.getMutator().setSafe(index, 0); + @Override + public int getValueCount() { + return valueCount; + } + + private int getValidityBufferValueCapacity() { + return (int)(validityBuffer.capacity() * 8L); + } + + public void setNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + reallocValidityBuffer(); } + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } - public void setNotNull(int index) { - bits.getMutator().setSafe(index, 1); + public void setNotNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + reallocValidityBuffer(); } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } - @Override - public void setValueCount(int valueCount) { - bits.getMutator().setValueCount(valueCount); - if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { - vector.setValueCount(valueCount * listSize); - } else { - vector.getMutator().setValueCount(valueCount * listSize); - } + @Override + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + while (valueCount > getValidityBufferValueCapacity()) { + reallocValidityBuffer(); } + vector.setValueCount(valueCount * listSize); } @Override @@ -392,7 +444,7 @@ public TransferPair makeTransferPair(ValueVector target) { private class TransferImpl implements TransferPair { FixedSizeListVector to; - TransferPair pairs[] = new TransferPair[2]; + TransferPair dataPair; public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { this(new FixedSizeListVector(name, allocator, fieldType, callBack)); @@ -401,19 +453,20 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { public TransferImpl(FixedSizeListVector to) { this.to = to; to.addOrGetVector(vector.getField().getFieldType()); - pairs[0] = bits.makeTransferPair(to.bits); - pairs[1] = vector.makeTransferPair(to.vector); + dataPair = vector.makeTransferPair(to.vector); } @Override public void transfer() { - for (TransferPair pair : pairs) { - pair.transfer(); - } + to.clear(); + dataPair.transfer(); + to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer; + to.setValueCount(valueCount); } @Override public void splitAndTransfer(int startIndex, int length) { + to.clear(); to.allocateNew(); for (int i = 0; i < length; i++) { copyValueSafe(startIndex + i, i); @@ -426,12 +479,15 @@ public ValueVector getTo() { } @Override - public void copyValueSafe(int from, int to) { - pairs[0].copyValueSafe(from, to); - int fromOffset = from * listSize; - int toOffset = to * listSize; + public void copyValueSafe(int fromIndex, int toIndex) { + while (toIndex >= to.getValueCapacity()) { + to.reAlloc(); + } + BitVectorHelper.setValidityBit(to.validityBuffer, toIndex, isSet(fromIndex)); + int fromOffset = fromIndex * listSize; + int toOffset = toIndex * listSize; for (int i = 0; i < listSize; i++) { - pairs[1].copyValueSafe(fromOffset + i, toOffset + i); + dataPair.copyValueSafe(fromOffset + i, toOffset + i); } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java new file mode 100644 index 00000000000..435d3d8ccf2 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.complex; + +import java.util.Collections; +import java.util.Iterator; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.AddOrGetResult; +import org.apache.arrow.vector.BaseValueVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.UInt4Vector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.SchemaChangeRuntimeException; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; + +public abstract class LegacyBaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector { + + public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE; + public final static String OFFSETS_VECTOR_NAME = "$offsets$"; + public final static String DATA_VECTOR_NAME = "$data$"; + + protected final UInt4Vector offsets; + protected FieldVector vector; + protected final CallBack callBack; + + protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, CallBack callBack) { + this(name, allocator, DEFAULT_DATA_VECTOR, callBack); + } + + protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector, CallBack callBack) { + super(name, allocator); + this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator); + this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null"); + this.callBack = callBack; + } + + @Override + public boolean allocateNewSafe() { + /* boolean to keep track if all the memory allocation were successful + * Used in the case of composite vectors when we need to allocate multiple + * buffers for multiple vectors. If one of the allocations failed we need to + * clear all the memory that we allocated + */ + boolean success = false; + try { + if (!offsets.allocateNewSafe()) { + return false; + } + success = vector.allocateNewSafe(); + } finally { + if (!success) { + clear(); + } + } + offsets.zeroVector(); + return success; + } + + @Override + public void reAlloc() { + offsets.reAlloc(); + vector.reAlloc(); + } + + @Override + public UInt4Vector getOffsetVector() { + return offsets; + } + + @Override + public FieldVector getDataVector() { + return vector; + } + + @Override + public void setInitialCapacity(int numRecords) { + offsets.setInitialCapacity(numRecords + 1); + vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); + } + + @Override + public int getValueCapacity() { + final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0); + if (vector == DEFAULT_DATA_VECTOR) { + return offsetValueCapacity; + } + return Math.min(vector.getValueCapacity(), offsetValueCapacity); + } + + @Override + public int getBufferSize() { + if (getAccessor().getValueCount() == 0) { + return 0; + } + return offsets.getBufferSize() + vector.getBufferSize(); + } + + @Override + public int getBufferSizeFor(int valueCount) { + if (valueCount == 0) { + return 0; + } + + return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount); + } + + @Override + public Iterator iterator() { + return Collections.singleton(getDataVector()).iterator(); + } + + @Override + public void clear() { + offsets.clear(); + vector.clear(); + super.clear(); + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class); + if (clear) { + for (ArrowBuf buffer : buffers) { + buffer.retain(); + } + clear(); + } + return buffers; + } + + /** + * @return 1 if inner vector is explicitly set via #addOrGetVector else 0 + */ + public int size() { + return vector == DEFAULT_DATA_VECTOR ? 0 : 1; + } + + public AddOrGetResult addOrGetVector(FieldType fieldType) { + boolean created = false; + if (vector instanceof ZeroVector) { + vector = fieldType.createNewSingleVector(DATA_VECTOR_NAME, allocator, callBack); + // returned vector must have the same field + created = true; + if (callBack != null && + // not a schema change if changing from ZeroVector to ZeroVector + (fieldType.getType().getTypeID() != ArrowTypeID.Null)) { + callBack.doWork(); + } + } + + if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) { + final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", + fieldType.getType().getTypeID(), vector.getField().getType().getTypeID()); + throw new SchemaChangeRuntimeException(msg); + } + + return new AddOrGetResult<>((T) vector, created); + } + + protected void replaceDataVector(FieldVector v) { + vector.clear(); + vector = v; + } + + public abstract class LegacyBaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor { + + @Override + public int getValueCount() { + return Math.max(offsets.getAccessor().getValueCount() - 1, 0); + } + + @Override + public int getInnerValueCount() { + return vector.getAccessor().getValueCount(); + } + + @Override + public int getInnerValueCountAt(int index) { + return offsets.getAccessor().get(index + 1) - offsets.getAccessor().get(index); + } + + @Override + public boolean isNull(int index) { + return false; + } + + @Override + public boolean isEmpty(int index) { + return false; + } + } + + public abstract class LegacyBaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator { + + @Override + public int startNewValue(int index) { + while (offsets.getValueCapacity() <= index) { + offsets.reAlloc(); + } + int offset = offsets.getAccessor().get(index); + offsets.getMutator().setSafe(index + 1, offset); + setValueCount(index + 1); + return offset; + } + + @Override + public void setValueCount(int valueCount) { + // TODO: populate offset end points + offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount + 1); + final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); + vector.getMutator().setValueCount(childValueCount); + } + } + +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java new file mode 100644 index 00000000000..0843907d9df --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java @@ -0,0 +1,298 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector.complex; + +import static java.util.Collections.singletonList; +import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; +import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.JsonStringArrayList; +import org.apache.arrow.vector.util.SchemaChangeRuntimeException; +import org.apache.arrow.vector.util.TransferPair; + +public class LegacyFixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector { + + public static LegacyFixedSizeListVector empty(String name, int size, BufferAllocator allocator) { + FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size)); + return new LegacyFixedSizeListVector(name, allocator, fieldType, null); + } + + private Mutator mutator = new Mutator(); + private Accessor accessor = new Accessor(); + /* delegate */ + private final FixedSizeListVector fixedSizeListVector; + + // deprecated, use FieldType or static constructor instead + @Deprecated + public LegacyFixedSizeListVector(String name, + BufferAllocator allocator, + int listSize, + DictionaryEncoding dictionary, + CallBack schemaChangeCallback) { + super(name, allocator); + fixedSizeListVector = new FixedSizeListVector(name, allocator, listSize, dictionary, schemaChangeCallback); + } + + public LegacyFixedSizeListVector(String name, + BufferAllocator allocator, + FieldType fieldType, + CallBack schemaChangeCallback) { + super(name, allocator); + fixedSizeListVector = new FixedSizeListVector(name, allocator, fieldType, schemaChangeCallback); + } + + @Override + public Field getField() { + return fixedSizeListVector.getField(); + } + + @Override + public MinorType getMinorType() { + return MinorType.FIXED_SIZE_LIST; + } + + public int getListSize() { + return fixedSizeListVector.getListSize(); + } + + @Override + public void initializeChildrenFromFields(List children) { + fixedSizeListVector.initializeChildrenFromFields(children); + } + + @Override + public List getChildrenFromFields() { + return fixedSizeListVector.getChildrenFromFields(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + fixedSizeListVector.loadFieldBuffers(fieldNode, ownBuffers); + } + + @Override + public List getFieldBuffers() { + return fixedSizeListVector.getFieldBuffers(); + } + + @Override + public List getFieldInnerVectors() { + return fixedSizeListVector.getFieldInnerVectors(); + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + @Override + public UnionFixedSizeListReader getReader() { + return fixedSizeListVector.getReader(); + } + + @Override + public void allocateNew() throws OutOfMemoryException { + fixedSizeListVector.allocateNew(); + } + + @Override + public boolean allocateNewSafe() { + return fixedSizeListVector.allocateNewSafe(); + } + + @Override + public void reAlloc() { + fixedSizeListVector.reAlloc(); + } + + public FieldVector getDataVector() { + return fixedSizeListVector.getDataVector(); + } + + @Override + public void setInitialCapacity(int numRecords) { + fixedSizeListVector.setInitialCapacity(numRecords); + } + + @Override + public int getValueCapacity() { + return fixedSizeListVector.getValueCapacity(); + } + + @Override + public int getBufferSize() { + return fixedSizeListVector.getBufferSize(); + } + + @Override + public int getBufferSizeFor(int valueCount) { + return fixedSizeListVector.getBufferSizeFor(valueCount); + } + + @Override + public Iterator iterator() { + return fixedSizeListVector.iterator(); + } + + @Override + public void clear() { + fixedSizeListVector.clear(); + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + return fixedSizeListVector.getBuffers(clear); + } + + /** + * @return 1 if inner vector is explicitly set via #addOrGetVector else 0 + */ + public int size() { + return fixedSizeListVector.size(); + } + + @Override + @SuppressWarnings("unchecked") + public AddOrGetResult addOrGetVector(FieldType type) { + return fixedSizeListVector.addOrGetVector(type); + } + + public void copyFromSafe(int inIndex, int outIndex, FixedSizeListVector from) { + fixedSizeListVector.copyFromSafe(inIndex, outIndex, from); + } + + public void copyFrom(int fromIndex, int thisIndex, FixedSizeListVector from) { + fixedSizeListVector.copyFrom(fromIndex, thisIndex, from); + } + + @Override + public UnionVector promoteToUnion() { + return fixedSizeListVector.promoteToUnion(); + } + + @Override + public long getValidityBufferAddress() { + return fixedSizeListVector.getValidityBufferAddress(); + } + + @Override + public long getDataBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getValidityBuffer() { + return fixedSizeListVector.getValidityBuffer(); + } + + @Override + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException(); + } + + public class Accessor extends BaseValueVector.BaseAccessor { + + @Override + public Object getObject(int index) { + return fixedSizeListVector.getObject(index); + } + + @Override + public boolean isNull(int index) { + return fixedSizeListVector.isNull(index); + } + + @Override + public int getNullCount() { + return fixedSizeListVector.getNullCount(); + } + + @Override + public int getValueCount() { + return fixedSizeListVector.getValueCount(); + } + } + + public class Mutator extends BaseValueVector.BaseMutator { + + public void setNull(int index) { + fixedSizeListVector.setNull(index); + } + + public void setNotNull(int index) { + fixedSizeListVector.setNotNull(index); + } + + @Override + public void setValueCount(int valueCount) { + fixedSizeListVector.setValueCount(valueCount); + } + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return fixedSizeListVector.getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return fixedSizeListVector.getTransferPair(ref, allocator, callBack); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return fixedSizeListVector.makeTransferPair(((LegacyFixedSizeListVector)target).fixedSizeListVector); + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java new file mode 100644 index 00000000000..732699ada7a --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java @@ -0,0 +1,288 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector.complex; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Collections.list; +import static java.util.Collections.singletonList; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.impl.ComplexCopier; +import org.apache.arrow.vector.complex.impl.UnionListReader; +import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.Null; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.JsonStringArrayList; +import org.apache.arrow.vector.util.TransferPair; + +public class LegacyListVector extends LegacyBaseRepeatedValueVector implements FieldVector, PromotableVector { + + public static LegacyListVector empty(String name, BufferAllocator allocator) { + return new LegacyListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); + } + + private Mutator mutator = new Mutator(); + private Accessor accessor = new Accessor(); + + private final ListVector listVector; + + // deprecated, use FieldType or static constructor instead + @Deprecated + public LegacyListVector(String name, BufferAllocator allocator, CallBack callBack) { + super(name, allocator, callBack); + listVector = new ListVector(name, allocator, callBack); + } + + // deprecated, use FieldType or static constructor instead + @Deprecated + public LegacyListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { + super(name, allocator, callBack); + listVector = new ListVector(name, allocator, dictionary, callBack); + } + + public LegacyListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, allocator, callBack); + listVector = new ListVector(name, allocator, fieldType, callBack); + } + + @Override + public void initializeChildrenFromFields(List children) { + listVector.initializeChildrenFromFields(children); + } + + @Override + public List getChildrenFromFields() { + return listVector.getChildrenFromFields(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + listVector.loadFieldBuffers(fieldNode, ownBuffers); + } + + @Override + public List getFieldBuffers() { + return listVector.getFieldBuffers(); + } + + @Override + public List getFieldInnerVectors() { + return listVector.getFieldInnerVectors(); + } + + public UnionListWriter getWriter() { + return listVector.getWriter(); + } + + @Override + public void allocateNew() throws OutOfMemoryException { + listVector.allocateNew(); + } + + @Override + public void reAlloc() { + listVector.reAlloc(); + } + + public void copyFromSafe(int inIndex, int outIndex, ListVector from) { + listVector.copyFrom(inIndex, outIndex, from); + } + + public void copyFrom(int inIndex, int outIndex, ListVector from) { + listVector.copyFrom(inIndex, outIndex, from); + } + + @Override + public FieldVector getDataVector() { + return listVector.getDataVector(); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return listVector.getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return listVector.getTransferPair(ref, allocator, callBack); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return listVector.makeTransferPair(((LegacyListVector)target).listVector); + } + + @Override + public long getValidityBufferAddress() { + return listVector.getValidityBufferAddress(); + } + + @Override + public long getDataBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public long getOffsetBufferAddress() { + return listVector.getOffsetBufferAddress(); + } + + @Override + public ArrowBuf getValidityBuffer() { + return listVector.getValidityBuffer(); + } + + @Override + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getOffsetBuffer() { + return listVector.getOffsetBuffer(); + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + @Override + public UnionListReader getReader() { + return listVector.getReader(); + } + + @Override + public boolean allocateNewSafe() { + return listVector.allocateNewSafe(); + } + + public AddOrGetResult addOrGetVector(FieldType fieldType) { + return listVector.addOrGetVector(fieldType); + } + + @Override + public int getBufferSize() { + return listVector.getBufferSize(); + } + + @Override + public Field getField() { + return listVector.getField(); + } + + @Override + public MinorType getMinorType() { + return MinorType.LIST; + } + + @Override + public void clear() { + listVector.clear(); + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + return listVector.getBuffers(clear); + } + + @Override + public UnionVector promoteToUnion() { + return listVector.promoteToUnion(); + } + + private int lastSet = 0; + + public class Accessor extends LegacyBaseRepeatedAccessor { + + @Override + public Object getObject(int index) { + return listVector.getObject(index); + } + + @Override + public boolean isNull(int index) { + return listVector.isNull(index); + } + + @Override + public int getNullCount() { + return listVector.getNullCount(); + } + } + + public class Mutator extends LegacyBaseRepeatedMutator { + public void setNotNull(int index) { + listVector.setNotNull(index); + } + + @Override + public int startNewValue(int index) { + return listVector.startNewValue(index); + } + + /** + * End the current value + * + * @param index index of the value to end + * @param size number of elements in the list that was written + */ + public void endValue(int index, int size) { + listVector.endValue(index, size); + } + + @Override + public void setValueCount(int valueCount) { + listVector.setValueCount(valueCount); + } + + public void setLastSet(int value) { + listVector.setLastSet(value); + } + + public int getLastSet() { + return listVector.getLastSet(); + } + } + +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java new file mode 100644 index 00000000000..77556ab774f --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.complex; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; +import com.google.common.primitives.Ints; + +import io.netty.buffer.ArrowBuf; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.ComplexHolder; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.JsonStringHashMap; +import org.apache.arrow.vector.util.TransferPair; + +public class LegacyMapVector extends AbstractMapVector { + + /* delegate */ + private final MapVector mapVector; + + public static LegacyMapVector empty(String name, BufferAllocator allocator) { + FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); + return new LegacyMapVector(name, allocator, fieldType, null); + } + + private final Accessor accessor = new Accessor(); + private final Mutator mutator = new Mutator(); + + @Deprecated + public LegacyMapVector(String name, BufferAllocator allocator, CallBack callBack) { + super(name, allocator, callBack); + mapVector = new MapVector(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); + } + + public LegacyMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, allocator, callBack); + mapVector = new MapVector(name, allocator, fieldType, callBack); + } + + @Override + public FieldReader getReader() { + return mapVector.getReader(); + } + + public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) { + mapVector.copyFromSafe(fromIndex, thisIndex, from); + } + + @Override + protected boolean supportsDirectRead() { + return true; + } + + public Iterator fieldNameIterator() { + return mapVector.fieldNameIterator(); + } + + @Override + public void setInitialCapacity(int numRecords) { + mapVector.setInitialCapacity(numRecords); + } + + @Override + public int getBufferSize() { + return mapVector.getBufferSize(); + } + + @Override + public int getBufferSizeFor(final int valueCount) { + return (int) mapVector.getBufferSizeFor(valueCount); + } + + @Override + public ArrowBuf getValidityBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return getTransferPair(name, allocator, null); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return mapVector.getTransferPair(ref, allocator, callBack); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return mapVector.makeTransferPair(((LegacyMapVector)to).mapVector); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return mapVector.getTransferPair(ref, allocator); + } + + @Override + public int getValueCapacity() { + return mapVector.getValueCapacity(); + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + public class Accessor extends BaseValueVector.BaseAccessor { + + @Override + public Object getObject(int index) { + return mapVector.getObject(index); + } + + public void get(int index, ComplexHolder holder) { + mapVector.get(index, holder); + } + + @Override + public int getValueCount() { + return mapVector.getValueCount(); + } + } + + public ValueVector getVectorById(int id) { + return mapVector.getVectorById(id); + } + + public class Mutator extends BaseValueVector.BaseMutator { + + @Override + public void setValueCount(int valueCount) { + mapVector.setValueCount(valueCount); + } + + @Override + public void reset() { } + + @Override + public void generateTestData(int values) { } + } + + @Override + public void clear() { + mapVector.clear(); + } + + @Override + public Field getField() { + return mapVector.getField(); + } + + @Override + public MinorType getMinorType() { + return MinorType.MAP; + } + + @Override + public void close() { + mapVector.close(); + } + + public void initializeChildrenFromFields(List children) { + mapVector.initializeChildrenFromFields(children); + } + + public List getChildrenFromFields() { + return mapVector.getChildren(); + } + + public boolean isNull(int index) { return false; } + + public int getNullCount() { return 0; } + + public int getValueCount() { return 0; } + + public void setValueCount(int valueCount) { } + + public Object getObject(int index) { return null; } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 258a46f553b..7869c36ad98 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -22,14 +22,14 @@ import static com.google.common.base.Preconditions.checkNotNull; import static java.util.Collections.singletonList; -import java.util.Arrays; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; import com.google.common.collect.ImmutableList; import com.google.common.collect.ObjectArrays; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.vector.*; @@ -47,6 +47,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.JsonStringArrayList; +import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector { @@ -55,14 +56,12 @@ public static ListVector empty(String name, BufferAllocator allocator) { return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); } - final UInt4Vector offsets; - final BitVector bits; - private final List innerVectors; - private Mutator mutator = new Mutator(); - private Accessor accessor = new Accessor(); + private ArrowBuf validityBuffer; private UnionListReader reader; private CallBack callBack; private final FieldType fieldType; + private int validityAllocationSizeInBytes; + private int lastSet; // deprecated, use FieldType or static constructor instead @Deprecated @@ -78,12 +77,12 @@ public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dic public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { super(name, allocator, callBack); - this.bits = new BitVector("$bits$", allocator); - this.offsets = getOffsetVector(); - this.innerVectors = Collections.unmodifiableList(Arrays.asList(bits, offsets)); + this.validityBuffer = allocator.getEmpty(); this.reader = new UnionListReader(this); this.fieldType = checkNotNull(fieldType); this.callBack = callBack; + this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.lastSet = 0; } @Override @@ -107,20 +106,43 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - // variable width values: truncate offset vector buffer to size (#1) - org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, offsets.getBufferSizeFor(fieldNode.getLength() + 1)); - BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); + if (ownBuffers.size() != 2) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + ArrowBuf offBuffer = ownBuffers.get(1); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + offsetBuffer.release(); + offsetBuffer = offBuffer.retain(allocator); + + validityAllocationSizeInBytes = validityBuffer.capacity(); + offsetAllocationSizeInBytes = offsetBuffer.capacity(); + lastSet = fieldNode.getLength(); + valueCount = fieldNode.getLength(); } @Override public List getFieldBuffers() { - return BaseDataValueVector.unload(getFieldInnerVectors()); + List result = new ArrayList<>(2); + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + offsetBuffer.readerIndex(0); + offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); + + result.add(validityBuffer); + result.add(offsetBuffer); + + return result; } @Override + @Deprecated public List getFieldInnerVectors() { - return innerVectors; + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); } public UnionListWriter getWriter() { @@ -129,14 +151,70 @@ public UnionListWriter getWriter() { @Override public void allocateNew() throws OutOfMemoryException { - super.allocateNewSafe(); - bits.allocateNewSafe(); + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory"); + } + } + + public boolean allocateNewSafe() { + boolean success = false; + try { + /* allocate validity buffer */ + allocateValidityBuffer(validityAllocationSizeInBytes); + /* allocate offset and data buffer */ + success = super.allocateNewSafe(); + } finally { + if (!success) { + clear(); + return false; + } + } + return true; + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int)size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + validityBuffer.setZero(0, validityBuffer.capacity()); } @Override public void reAlloc() { + /* reallocate the validity buffer */ + reallocValidityBuffer(); + /* reallocate the offset and data */ super.reAlloc(); - bits.reAlloc(); + } + + private void reallocValidityAndOffsetBuffers() { + reallocOffsetBuffer(); + reallocValidityBuffer(); + } + + private void reallocValidityBuffer() { + final int currentBufferCapacity = validityBuffer.capacity(); + long baseSize = validityAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + validityBuffer.release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int)newAllocationSize; } public void copyFromSafe(int inIndex, int outIndex, ListVector from) { @@ -173,7 +251,7 @@ public TransferPair makeTransferPair(ValueVector target) { @Override public long getValidityBufferAddress() { - return (bits.getDataBuffer().memoryAddress()); + return (validityBuffer.memoryAddress()); } @Override @@ -183,11 +261,11 @@ public long getDataBufferAddress() { @Override public long getOffsetBufferAddress() { - return (offsets.getDataBuffer().memoryAddress()); + return (offsetBuffer.memoryAddress()); } @Override - public ArrowBuf getValidityBuffer() { return bits.getDataBuffer(); } + public ArrowBuf getValidityBuffer() { return validityBuffer; } @Override public ArrowBuf getDataBuffer() { @@ -195,17 +273,13 @@ public ArrowBuf getDataBuffer() { } @Override - public ArrowBuf getOffsetBuffer() { return offsets.getDataBuffer(); } + public ArrowBuf getOffsetBuffer() { return offsetBuffer; } private class TransferImpl implements TransferPair { ListVector to; - TransferPair bitsTransferPair; - TransferPair offsetsTransferPair; TransferPair dataTransferPair; - TransferPair[] pairs; - public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { this(new ListVector(name, allocator, fieldType, callBack)); } @@ -213,39 +287,99 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { public TransferImpl(ListVector to) { this.to = to; to.addOrGetVector(vector.getField().getFieldType()); - offsetsTransferPair = offsets.makeTransferPair(to.offsets); - bitsTransferPair = bits.makeTransferPair(to.bits); if (to.getDataVector() instanceof ZeroVector) { to.addOrGetVector(vector.getField().getFieldType()); } dataTransferPair = getDataVector().makeTransferPair(to.getDataVector()); - pairs = new TransferPair[] {bitsTransferPair, offsetsTransferPair, dataTransferPair}; } @Override public void transfer() { - for (TransferPair pair : pairs) { - pair.transfer(); - } + dataTransferPair.transfer(); + to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer; + to.offsetBuffer = offsetBuffer.transferOwnership(to.allocator).buffer; to.lastSet = lastSet; + to.setValueCount(valueCount); } @Override public void splitAndTransfer(int startIndex, int length) { - UInt4Vector.Accessor offsetVectorAccessor = ListVector.this.offsets.getAccessor(); - final int startPoint = offsetVectorAccessor.get(startIndex); - final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint; + final int startPoint = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); + final int sliceLength = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH) - startPoint; to.clear(); - to.offsets.allocateNew(length + 1); - offsetVectorAccessor = ListVector.this.offsets.getAccessor(); - final UInt4Vector.Mutator targetOffsetVectorMutator = to.offsets.getMutator(); + to.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH); + /* splitAndTransfer offset buffer */ for (int i = 0; i < length + 1; i++) { - targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint); + final int relativeOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - startPoint; + to.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeOffset); } - bitsTransferPair.splitAndTransfer(startIndex, length); + /* splitAndTransfer validity buffer */ + splitAndTransferValidityBuffer(startIndex, length, to); + /* splitAndTransfer data buffer */ dataTransferPair.splitAndTransfer(startPoint, sliceLength); to.lastSet = length; - to.mutator.setValueCount(length); + to.setValueCount(length); + } + + /* + * transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, ListVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } } @Override @@ -259,47 +393,11 @@ public void copyValueSafe(int from, int to) { } } - @Override - public Accessor getAccessor() { - return accessor; - } - - @Override - public Mutator getMutator() { - return mutator; - } - @Override public UnionListReader getReader() { return reader; } - @Override - public boolean allocateNewSafe() { - /* boolean to keep track if all the memory allocation were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; - try { - if (!offsets.allocateNewSafe()) { - return false; - } - success = vector.allocateNewSafe(); - success = success && bits.allocateNewSafe(); - } finally { - if (!success) { - clear(); - } - } - if (success) { - offsets.zeroVector(); - bits.zeroVector(); - } - return success; - } - public AddOrGetResult addOrGetVector(FieldType fieldType) { AddOrGetResult result = super.addOrGetVector(fieldType); reader = new UnionListReader(this); @@ -308,10 +406,12 @@ public AddOrGetResult addOrGetVector(FieldType fieldT @Override public int getBufferSize() { - if (getAccessor().getValueCount() == 0) { + if (getValueCount() == 0) { return 0; } - return offsets.getBufferSize() + bits.getBufferSize() + vector.getBufferSize(); + final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + return offsetBufferSize + validityBufferSize + vector.getBufferSize(); } @Override @@ -329,16 +429,14 @@ public MinorType getMinorType() { @Override public void clear() { - offsets.clear(); - vector.clear(); - bits.clear(); - lastSet = 0; super.clear(); + validityBuffer = releaseBuffer(validityBuffer); + lastSet = 0; } @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), ObjectArrays.concat(bits.getBuffers(false), + final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, ObjectArrays.concat(new ArrowBuf[] {validityBuffer}, vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class); if (clear) { for (ArrowBuf buffer : buffers) { @@ -360,99 +458,97 @@ public UnionVector promoteToUnion() { return vector; } - private int lastSet = 0; + @Override + public Object getObject(int index) { + if (isSet(index) == 0) { + return null; + } + final List vals = new JsonStringArrayList<>(); + final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); + final int end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); + final ValueVector vv = getDataVector(); + for (int i = start; i < end; i++) { + vals.add(vv.getObject(i)); + } - public class Accessor extends BaseRepeatedAccessor { + return vals; + } - @Override - public Object getObject(int index) { - if (isNull(index)) { - return null; - } - final List vals = new JsonStringArrayList<>(); - final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor(); - final int start = offsetsAccessor.get(index); - final int end = offsetsAccessor.get(index + 1); - final ValueVector vv = getDataVector(); - if (vv instanceof NullableIntVector || vv instanceof NullableVarCharVector) { - for (int i = start; i < end; i++) { - vals.add(vv.getObject(i)); - } - } else { - final ValueVector.Accessor valuesAccessor = vv.getAccessor(); - for (int i = start; i < end; i++) { - vals.add(valuesAccessor.getObject(i)); - } - } + @Override + public boolean isNull(int index) { + return (isSet(index) == 0); + } - return vals; - } + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } - @Override - public boolean isNull(int index) { - return bits.getAccessor().get(index) == 0; - } + @Override + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } - @Override - public int getNullCount() { - return bits.getAccessor().getNullCount(); + public void setNotNull(int index) { + if (index >= getValueCapacity()) { + reallocValidityAndOffsetBuffers(); } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + lastSet = index + 1; } - public class Mutator extends BaseRepeatedMutator { - public void setNotNull(int index) { - bits.getMutator().setSafe(index, 1); - lastSet = index + 1; + @Override + public int startNewValue(int index) { + if (index >= getValueCapacity()) { + reallocValidityAndOffsetBuffers(); } - - @Override - public int startNewValue(int index) { - for (int i = lastSet; i <= index; i++) { - offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i)); - } - setNotNull(index); - lastSet = index + 1; - return offsets.getAccessor().get(lastSet); + for (int i = lastSet; i <= index; i++) { + final int currentOffset = offsetBuffer.getInt(i * OFFSET_WIDTH); + offsetBuffer.setInt((i + 1) * OFFSET_WIDTH, currentOffset); } + setNotNull(index); + lastSet = index + 1; + return offsetBuffer.getInt(lastSet * OFFSET_WIDTH); + } - /** - * End the current value - * - * @param index index of the value to end - * @param size number of elements in the list that was written - */ - public void endValue(int index, int size) { - offsets.getMutator().set(index + 1, offsets.getAccessor().get(index + 1) + size); - } + /** + * End the current value + * + * @param index index of the value to end + * @param size number of elements in the list that was written + */ + public void endValue(int index, int size) { + final int currentOffset = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, currentOffset + size); + } - @Override - public void setValueCount(int valueCount) { - // TODO: populate offset end points - if (valueCount == 0) { - offsets.getMutator().setValueCount(0); - } else { - for (int i = lastSet; i < valueCount; i++) { - offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i)); - } - offsets.getMutator().setValueCount(valueCount + 1); + @Override + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + if (valueCount > 0) { + while (valueCount > getValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityAndOffsetBuffers(); } - final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); - if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { - vector.setValueCount(childValueCount); - } else { - vector.getMutator().setValueCount(childValueCount); + for (int i = lastSet; i < valueCount; i++) { + /* fill the holes with offsets */ + final int currentOffset = offsetBuffer.getInt(i * OFFSET_WIDTH); + offsetBuffer.setInt((i + 1) * OFFSET_WIDTH, currentOffset); } - - bits.getMutator().setValueCount(valueCount); - } - - public void setLastSet(int value) { - lastSet = value; } + /* valueCount for the data vector is the current end offset */ + final int childValueCount = (valueCount == 0) ? 0 : + offsetBuffer.getInt(valueCount * OFFSET_WIDTH); + vector.setValueCount(childValueCount); + } - public int getLastSet() { - return lastSet; - } + public void setLastSet(int value) { + lastSet = value; } + public int getLastSet() { + return lastSet; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index 95efa608718..6089a67924f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -56,8 +56,6 @@ public static MapVector empty(String name, BufferAllocator allocator) { } private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this); - private final Accessor accessor = new Accessor(); - private final Mutator mutator = new Mutator(); protected final FieldType fieldType; public int valueCount; @@ -70,6 +68,7 @@ public MapVector(String name, BufferAllocator allocator, CallBack callBack) { public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { super(name, allocator, callBack); this.fieldType = checkNotNull(fieldType); + this.valueCount = 0; } @Override @@ -230,7 +229,7 @@ public void splitAndTransfer(int startIndex, int length) { for (TransferPair p : pairs) { p.splitAndTransfer(startIndex, length); } - to.getMutator().setValueCount(length); + to.setValueCount(length); } } @@ -255,76 +254,54 @@ public int compare(@Nullable ValueVector left, @Nullable ValueVector right) { @Override public Accessor getAccessor() { - return accessor; + throw new UnsupportedOperationException("accessor is not needed for MAP"); } @Override public Mutator getMutator() { - return mutator; + throw new UnsupportedOperationException("mutator is not needed for MAP"); } - public class Accessor extends BaseValueVector.BaseAccessor { - - @Override - public Object getObject(int index) { - Map vv = new JsonStringHashMap<>(); - for (String child : getChildFieldNames()) { - ValueVector v = getChild(child); - if (v instanceof NullableVarCharVector || v instanceof NullableIntVector) { - if (v != null && index < v.getValueCount()) { - Object value = v.getObject(index); - if (value != null) { - vv.put(child, value); - } - } - } else { - if (v != null && index < v.getAccessor().getValueCount()) { - Object value = v.getAccessor().getObject(index); - if (value != null) { - vv.put(child, value); - } - } + @Override + public Object getObject(int index) { + Map vv = new JsonStringHashMap<>(); + for (String child : getChildFieldNames()) { + ValueVector v = getChild(child); + if (v != null && index < v.getValueCount()) { + Object value = v.getObject(index); + if (value != null) { + vv.put(child, value); } } - return vv; } + return vv; + } - public void get(int index, ComplexHolder holder) { - reader.setPosition(index); - holder.reader = reader; - } + @Override + public boolean isNull(int index) { return false; } + @Override + public int getNullCount() { return 0; } - @Override - public int getValueCount() { - return valueCount; - } + public void get(int index, ComplexHolder holder) { + reader.setPosition(index); + holder.reader = reader; } - public ValueVector getVectorById(int id) { - return getChildByOrdinal(id); + @Override + public int getValueCount() { + return valueCount; } - public class Mutator extends BaseValueVector.BaseMutator { - - @Override - public void setValueCount(int valueCount) { - for (final ValueVector v : getChildren()) { - if (v instanceof NullableIntVector || v instanceof NullableVarCharVector) { - v.setValueCount(valueCount); - } else { - v.getMutator().setValueCount(valueCount); - } - } - MapVector.this.valueCount = valueCount; - } - - @Override - public void reset() { - } + public ValueVector getVectorById(int id) { + return getChildByOrdinal(id); +} - @Override - public void generateTestData(int values) { + @Override + public void setValueCount(int valueCount) { + for (final ValueVector v : getChildren()) { + v.setValueCount(valueCount); } + MapVector.this.valueCount = valueCount; } @Override @@ -372,11 +349,4 @@ public void initializeChildrenFromFields(List children) { public List getChildrenFromFields() { return getChildren(); } - - public int getValueCount() { return 0; } - - public void setValueCount(int valueCount) { } - - public Object getObject(int index) { return null; } - } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java index 1c9d5aa2659..a1478c5a721 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkNotNull; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -27,13 +28,9 @@ import com.google.common.collect.ObjectArrays; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.BaseDataValueVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.BufferBacked; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.NullableVectorDefinitionSetter; -import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl; import org.apache.arrow.vector.complex.impl.NullableMapWriter; import org.apache.arrow.vector.holders.ComplexHolder; @@ -44,6 +41,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; public class NullableMapVector extends MapVector implements FieldVector { @@ -56,12 +54,8 @@ public static NullableMapVector empty(String name, BufferAllocator allocator) { private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this); private final NullableMapWriter writer = new NullableMapWriter(this); - protected final BitVector bits; - - private final List innerVectors; - - private final Accessor accessor; - private final Mutator mutator; + private ArrowBuf validityBuffer; + private int validityAllocationSizeInBytes; // deprecated, use FieldType or static constructor instead @Deprecated @@ -77,10 +71,8 @@ public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncod public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { super(name, checkNotNull(allocator), fieldType, callBack); - this.bits = new BitVector("$bits$", allocator); - this.innerVectors = Collections.unmodifiableList(Arrays.asList(bits)); - this.accessor = new Accessor(); - this.mutator = new Mutator(); + this.validityBuffer = allocator.getEmpty(); + this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); } @Override @@ -92,18 +84,33 @@ public Field getField() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); - this.valueCount = fieldNode.getLength(); + if (ownBuffers.size() != 1) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + valueCount = fieldNode.getLength(); + validityAllocationSizeInBytes = validityBuffer.capacity(); } @Override public List getFieldBuffers() { - return BaseDataValueVector.unload(getFieldInnerVectors()); + List result = new ArrayList<>(1); + + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + result.add(validityBuffer); + + return result; } @Override + @Deprecated public List getFieldInnerVectors() { - return innerVectors; + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); } @Override @@ -146,49 +153,129 @@ protected NullableMapTransferPair(NullableMapVector from, NullableMapVector to, @Override public void transfer() { - bits.transferTo(target.bits); + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; super.transfer(); } @Override public void copyValueSafe(int fromIndex, int toIndex) { - target.bits.copyFromSafe(fromIndex, toIndex, bits); + while (toIndex >= target.getValidityBufferValueCapacity()) { + target.reallocValidityBuffer(); + } + BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex)); super.copyValueSafe(fromIndex, toIndex); } @Override public void splitAndTransfer(int startIndex, int length) { - bits.splitAndTransferTo(startIndex, length, target.bits); + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); super.splitAndTransfer(startIndex, length); } } + /* + * transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, NullableMapVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + } + + private int getValidityBufferValueCapacity() { + return (int)(validityBuffer.capacity() * 8L); + } + @Override public int getValueCapacity() { - return Math.min(bits.getValueCapacity(), super.getValueCapacity()); + return Math.min(getValidityBufferValueCapacity(), + super.getValueCapacity()); } @Override public ArrowBuf[] getBuffers(boolean clear) { - return ObjectArrays.concat(bits.getBuffers(clear), super.getBuffers(clear), ArrowBuf.class); + if (clear) { + validityBuffer.retain(1); + } + return ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(clear), ArrowBuf.class); } @Override public void close() { - bits.close(); + clearValidityBuffer(); super.close(); } @Override public void clear() { - bits.clear(); + clearValidityBuffer(); super.clear(); } + private void clearValidityBuffer() { + validityBuffer.release(); + validityBuffer = allocator.getEmpty(); + } @Override public int getBufferSize() { - return super.getBufferSize() + bits.getBufferSize(); + if (valueCount == 0) { return 0; } + return super.getBufferSize() + + BitVectorHelper.getValidityBufferSize(valueCount); } @Override @@ -197,12 +284,12 @@ public int getBufferSizeFor(final int valueCount) { return 0; } return super.getBufferSizeFor(valueCount) - + bits.getBufferSizeFor(valueCount); + + BitVectorHelper.getValidityBufferSize(valueCount); } @Override public void setInitialCapacity(int numRecords) { - bits.setInitialCapacity(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); super.setInitialCapacity(numRecords); } @@ -215,25 +302,59 @@ public boolean allocateNewSafe() { */ boolean success = false; try { - success = super.allocateNewSafe() && bits.allocateNewSafe(); + clearValidityBuffer(); + allocateValidityBuffer(validityAllocationSizeInBytes); + success = super.allocateNewSafe(); } finally { if (!success) { clear(); + return false; } } - bits.zeroVector(); - return success; + return true; + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int)size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + validityBuffer.setZero(0, validityBuffer.capacity()); } @Override public void reAlloc() { - bits.reAlloc(); + /* reallocate the validity buffer */ + reallocValidityBuffer(); super.reAlloc(); } + private void reallocValidityBuffer() { + final int currentBufferCapacity = validityBuffer.capacity(); + long baseSize = validityAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setZero(0, newBuf.capacity()); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + validityBuffer.release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int)newAllocationSize; + } + @Override public long getValidityBufferAddress() { - return bits.getBuffer().memoryAddress(); + return validityBuffer.memoryAddress(); } @Override @@ -248,7 +369,7 @@ public long getOffsetBufferAddress() { @Override public ArrowBuf getValidityBuffer() { - return bits.getDataBuffer(); + return validityBuffer; } @Override @@ -261,82 +382,76 @@ public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } - public final class Accessor extends MapVector.Accessor { - final BitVector.Accessor bAccessor = bits.getAccessor(); - - @Override - public Object getObject(int index) { - if (isNull(index)) { - return null; - } else { - return super.getObject(index); - } - } - - @Override - public void get(int index, ComplexHolder holder) { - holder.isSet = isSet(index); - super.get(index, holder); - } - - @Override - public int getNullCount() { - return bits.getAccessor().getNullCount(); - } - - @Override - public boolean isNull(int index) { - return isSet(index) == 0; - } - - public int isSet(int index) { - return bAccessor.get(index); + @Override + public Object getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return super.getObject(index); } - } - public final class Mutator extends MapVector.Mutator implements NullableVectorDefinitionSetter { + @Override + public void get(int index, ComplexHolder holder) { + holder.isSet = isSet(index); + super.get(index, holder); + } - private Mutator() { - } + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } - @Override - public void setIndexDefined(int index) { - bits.getMutator().setSafe(index, 1); - } + public boolean isNull(int index) { + return isSet(index) == 0; + } - public void setNull(int index) { - bits.getMutator().setSafe(index, 0); - } + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } - @Override - public void setValueCount(int valueCount) { - assert valueCount >= 0; - super.setValueCount(valueCount); - bits.getMutator().setValueCount(valueCount); + public void setIndexDefined(int index) { + while (index >= getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } - @Override - public void generateTestData(int valueCount) { - super.generateTestData(valueCount); - bits.getMutator().generateTestDataAlt(valueCount); + public void setNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); } + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } - @Override - public void reset() { - bits.getMutator().setValueCount(0); + @Override + public void setValueCount(int valueCount) { + assert valueCount >= 0; + while (valueCount > getValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); } + super.setValueCount(valueCount); + this.valueCount = valueCount; + } + public void reset() { + valueCount = 0; } @Override + @Deprecated public Accessor getAccessor() { - return accessor; + throw new UnsupportedOperationException("Accessor is not supported for reading from Nullable MAP"); } @Override + @Deprecated public Mutator getMutator() { - return mutator; + throw new UnsupportedOperationException("Mutator is not supported for writing to Nullable MAP"); } - } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java index 614c266acf1..06b0f4d4327 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java @@ -52,6 +52,6 @@ public void copyAsField(String name, MapWriter writer) { @Override public boolean isSet() { - return !nullableMapVector.getAccessor().isNull(idx()); + return !nullableMapVector.isNull(idx()); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java index 9722196ed7c..5bd439cac42 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java @@ -162,7 +162,7 @@ private FieldWriter promoteToUnion() { writer = new UnionWriter(unionVector, nullableMapWriterFactory); writer.setPosition(idx()); for (int i = 0; i <= idx(); i++) { - unionVector.getMutator().setType(i, vector.getMinorType()); + unionVector.setType(i, vector.getMinorType()); } vector = null; state = State.UNION; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java index 3ebd0cd7dd9..c77ca4e8f23 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java @@ -78,7 +78,7 @@ public void setPosition(int index) { @Override public Object readObject() { - return vector.getAccessor().getObject(idx()); + return vector.getObject(idx()); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java index f3e9b8773f2..4ad2f6f5fad 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java @@ -46,7 +46,7 @@ public UnionFixedSizeListReader(FixedSizeListVector vector) { @Override public boolean isSet() { - return !vector.getAccessor().isNull(idx()); + return !vector.isNull(idx()); } @Override @@ -56,7 +56,7 @@ public FieldReader reader() { @Override public Object readObject() { - return vector.getAccessor().getObject(idx()); + return vector.getObject(idx()); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java index b98c36d2bf7..6243a2833c0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java @@ -19,6 +19,7 @@ package org.apache.arrow.vector.complex.impl; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.ListVector; @@ -33,12 +34,11 @@ public class UnionListReader extends AbstractFieldReader { private ListVector vector; private ValueVector data; - private UInt4Vector offsets; + private static final int OFFSET_WIDTH = 4; public UnionListReader(ListVector vector) { this.vector = vector; this.data = vector.getDataVector(); - this.offsets = vector.getOffsetVector(); } @Override @@ -48,7 +48,7 @@ public Field getField() { @Override public boolean isSet() { - return !vector.getAccessor().isNull(idx()); + return !vector.isNull(idx()); } private int currentOffset; @@ -57,8 +57,8 @@ public boolean isSet() { @Override public void setPosition(int index) { super.setPosition(index); - currentOffset = offsets.getAccessor().get(index) - 1; - maxOffset = offsets.getAccessor().get(index + 1); + currentOffset = vector.getOffsetBuffer().getInt(index * OFFSET_WIDTH) - 1; + maxOffset = vector.getOffsetBuffer().getInt((index + 1) * OFFSET_WIDTH); } @Override @@ -68,7 +68,7 @@ public FieldReader reader() { @Override public Object readObject() { - return vector.getAccessor().getObject(idx()); + return vector.getObject(idx()); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java index 3b7dc4a56e1..762a442c983 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java @@ -47,11 +47,10 @@ public class DictionaryEncoder { public static ValueVector encode(ValueVector vector, Dictionary dictionary) { validateType(vector.getMinorType()); // load dictionary values into a hashmap for lookup - ValueVector.Accessor dictionaryAccessor = dictionary.getVector().getAccessor(); - Map lookUps = new HashMap<>(dictionaryAccessor.getValueCount()); - for (int i = 0; i < dictionaryAccessor.getValueCount(); i++) { + Map lookUps = new HashMap<>(dictionary.getVector().getValueCount()); + for (int i = 0; i < dictionary.getVector().getValueCount(); i++) { // for primitive array types we need a wrapper that implements equals and hashcode appropriately - lookUps.put(dictionaryAccessor.getObject(i), i); + lookUps.put(dictionary.getVector().getObject(i), i); } Field valueField = vector.getField(); @@ -61,14 +60,13 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { // vector to hold our indices (dictionary encoded values) FieldVector indices = indexField.createVector(vector.getAllocator()); - ValueVector.Mutator mutator = indices.getMutator(); // use reflection to pull out the set method // TODO implement a common interface for int vectors Method setter = null; for (Class c : ImmutableList.of(int.class, long.class)) { try { - setter = mutator.getClass().getMethod("setSafe", int.class, c); + setter = indices.getClass().getMethod("setSafe", int.class, c); break; } catch (NoSuchMethodException e) { // ignore @@ -78,21 +76,20 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { throw new IllegalArgumentException("Dictionary encoding does not have a valid int type:" + indices.getClass()); } - ValueVector.Accessor accessor = vector.getAccessor(); - int count = accessor.getValueCount(); + int count = vector.getValueCount(); indices.allocateNew(); try { for (int i = 0; i < count; i++) { - Object value = accessor.getObject(i); + Object value = vector.getObject(i); if (value != null) { // if it's null leave it null // note: this may fail if value was not included in the dictionary Object encoded = lookUps.get(value); if (encoded == null) { throw new IllegalArgumentException("Dictionary encoding not defined for value:" + value); } - setter.invoke(mutator, i, encoded); + setter.invoke(indices, i, encoded); } } } catch (IllegalAccessException e) { @@ -101,7 +98,7 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { throw new RuntimeException("InvocationTargetException invoking vector mutator set():", e.getCause()); } - mutator.setValueCount(count); + indices.setValueCount(count); return indices; } @@ -114,15 +111,14 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { * @return vector with values restored from dictionary */ public static ValueVector decode(ValueVector indices, Dictionary dictionary) { - ValueVector.Accessor accessor = indices.getAccessor(); - int count = accessor.getValueCount(); + int count = indices.getValueCount(); ValueVector dictionaryVector = dictionary.getVector(); - int dictionaryCount = dictionaryVector.getAccessor().getValueCount(); + int dictionaryCount = dictionaryVector.getValueCount(); // copy the dictionary values into the decoded vector TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator()); transfer.getTo().allocateNewSafe(); for (int i = 0; i < count; i++) { - Object index = accessor.getObject(i); + Object index = indices.getObject(i); if (index != null) { int indexAsInt = ((Number) index).intValue(); if (indexAsInt > dictionaryCount) { @@ -133,7 +129,7 @@ public static ValueVector decode(ValueVector indices, Dictionary dictionary) { } // TODO do we need to worry about the field? ValueVector decoded = transfer.getTo(); - decoded.getMutator().setValueCount(count); + decoded.setValueCount(count); return decoded; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java index b35aba5426e..7dc10b5e629 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java @@ -82,7 +82,7 @@ protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, Writab for (long id : dictionaryIdsUsed) { Dictionary dictionary = provider.lookup(id); FieldVector vector = dictionary.getVector(); - int count = vector.getAccessor().getValueCount(); + int count = vector.getValueCount(); VectorSchemaRoot dictRoot = new VectorSchemaRoot(ImmutableList.of(vector.getField()), ImmutableList.of(vector), count); VectorUnloader unloader = new VectorUnloader(dictRoot); ArrowRecordBatch batch = unloader.getRecordBatch(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index 27781f5777d..c6a82510e60 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -23,55 +23,22 @@ import static com.fasterxml.jackson.core.JsonToken.START_ARRAY; import static com.fasterxml.jackson.core.JsonToken.START_OBJECT; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.arrow.vector.schema.ArrowVectorType.OFFSET; +import static org.apache.arrow.vector.schema.ArrowVectorType.*; import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import com.google.common.collect.ImmutableList; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.BigIntVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.BufferBacked; -import org.apache.arrow.vector.DateDayVector; -import org.apache.arrow.vector.DateMilliVector; -import org.apache.arrow.vector.DecimalVector; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.Float4Vector; -import org.apache.arrow.vector.Float8Vector; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.NullableVarBinaryVector; -import org.apache.arrow.vector.NullableVarCharVector; -import org.apache.arrow.vector.SmallIntVector; -import org.apache.arrow.vector.TimeMicroVector; -import org.apache.arrow.vector.TimeMilliVector; -import org.apache.arrow.vector.TimeNanoVector; -import org.apache.arrow.vector.TimeSecVector; -import org.apache.arrow.vector.TimeStampMicroTZVector; -import org.apache.arrow.vector.TimeStampMicroVector; -import org.apache.arrow.vector.TimeStampMilliTZVector; -import org.apache.arrow.vector.TimeStampMilliVector; -import org.apache.arrow.vector.TimeStampNanoTZVector; -import org.apache.arrow.vector.TimeStampNanoVector; -import org.apache.arrow.vector.TimeStampSecTZVector; -import org.apache.arrow.vector.TimeStampSecVector; -import org.apache.arrow.vector.TinyIntVector; -import org.apache.arrow.vector.UInt1Vector; -import org.apache.arrow.vector.UInt2Vector; -import org.apache.arrow.vector.UInt4Vector; -import org.apache.arrow.vector.UInt8Vector; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.VarBinaryVector; -import org.apache.arrow.vector.VarCharVector; -import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.schema.ArrowVectorType; +import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.DecimalUtility; @@ -151,7 +118,7 @@ private void readDictionaryBatches() throws JsonParseException, IOException { FieldVector vector = dict.getVector(); List fields = ImmutableList.of(vector.getField()); List vectors = ImmutableList.of(vector); - VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getAccessor().getValueCount()); + VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getValueCount()); read(root); readToken(END_OBJECT); @@ -175,7 +142,7 @@ public boolean read(VectorSchemaRoot root) throws IOException { { for (Field field : root.getSchema().getFields()) { FieldVector vector = root.getVector(field.getName()); - readVector(field, vector); + readFromJsonIntoVector(field, vector); } } readToken(END_ARRAY); @@ -202,7 +169,7 @@ public VectorSchemaRoot read() throws IOException { { for (Field field : schema.getFields()) { FieldVector vector = recordBatch.getVector(field.getName()); - readVector(field, vector); + readFromJsonIntoVector(field, vector); } } readToken(END_ARRAY); @@ -216,16 +183,24 @@ public VectorSchemaRoot read() throws IOException { } } - /** - * TODO: A better way of implementing this function is to use `loadFieldBuffers` methods in - * FieldVector to set the inner-vector data as done in `ArrowFileReader`. - */ - private void readVector(Field field, FieldVector vector) throws JsonParseException, IOException { + private void readFromJsonIntoVector(Field field, FieldVector vector) throws JsonParseException, IOException { List vectorTypes = field.getTypeLayout().getVectorTypes(); - List fieldInnerVectors = vector.getFieldInnerVectors(); - if (vectorTypes.size() != fieldInnerVectors.size()) { - throw new IllegalArgumentException("vector types and inner vectors are not the same size: " + vectorTypes.size() + " != " + fieldInnerVectors.size()); - } + ArrowBuf[] vectorBuffers = new ArrowBuf[vectorTypes.size()]; + /* + * The order of inner buffers is : + * Fixed width vector: + * -- validity buffer + * -- data buffer + * Variable width vector: + * -- validity buffer + * -- offset buffer + * -- data buffer + * + * This is similar to what getFieldInnerVectors() used to give but now that we don't have + * inner vectors anymore, we will work directly at the buffer level -- populate buffers + * locally as we read from Json parser and do loadFieldBuffers on the vector followed by + * releasing the local buffers. + */ readToken(START_OBJECT); { // If currently reading dictionaries, field name is not important so don't check @@ -234,39 +209,41 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti throw new IllegalArgumentException("Expected field " + field.getName() + " but got " + name); } - // Initialize the vector with required capacity - int count = readNextField("count", Integer.class); - vector.setInitialCapacity(count); - vector.allocateNew(); + /* Initialize the vector with required capacity but don't allocate since we would + * be doing loadFieldBuffers. + */ + int valueCount = readNextField("count", Integer.class); + vector.setInitialCapacity(valueCount); - // Read inner vectors for (int v = 0; v < vectorTypes.size(); v++) { ArrowVectorType vectorType = vectorTypes.get(v); - ValueVector valueVector = (ValueVector) fieldInnerVectors.get(v); nextFieldIs(vectorType.getName()); readToken(START_ARRAY); - int innerVectorCount = vectorType.equals(OFFSET) ? count + 1 : count; - for (int i = 0; i < innerVectorCount; i++) { + int innerBufferValueCount = valueCount; + if (vectorType.equals(OFFSET)) { + /* offset buffer has 1 additional value capacity */ + innerBufferValueCount = valueCount + 1; + } + for (int i = 0; i < innerBufferValueCount; i++) { + /* write data to the buffer */ parser.nextToken(); - setValueFromParser(valueVector, i); + /* for variable width vectors, value count doesn't help pre-determining the capacity of + * the underlying data buffer. So we need to pass down the offset buffer (which was already + * populated in the previous iteration of this loop). + */ + if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR + || vector.getMinorType() == Types.MinorType.VARBINARY)) { + vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v], + vectorBuffers[v-1], i, innerBufferValueCount); + } else { + vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v], + null, i, innerBufferValueCount); + } } readToken(END_ARRAY); } - // Set lastSet before valueCount to prevent setValueCount from filling empty values - switch (vector.getMinorType()) { - case LIST: - // ListVector starts lastSet from index 0, so lastSet value is always last index written + 1 - ((ListVector) vector).getMutator().setLastSet(count); - break; - case VARBINARY: - ((NullableVarBinaryVector) vector).getMutator().setLastSet(count - 1); - break; - case VARCHAR: - ((NullableVarCharVector) vector).setLastSet(count - 1); - break; - } - vector.getMutator().setValueCount(count); + vector.loadFieldBuffers(new ArrowFieldNode(valueCount, 0), Arrays.asList(vectorBuffers)); // read child vectors, if any List fields = field.getChildren(); @@ -280,12 +257,16 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti for (int i = 0; i < fields.size(); i++) { Field childField = fields.get(i); FieldVector childVector = vectorChildren.get(i); - readVector(childField, childVector); + readFromJsonIntoVector(childField, childVector); } readToken(END_ARRAY); } } readToken(END_OBJECT); + + for (ArrowBuf buffer: vectorBuffers) { + buffer.release(); + } } private byte[] decodeHexSafe(String hexString) throws IOException { @@ -296,98 +277,123 @@ private byte[] decodeHexSafe(String hexString) throws IOException { } } - private void setValueFromParser(ValueVector valueVector, int i) throws IOException { - switch (valueVector.getMinorType()) { - case BIT: - ((BitVector) valueVector).getMutator().set(i, parser.readValueAs(Boolean.class) ? 1 : 0); - break; - case TINYINT: - ((TinyIntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case SMALLINT: - ((SmallIntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case INT: - ((IntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case BIGINT: - ((BigIntVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case UINT1: - ((UInt1Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case UINT2: - ((UInt2Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case UINT4: - ((UInt4Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case UINT8: - ((UInt8Vector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case FLOAT4: - ((Float4Vector) valueVector).getMutator().set(i, parser.readValueAs(Float.class)); - break; - case FLOAT8: - ((Float8Vector) valueVector).getMutator().set(i, parser.readValueAs(Double.class)); - break; - case DECIMAL: { - DecimalVector decimalVector = ((DecimalVector) valueVector); - byte[] value = decodeHexSafe(parser.readValueAs(String.class)); - DecimalUtility.writeByteArrayToArrowBuf(value, decimalVector.getBuffer(), i); - } - break; - case VARBINARY: - ((VarBinaryVector) valueVector).getMutator().setSafe(i, decodeHexSafe(parser.readValueAs(String.class))); - break; - case VARCHAR: - ((VarCharVector) valueVector).getMutator().setSafe(i, parser.readValueAs(String.class).getBytes(UTF_8)); - break; - case DATEDAY: - ((DateDayVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case DATEMILLI: - ((DateMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESEC: - ((TimeSecVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case TIMEMILLI: - ((TimeMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case TIMEMICRO: - ((TimeMicroVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMENANO: - ((TimeNanoVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPSEC: - ((TimeStampSecVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMILLI: - ((TimeStampMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMICRO: - ((TimeStampMicroVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPNANO: - ((TimeStampNanoVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPSECTZ: - ((TimeStampSecTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMILLITZ: - ((TimeStampMilliTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMICROTZ: - ((TimeStampMicroTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPNANOTZ: - ((TimeStampNanoTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - default: - throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType()); + private ArrowBuf setValueFromParser(ArrowVectorType bufferType, FieldVector vector, + ArrowBuf buffer, ArrowBuf offsetBuffer, int index, int valueCount) throws IOException { + if (bufferType.equals(TYPE)) { + buffer = NullableTinyIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Byte.class)); + } else if (bufferType.equals(OFFSET)) { + buffer = BaseNullableVariableWidthVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + } else if (bufferType.equals(VALIDITY)) { + buffer = BitVectorHelper.setValidityBit(buffer, allocator, + valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0); + } else if (bufferType.equals(DATA)) { + switch (vector.getMinorType()) { + case BIT: + buffer = BitVectorHelper.setValidityBit(buffer, allocator, + valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0); + break; + case TINYINT: + buffer = NullableTinyIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Byte.class)); + break; + case SMALLINT: + buffer = NullableSmallIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Short.class)); + break; + case INT: + buffer = NullableIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + break; + case BIGINT: + buffer = NullableBigIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case FLOAT4: + buffer = NullableFloat4Vector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Float.class)); + break; + case FLOAT8: + buffer = NullableFloat8Vector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Double.class)); + break; + case DECIMAL: + buffer = NullableDecimalVector.set(buffer, allocator, + valueCount, index, decodeHexSafe(parser.readValueAs(String.class))); + break; + case VARBINARY: + assert (offsetBuffer != null); + buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index, + decodeHexSafe(parser.readValueAs(String.class)), valueCount); + break; + case VARCHAR: + assert (offsetBuffer != null); + buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index, + parser.readValueAs(String.class).getBytes(UTF_8), valueCount); + break; + case DATEDAY: + buffer = NullableDateDayVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + break; + case DATEMILLI: + buffer = NullableDateMilliVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESEC: + buffer = NullableTimeSecVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + break; + case TIMEMILLI: + buffer = NullableTimeMilliVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + break; + case TIMEMICRO: + buffer = NullableTimeMicroVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMENANO: + buffer = NullableTimeNanoVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPSEC: + buffer = NullableTimeStampSecVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMILLI: + buffer = NullableTimeStampMilliVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMICRO: + buffer = NullableTimeStampMicroVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPNANO: + buffer = NullableTimeStampNanoVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPSECTZ: + buffer = NullableTimeStampSecTZVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMILLITZ: + buffer = NullableTimeStampMilliTZVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMICROTZ: + buffer = NullableTimeStampMicroTZVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPNANOTZ: + buffer = NullableTimeStampNanoTZVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + default: + throw new UnsupportedOperationException("minor type: " + vector.getMinorType()); + } } + + return buffer; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java index 04e44379e5d..d3aa1770e7e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java @@ -18,6 +18,9 @@ package org.apache.arrow.vector.file.json; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.arrow.vector.schema.ArrowVectorType.*; + import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -27,27 +30,12 @@ import com.google.common.collect.ImmutableList; import io.netty.buffer.ArrowBuf; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.BufferBacked; -import org.apache.arrow.vector.DateDayVector; -import org.apache.arrow.vector.DateMilliVector; -import org.apache.arrow.vector.DecimalVector; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.TimeMicroVector; -import org.apache.arrow.vector.TimeMilliVector; -import org.apache.arrow.vector.TimeNanoVector; -import org.apache.arrow.vector.TimeSecVector; -import org.apache.arrow.vector.TimeStampMicroVector; -import org.apache.arrow.vector.TimeStampMilliVector; -import org.apache.arrow.vector.TimeStampNanoVector; -import org.apache.arrow.vector.TimeStampSecVector; -import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.ValueVector.Accessor; -import org.apache.arrow.vector.VarBinaryVector; -import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.schema.ArrowVectorType; +import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; @@ -133,7 +121,7 @@ private void writeDictionaryBatches(JsonGenerator generator, Set dictionar FieldVector vector = dictionary.getVector(); List fields = ImmutableList.of(vector.getField()); List vectors = ImmutableList.of(vector); - VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getAccessor().getValueCount()); + VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getValueCount()); writeBatch(root); generator.writeEndObject(); @@ -155,31 +143,36 @@ private void writeBatch(VectorSchemaRoot recordBatch) throws IOException { generator.writeArrayFieldStart("columns"); for (Field field : recordBatch.getSchema().getFields()) { FieldVector vector = recordBatch.getVector(field.getName()); - writeVector(field, vector); + writeFromVectorIntoJson(field, vector); } generator.writeEndArray(); } generator.writeEndObject(); } - private void writeVector(Field field, FieldVector vector) throws IOException { + private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOException { List vectorTypes = field.getTypeLayout().getVectorTypes(); - List fieldInnerVectors = vector.getFieldInnerVectors(); - if (vectorTypes.size() != fieldInnerVectors.size()) { - throw new IllegalArgumentException("vector types and inner vectors are not the same size: " + vectorTypes.size() + " != " + fieldInnerVectors.size()); + List vectorBuffers = vector.getFieldBuffers(); + if (vectorTypes.size() != vectorBuffers.size()) { + throw new IllegalArgumentException("vector types and inner vector buffers are not the same size: " + vectorTypes.size() + " != " + vectorBuffers.size()); } generator.writeStartObject(); { generator.writeObjectField("name", field.getName()); - int valueCount = vector.getAccessor().getValueCount(); + int valueCount = vector.getValueCount(); generator.writeObjectField("count", valueCount); for (int v = 0; v < vectorTypes.size(); v++) { ArrowVectorType vectorType = vectorTypes.get(v); - BufferBacked innerVector = fieldInnerVectors.get(v); + ArrowBuf vectorBuffer = vectorBuffers.get(v); generator.writeArrayFieldStart(vectorType.getName()); - ValueVector valueVector = (ValueVector) innerVector; - for (int i = 0; i < valueVector.getAccessor().getValueCount(); i++) { - writeValueToGenerator(valueVector, i); + final int bufferValueCount = (vectorType.equals(OFFSET)) ? valueCount + 1 : valueCount; + for (int i = 0; i < bufferValueCount; i++) { + if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR || + vector.getMinorType() == Types.MinorType.VARBINARY)) { + writeValueToGenerator(vectorType, vectorBuffer, vectorBuffers.get(v-1), vector, i); + } else { + writeValueToGenerator(vectorType, vectorBuffer, null, vector, i); + } } generator.writeEndArray(); } @@ -193,7 +186,7 @@ private void writeVector(Field field, FieldVector vector) throws IOException { for (int i = 0; i < fields.size(); i++) { Field childField = fields.get(i); FieldVector childVector = children.get(i); - writeVector(childField, childVector); + writeFromVectorIntoJson(childField, childVector); } generator.writeEndArray(); } @@ -201,62 +194,99 @@ private void writeVector(Field field, FieldVector vector) throws IOException { generator.writeEndObject(); } - private void writeValueToGenerator(ValueVector valueVector, int i) throws IOException { - switch (valueVector.getMinorType()) { - case DATEDAY: - generator.writeNumber(((DateDayVector) valueVector).getAccessor().get(i)); - break; - case DATEMILLI: - generator.writeNumber(((DateMilliVector) valueVector).getAccessor().get(i)); - break; - case TIMESEC: - generator.writeNumber(((TimeSecVector) valueVector).getAccessor().get(i)); - break; - case TIMEMILLI: - generator.writeNumber(((TimeMilliVector) valueVector).getAccessor().get(i)); - break; - case TIMEMICRO: - generator.writeNumber(((TimeMicroVector) valueVector).getAccessor().get(i)); - break; - case TIMENANO: - generator.writeNumber(((TimeNanoVector) valueVector).getAccessor().get(i)); - break; - case TIMESTAMPSEC: - generator.writeNumber(((TimeStampSecVector) valueVector).getAccessor().get(i)); - break; - case TIMESTAMPMILLI: - generator.writeNumber(((TimeStampMilliVector) valueVector).getAccessor().get(i)); - break; - case TIMESTAMPMICRO: - generator.writeNumber(((TimeStampMicroVector) valueVector).getAccessor().get(i)); - break; - case TIMESTAMPNANO: - generator.writeNumber(((TimeStampNanoVector) valueVector).getAccessor().get(i)); - break; - case BIT: - generator.writeNumber(((BitVector) valueVector).getAccessor().get(i)); - break; - case VARBINARY: { - String hexString = Hex.encodeHexString(((VarBinaryVector) valueVector).getAccessor().get(i)); - generator.writeString(hexString); + private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer, + ArrowBuf offsetBuffer, FieldVector vector, int index) throws IOException { + if (bufferType.equals(TYPE)) { + generator.writeNumber(buffer.getByte(index * NullableTinyIntVector.TYPE_WIDTH)); + } else if (bufferType.equals(OFFSET)) { + generator.writeNumber(buffer.getInt(index * BaseNullableVariableWidthVector.OFFSET_WIDTH)); + } else if(bufferType.equals(VALIDITY)) { + generator.writeNumber(vector.isNull(index) ? 0 : 1); + } else if (bufferType.equals(DATA)) { + switch (vector.getMinorType()) { + case TINYINT: + generator.writeNumber(NullableTinyIntVector.get(buffer, index)); + break; + case SMALLINT: + generator.writeNumber(NullableSmallIntVector.get(buffer, index)); + break; + case INT: + generator.writeNumber(NullableIntVector.get(buffer, index)); + break; + case BIGINT: + generator.writeNumber(NullableBigIntVector.get(buffer, index)); + break; + case FLOAT4: + generator.writeNumber(NullableFloat4Vector.get(buffer, index)); + break; + case FLOAT8: + generator.writeNumber(NullableFloat8Vector.get(buffer, index)); + break; + case DATEDAY: + generator.writeNumber(NullableDateDayVector.get(buffer, index)); + break; + case DATEMILLI: + generator.writeNumber(NullableDateMilliVector.get(buffer, index)); + break; + case TIMESEC: + generator.writeNumber(NullableTimeSecVector.get(buffer, index)); + break; + case TIMEMILLI: + generator.writeNumber(NullableTimeMilliVector.get(buffer, index)); + break; + case TIMEMICRO: + generator.writeNumber(NullableTimeMicroVector.get(buffer, index)); + break; + case TIMENANO: + generator.writeNumber(NullableTimeNanoVector.get(buffer, index)); + break; + case TIMESTAMPSEC: + generator.writeNumber(NullableTimeStampSecVector.get(buffer, index)); + break; + case TIMESTAMPMILLI: + generator.writeNumber(NullableTimeStampMilliVector.get(buffer, index)); + break; + case TIMESTAMPMICRO: + generator.writeNumber(NullableTimeStampMicroVector.get(buffer, index)); + break; + case TIMESTAMPNANO: + generator.writeNumber(NullableTimeStampNanoVector.get(buffer, index)); + break; + case TIMESTAMPSECTZ: + generator.writeNumber(NullableTimeStampSecTZVector.get(buffer, index)); + break; + case TIMESTAMPMILLITZ: + generator.writeNumber(NullableTimeStampMilliTZVector.get(buffer, index)); + break; + case TIMESTAMPMICROTZ: + generator.writeNumber(NullableTimeStampMicroTZVector.get(buffer, index)); + break; + case TIMESTAMPNANOTZ: + generator.writeNumber(NullableTimeStampNanoTZVector.get(buffer, index)); + break; + case BIT: + generator.writeNumber(BitVectorHelper.get(buffer, index)); + break; + case VARBINARY: { + assert offsetBuffer != null; + String hexString = Hex.encodeHexString(BaseNullableVariableWidthVector.get(buffer, offsetBuffer, index)); + generator.writeObject(hexString); + break; } - break; - case DECIMAL: { - ArrowBuf bytebuf = valueVector.getDataBuffer(); - String hexString = Hex.encodeHexString(DecimalUtility.getByteArrayFromArrowBuf(bytebuf, i)); - generator.writeString(hexString); + case VARCHAR: { + assert offsetBuffer != null; + byte[] b = (BaseNullableVariableWidthVector.get(buffer, offsetBuffer, index)); + generator.writeString(new String(b, "UTF-8")); + break; } - break; - default: - // TODO: each type - Accessor accessor = valueVector.getAccessor(); - Object value = accessor.getObject(i); - if (value instanceof Number || value instanceof Boolean) { - generator.writeObject(value); - } else { - generator.writeObject(value.toString()); + case DECIMAL: { + String hexString = Hex.encodeHexString(DecimalUtility.getByteArrayFromArrowBuf(buffer, index)); + generator.writeString(hexString); + break; } - break; + default: + throw new UnsupportedOperationException("minor type: " + vector.getMinorType()); + } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java index 5851bd5fa5d..c27e5e5c85c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java @@ -114,13 +114,13 @@ public static void compareFieldVectors(FieldVector vector1, FieldVector vector2) if (!field1.equals(vector2.getField())) { throw new IllegalArgumentException("Different Fields:\n" + field1 + "\n!=\n" + vector2.getField()); } - int valueCount = vector1.getAccessor().getValueCount(); - if (valueCount != vector2.getAccessor().getValueCount()) { - throw new IllegalArgumentException("Different value count for field " + field1 + " : " + valueCount + " != " + vector2.getAccessor().getValueCount()); + int valueCount = vector1.getValueCount(); + if (valueCount != vector2.getValueCount()) { + throw new IllegalArgumentException("Different value count for field " + field1 + " : " + valueCount + " != " + vector2.getValueCount()); } for (int j = 0; j < valueCount; j++) { - Object obj1 = vector1.getAccessor().getObject(j); - Object obj2 = vector2.getAccessor().getObject(j); + Object obj1 = vector1.getObject(j); + Object obj2 = vector2.getObject(j); if (!equals(field1.getType(), obj1, obj2)) { throw new IllegalArgumentException( "Different values in column:\n" + field1 + " at index " + j + ": " + obj1 + " != " + obj2); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java index 17fcf05fcd9..ada34149787 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java @@ -340,59 +340,56 @@ public void testReallocAfterVectorTransfer2() { int valueCapacity = vector.getValueCapacity(); assertEquals(4096, valueCapacity); - final NullableBitVector.Mutator mutator = vector.getMutator(); - final NullableBitVector.Accessor accessor = vector.getAccessor(); - for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - mutator.set(i, 1); + vector.set(i, 1); } } for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i)); + assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i)); } else { - assertTrue("unexpected set bit at index: " + i, accessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); } } /* trigger first realloc */ - mutator.setSafe(valueCapacity, 1, 1); + vector.setSafe(valueCapacity, 1, 1); assertEquals(valueCapacity * 2, vector.getValueCapacity()); for (int i = valueCapacity; i < valueCapacity*2; i++) { if ((i & 1) == 1) { - mutator.set(i, 1); + vector.set(i, 1); } } for (int i = 0; i < valueCapacity*2; i++) { if (((i & 1) == 1) || (i == valueCapacity)) { - assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i)); + assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i)); } else { - assertTrue("unexpected set bit at index: " + i, accessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); } } /* trigger second realloc */ - mutator.setSafe(valueCapacity*2, 1, 1); + vector.setSafe(valueCapacity*2, 1, 1); assertEquals(valueCapacity * 4, vector.getValueCapacity()); for (int i = valueCapacity*2; i < valueCapacity*4; i++) { if ((i & 1) == 1) { - mutator.set(i, 1); + vector.set(i, 1); } } for (int i = 0; i < valueCapacity*4; i++) { if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity*2)) { - assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i)); + assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i)); } else { - assertTrue("unexpected set bit at index: " + i, accessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); } } @@ -400,26 +397,24 @@ public void testReallocAfterVectorTransfer2() { TransferPair transferPair = vector.getTransferPair(allocator); transferPair.transfer(); final NullableBitVector toVector = (NullableBitVector)transferPair.getTo(); - final NullableBitVector.Accessor toAccessor = toVector.getAccessor(); - final NullableBitVector.Mutator toMutator = toVector.getMutator(); assertEquals(valueCapacity * 4, toVector.getValueCapacity()); /* realloc the toVector */ - toMutator.setSafe(valueCapacity * 4, 1, 1); + toVector.setSafe(valueCapacity * 4, 1, 1); for (int i = 0; i < toVector.getValueCapacity(); i++) { if (i <= valueCapacity * 4) { if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity*2) || (i == valueCapacity*4)) { - assertFalse("unexpected cleared bit at index: " + i, toAccessor.isNull(i)); + assertFalse("unexpected cleared bit at index: " + i, toVector.isNull(i)); } else { - assertTrue("unexpected set bit at index: " + i, toAccessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, toVector.isNull(i)); } } else { - assertTrue("unexpected set bit at index: " + i, toAccessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, toVector.isNull(i)); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java index 56d22932764..4d844d6d3ca 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java @@ -71,14 +71,14 @@ public void testValuesWriteRead() { for (int i = 0; i < intValues.length; i++) { BigDecimal decimal = new BigDecimal(BigInteger.valueOf(intValues[i]), scale); values[i] = decimal; - decimalVector.getMutator().setSafe(i, decimal); + decimalVector.setSafe(i, decimal); } - decimalVector.getMutator().setValueCount(intValues.length); + decimalVector.setValueCount(intValues.length); for (int i = 0; i < intValues.length; i++) { - BigDecimal value = decimalVector.getAccessor().getObject(i); - assertEquals(values[i], value); + BigDecimal value = decimalVector.getObject(i); + assertEquals("unexpected data at index: " + i, values[i], value); } } } @@ -92,7 +92,7 @@ public void testBigDecimalDifferentScaleAndPrecision() { boolean hasError = false; try { BigDecimal decimal = new BigDecimal(BigInteger.valueOf(0), 3); - decimalVector.getMutator().setSafe(0, decimal); + decimalVector.setSafe(0, decimal); } catch (UnsupportedOperationException ue) { hasError = true; } finally { @@ -103,7 +103,7 @@ public void testBigDecimalDifferentScaleAndPrecision() { hasError = false; try { BigDecimal decimal = new BigDecimal(BigInteger.valueOf(12345), 2); - decimalVector.getMutator().setSafe(0, decimal); + decimalVector.setSafe(0, decimal); } catch (UnsupportedOperationException ue) { hasError = true; } finally { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index 1185246f88c..46a2bafa002 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -135,9 +135,9 @@ public void testEncodeLargeVector() { // now run through the decoder and verify we get the original back try (ValueVector decoded = DictionaryEncoder.decode(encoded, dictionary)) { assertEquals(vector.getClass(), decoded.getClass()); - assertEquals(vector.getAccessor().getValueCount(), decoded.getAccessor().getValueCount()); + assertEquals(vector.getValueCount(), decoded.getValueCount()); for (int i = 0; i < count; ++i) { - assertEquals(vector.getAccessor().getObject(i), decoded.getAccessor().getObject(i)); + assertEquals(vector.getObject(i), decoded.getObject(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index 11cd62da52e..efa141461ca 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -56,7 +56,7 @@ public void testIntType() { vector.allocateNew(); for (int i = 0; i < 10; i++) { - vector.getMutator().setNotNull(i); + vector.setNotNull(i); nested.set(i * 2, i); nested.set(i * 2 + 1, i + 10); } @@ -80,17 +80,16 @@ public void testIntType() { public void testFloatTypeNullable() { try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) { NullableFloat4Vector nested = (NullableFloat4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); - NullableFloat4Vector.Mutator mutator = nested.getMutator(); vector.allocateNew(); for (int i = 0; i < 10; i++) { if (i % 2 == 0) { - vector.getMutator().setNotNull(i); - mutator.set(i * 2, i + 0.1f); - mutator.set(i * 2 + 1, i + 10.1f); + vector.setNotNull(i); + nested.set(i * 2, i + 0.1f); + nested.set(i * 2 + 1, i + 10.1f); } } - vector.getMutator().setValueCount(10); + vector.setValueCount(10); UnionFixedSizeListReader reader = vector.getReader(); for (int i = 0; i < 10; i++) { @@ -114,24 +113,22 @@ public void testFloatTypeNullable() { @Test public void testNestedInList() { try (ListVector vector = ListVector.empty("list", allocator)) { - ListVector.Mutator mutator = vector.getMutator(); FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector(); - FixedSizeListVector.Mutator tupleMutator = tuples.getMutator(); NullableIntVector innerVector = (NullableIntVector) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); vector.allocateNew(); for (int i = 0; i < 10; i++) { if (i % 2 == 0) { - int position = mutator.startNewValue(i); + int position = vector.startNewValue(i); for (int j = 0; j < i % 7; j++) { - tupleMutator.setNotNull(position + j); + tuples.setNotNull(position + j); innerVector.set((position + j) * 2, j); innerVector.set((position + j) * 2 + 1, j + 1); } - mutator.endValue(i, i % 7); + vector.endValue(i, i % 7); } } - mutator.setValueCount(10); + vector.setValueCount(10); UnionListReader reader = vector.getReader(); for (int i = 0; i < 10; i++) { @@ -160,24 +157,24 @@ public void testTransferPair() { try (FixedSizeListVector from = new FixedSizeListVector("from", allocator, 2, null, null); FixedSizeListVector to = new FixedSizeListVector("to", allocator, 2, null, null)) { NullableFloat4Vector nested = (NullableFloat4Vector) from.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); - NullableFloat4Vector.Mutator mutator = nested.getMutator(); from.allocateNew(); for (int i = 0; i < 10; i++) { if (i % 2 == 0) { - from.getMutator().setNotNull(i); - mutator.set(i * 2, i + 0.1f); - mutator.set(i * 2 + 1, i + 10.1f); + from.setNotNull(i); + nested.set(i * 2, i + 0.1f); + nested.set(i * 2 + 1, i + 10.1f); } } - from.getMutator().setValueCount(10); + from.setValueCount(10); TransferPair pair = from.makeTransferPair(to); pair.copyValueSafe(0, 1); pair.copyValueSafe(2, 2); to.copyFromSafe(4, 3, from); - to.getMutator().setValueCount(10); + + to.setValueCount(10); UnionFixedSizeListReader reader = to.getReader(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 1c9b5749980..2e55ccc3e74 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -88,7 +88,7 @@ public void testCopyFrom() throws Exception { for (int i = 0; i < 3; i++) { outVector.copyFrom(i, i, inVector); } - outVector.getMutator().setValueCount(3); + outVector.setValueCount(3); // assert the output vector is correct FieldReader reader = outVector.getReader(); @@ -98,11 +98,9 @@ public void testCopyFrom() throws Exception { reader.setPosition(2); Assert.assertTrue("shouldn't be null", reader.isSet()); - /* check the exact contents of vector */ - final ListVector.Accessor accessor = outVector.getAccessor(); /* index 0 */ - Object result = accessor.getObject(0); + Object result = outVector.getObject(0); ArrayList resultSet = (ArrayList) result; assertEquals(3, resultSet.size()); assertEquals(new Long(1), (Long) resultSet.get(0)); @@ -110,11 +108,11 @@ public void testCopyFrom() throws Exception { assertEquals(new Long(3), (Long) resultSet.get(2)); /* index 1 */ - result = accessor.getObject(1); + result = outVector.getObject(1); assertNull(result); /* index 2 */ - result = accessor.getObject(2); + result = outVector.getObject(2); resultSet = (ArrayList) result; assertEquals(0, resultSet.size()); } @@ -131,46 +129,46 @@ public void testSetLastSetUsage() throws Exception { /* allocate memory */ listVector.allocateNew(); - /* get inner vectors; bitVector and offsetVector */ - List innerVectors = listVector.getFieldInnerVectors(); - BitVector bitVector = (BitVector) innerVectors.get(0); - UInt4Vector offsetVector = (UInt4Vector) innerVectors.get(1); + /* get inner buffers; validityBuffer and offsetBuffer */ + + ArrowBuf validityBuffer = listVector.getValidityBuffer(); + ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); /* get the underlying data vector -- NullableBigIntVector */ NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector(); /* check current lastSet */ - assertEquals(Integer.toString(0), Integer.toString(listVector.getMutator().getLastSet())); + assertEquals(Integer.toString(0), Integer.toString(listVector.getLastSet())); int index = 0; int offset = 0; - /* write [10, 11, 12] to the list vector at index */ - bitVector.getMutator().setSafe(index, 1); - dataVector.getMutator().setSafe(0, 1, 10); - dataVector.getMutator().setSafe(1, 1, 11); - dataVector.getMutator().setSafe(2, 1, 12); - offsetVector.getMutator().setSafe(index + 1, 3); + /* write [10, 11, 12] to the list vector at index 0 */ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + dataVector.setSafe(0, 1, 10); + dataVector.setSafe(1, 1, 11); + dataVector.setSafe(2, 1, 12); + offsetBuffer.setInt((index + 1) * ListVector.OFFSET_WIDTH, 3); index += 1; /* write [13, 14] to the list vector at index 1 */ - bitVector.getMutator().setSafe(index, 1); - dataVector.getMutator().setSafe(3, 1, 13); - dataVector.getMutator().setSafe(4, 1, 14); - offsetVector.getMutator().setSafe(index + 1, 5); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + dataVector.setSafe(3, 1, 13); + dataVector.setSafe(4, 1, 14); + offsetBuffer.setInt((index + 1) * ListVector.OFFSET_WIDTH, 5); index += 1; /* write [15, 16, 17] to the list vector at index 2 */ - bitVector.getMutator().setSafe(index, 1); - dataVector.getMutator().setSafe(5, 1, 15); - dataVector.getMutator().setSafe(6, 1, 16); - dataVector.getMutator().setSafe(7, 1, 17); - offsetVector.getMutator().setSafe(index + 1, 8); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + dataVector.setSafe(5, 1, 15); + dataVector.setSafe(6, 1, 16); + dataVector.setSafe(7, 1, 17); + offsetBuffer.setInt((index + 1) * ListVector.OFFSET_WIDTH, 8); /* check current lastSet */ - assertEquals(Integer.toString(0), Integer.toString(listVector.getMutator().getLastSet())); + assertEquals(Integer.toString(0), Integer.toString(listVector.getLastSet())); /* set lastset and arbitrary valuecount for list vector. * @@ -211,54 +209,50 @@ public void testSetLastSetUsage() throws Exception { * [15, 16, 17] * } */ - listVector.getMutator().setLastSet(3); - listVector.getMutator().setValueCount(10); - - /* check the vector output */ - final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor(); - final ValueVector.Accessor valueAccessor = dataVector.getAccessor(); + listVector.setLastSet(3); + listVector.setValueCount(10); index = 0; - offset = offsetAccessor.get(index); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(0), Integer.toString(offset)); - Object actual = valueAccessor.getObject(offset); + Object actual = dataVector.getObject(offset); assertEquals(new Long(10), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(11), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(12), (Long) actual); index++; - offset = offsetAccessor.get(index); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(3), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(13), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(14), (Long) actual); index++; - offset = offsetAccessor.get(index); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(5), Integer.toString(offset)); - actual = valueAccessor.getObject(offsetAccessor.get(index)); + actual = dataVector.getObject(offset); assertEquals(new Long(15), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(16), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(17), (Long) actual); index++; - offset = offsetAccessor.get(index); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(8), Integer.toString(offset)); - actual = valueAccessor.getObject(offsetAccessor.get(index)); + actual = dataVector.getObject(offset); assertNull(actual); } } @@ -311,99 +305,97 @@ public void testSplitAndTransfer() throws Exception { listWriter.bigInt().writeBigInt(23); listWriter.endList(); - listVector.getMutator().setValueCount(5); + listVector.setValueCount(5); - assertEquals(5, listVector.getMutator().getLastSet()); + assertEquals(5, listVector.getLastSet()); - /* get offsetVector */ - UInt4Vector offsetVector = (UInt4Vector) listVector.getOffsetVector(); + /* get offset buffer */ + final ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); /* get dataVector */ NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector(); /* check the vector output */ - final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor(); - final ValueVector.Accessor valueAccessor = dataVector.getAccessor(); int index = 0; int offset = 0; Object actual = null; /* index 0 */ - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(0), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(10), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(11), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(12), (Long) actual); /* index 1 */ index++; - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(3), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(13), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(14), (Long) actual); /* index 2 */ index++; - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(5), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(15), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(16), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(17), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(18), (Long) actual); /* index 3 */ index++; - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(9), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(19), (Long) actual); /* index 4 */ index++; - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(10), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(20), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(21), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(22), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(23), (Long) actual); /* index 5 */ index++; - assertTrue(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertTrue(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(14), Integer.toString(offset)); /* do split and transfer */ @@ -426,26 +418,26 @@ public void testSplitAndTransfer() throws Exception { transferPair.splitAndTransfer(start, splitLength); /* get offsetVector of toVector */ - UInt4Vector offsetVector1 = (UInt4Vector) toVector.getOffsetVector(); - UInt4Vector.Accessor offsetAccessor1 = offsetVector1.getAccessor(); + final ArrowBuf toOffsetBuffer = toVector.getOffsetBuffer(); /* get dataVector of toVector */ NullableBigIntVector dataVector1 = (NullableBigIntVector) toVector.getDataVector(); - NullableBigIntVector.Accessor valueAccessor1 = dataVector1.getAccessor(); for (int i = 0; i < splitLength; i++) { - dataLength1 = offsetAccessor.get(start + i + 1) - offsetAccessor.get(start + i); - dataLength2 = offsetAccessor1.get(i + 1) - offsetAccessor1.get(i); + dataLength1 = offsetBuffer.getInt((start + i + 1) * ListVector.OFFSET_WIDTH) - + offsetBuffer.getInt((start + i) * ListVector.OFFSET_WIDTH); + dataLength2 = toOffsetBuffer.getInt((i + 1) * ListVector.OFFSET_WIDTH) + - toOffsetBuffer.getInt(i * ListVector.OFFSET_WIDTH); assertEquals("Different data lengths at index: " + i + " and start: " + start, dataLength1, dataLength2); - offset1 = offsetAccessor.get(start + i); - offset2 = offsetAccessor1.get(i); + offset1 = offsetBuffer.getInt((start + i) * ListVector.OFFSET_WIDTH); + offset2 = toOffsetBuffer.getInt(i * ListVector.OFFSET_WIDTH); for (int j = 0; j < dataLength1; j++) { assertEquals("Different data at indexes: " + offset1 + " and " + offset2, - valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2)); + dataVector.getObject(offset1), dataVector1.getObject(offset2)); offset1++; offset2++; @@ -509,15 +501,14 @@ public void testNestedListVector() throws Exception { listWriter.endList(); - assertEquals(2, listVector.getMutator().getLastSet()); + assertEquals(2, listVector.getLastSet()); - listVector.getMutator().setValueCount(2); + listVector.setValueCount(2); - final ListVector.Accessor accessor = listVector.getAccessor(); - assertEquals(2, accessor.getValueCount()); + assertEquals(2, listVector.getValueCount()); /* get listVector value at index 0 -- the value itself is a listvector */ - Object result = accessor.getObject(0); + Object result = listVector.getObject(0); ArrayList> resultSet = (ArrayList>) result; ArrayList list; @@ -537,7 +528,7 @@ public void testNestedListVector() throws Exception { assertEquals(new Long(175), list.get(3)); /* get listVector value at index 1 -- the value itself is a listvector */ - result = accessor.getObject(1); + result = listVector.getObject(1); resultSet = (ArrayList>) result; assertEquals(3, resultSet.size()); /* 3 inner lists at index 1 */ @@ -558,17 +549,16 @@ public void testNestedListVector() throws Exception { assertEquals(new Long(35), list.get(2)); /* check underlying bitVector */ - assertFalse(accessor.isNull(0)); - assertFalse(accessor.isNull(1)); + assertFalse(listVector.isNull(0)); + assertFalse(listVector.isNull(1)); - /* check underlying offsetVector */ - UInt4Vector offsetVector = listVector.getOffsetVector(); - final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor(); + /* check underlying offsets */ + final ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); /* listVector has 2 lists at index 0 and 3 lists at index 1 */ - assertEquals(0, offsetAccessor.get(0)); - assertEquals(2, offsetAccessor.get(1)); - assertEquals(5, offsetAccessor.get(2)); + assertEquals(0, offsetBuffer.getInt(0 * ListVector.OFFSET_WIDTH)); + assertEquals(2, offsetBuffer.getInt(1 * ListVector.OFFSET_WIDTH)); + assertEquals(5, offsetBuffer.getInt(2 * ListVector.OFFSET_WIDTH)); } } @@ -594,17 +584,15 @@ public void testGetBufferAddress() throws Exception { listWriter.bigInt().writeBigInt(300); listWriter.endList(); - final ListVector.Accessor accessor = listVector.getAccessor(); - /* check listVector contents */ - Object result = accessor.getObject(0); + Object result = listVector.getObject(0); ArrayList resultSet = (ArrayList) result; assertEquals(3, resultSet.size()); assertEquals(new Long(50), resultSet.get(0)); assertEquals(new Long(100), resultSet.get(1)); assertEquals(new Long(200), resultSet.get(2)); - result = accessor.getObject(1); + result = listVector.getObject(1); resultSet = (ArrayList) result; assertEquals(2, resultSet.size()); assertEquals(new Long(250), resultSet.get(0)); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java index 86f0bf337f9..ede4f992291 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java @@ -64,26 +64,24 @@ public void testUnionVector() throws Exception { unionVector.allocateNew(); // write some data - final UnionVector.Mutator mutator = unionVector.getMutator(); - mutator.setType(0, Types.MinorType.UINT4); - mutator.setSafe(0, uInt4Holder); - mutator.setType(2, Types.MinorType.UINT4); - mutator.setSafe(2, uInt4Holder); - mutator.setValueCount(4); + unionVector.setType(0, Types.MinorType.UINT4); + unionVector.setSafe(0, uInt4Holder); + unionVector.setType(2, Types.MinorType.UINT4); + unionVector.setSafe(2, uInt4Holder); + unionVector.setValueCount(4); // check that what we wrote is correct - final UnionVector.Accessor accessor = unionVector.getAccessor(); - assertEquals(4, accessor.getValueCount()); + assertEquals(4, unionVector.getValueCount()); - assertEquals(false, accessor.isNull(0)); - assertEquals(100, accessor.getObject(0)); + assertEquals(false, unionVector.isNull(0)); + assertEquals(100, unionVector.getObject(0)); - assertEquals(true, accessor.isNull(1)); + assertEquals(true, unionVector.isNull(1)); - assertEquals(false, accessor.isNull(2)); - assertEquals(100, accessor.getObject(2)); + assertEquals(false, unionVector.isNull(2)); + assertEquals(100, unionVector.getObject(2)); - assertEquals(true, accessor.isNull(3)); + assertEquals(true, unionVector.isNull(3)); } } @@ -93,16 +91,15 @@ public void testTransfer() throws Exception { srcVector.allocateNew(); // write some data - final UnionVector.Mutator mutator = srcVector.getMutator(); - mutator.setType(0, MinorType.INT); - mutator.setSafe(0, newIntHolder(5)); - mutator.setType(1, MinorType.BIT); - mutator.setSafe(1, newBitHolder(false)); - mutator.setType(3, MinorType.INT); - mutator.setSafe(3, newIntHolder(10)); - mutator.setType(5, MinorType.BIT); - mutator.setSafe(5, newBitHolder(false)); - mutator.setValueCount(6); + srcVector.setType(0, MinorType.INT); + srcVector.setSafe(0, newIntHolder(5)); + srcVector.setType(1, MinorType.BIT); + srcVector.setSafe(1, newBitHolder(false)); + srcVector.setType(3, MinorType.INT); + srcVector.setSafe(3, newIntHolder(10)); + srcVector.setType(5, MinorType.BIT); + srcVector.setSafe(5, newBitHolder(false)); + srcVector.setValueCount(6); try (UnionVector destVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { TransferPair pair = srcVector.makeTransferPair(destVector); @@ -116,9 +113,9 @@ public void testTransfer() throws Exception { assertEquals(srcVector.getField(), destVector.getField()); // now check the values are transferred - assertEquals(srcVector.getAccessor().getValueCount(), destVector.getAccessor().getValueCount()); - for (int i = 0; i < srcVector.getAccessor().getValueCount(); i++) { - assertEquals("Different values at index " + i, srcVector.getAccessor().get(i), destVector.getAccessor().get(i)); + assertEquals(srcVector.getValueCount(), destVector.getValueCount()); + for (int i = 0; i < srcVector.getValueCount(); i++) { + assertEquals("Different values at index " + i, srcVector.get(i), destVector.get(i)); } } } @@ -127,61 +124,58 @@ public void testTransfer() throws Exception { @Test public void testSplitAndTransfer() throws Exception { try (UnionVector sourceVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { - final UnionVector.Mutator sourceMutator = sourceVector.getMutator(); - final UnionVector.Accessor sourceAccessor = sourceVector.getAccessor(); sourceVector.allocateNew(); /* populate the UnionVector */ - sourceMutator.setType(0, MinorType.INT); - sourceMutator.setSafe(0, newIntHolder(5)); - sourceMutator.setType(1, MinorType.INT); - sourceMutator.setSafe(1, newIntHolder(10)); - sourceMutator.setType(2, MinorType.INT); - sourceMutator.setSafe(2, newIntHolder(15)); - sourceMutator.setType(3, MinorType.INT); - sourceMutator.setSafe(3, newIntHolder(20)); - sourceMutator.setType(4, MinorType.INT); - sourceMutator.setSafe(4, newIntHolder(25)); - sourceMutator.setType(5, MinorType.INT); - sourceMutator.setSafe(5, newIntHolder(30)); - sourceMutator.setType(6, MinorType.INT); - sourceMutator.setSafe(6, newIntHolder(35)); - sourceMutator.setType(7, MinorType.INT); - sourceMutator.setSafe(7, newIntHolder(40)); - sourceMutator.setType(8, MinorType.INT); - sourceMutator.setSafe(8, newIntHolder(45)); - sourceMutator.setType(9, MinorType.INT); - sourceMutator.setSafe(9, newIntHolder(50)); - sourceMutator.setValueCount(10); + sourceVector.setType(0, MinorType.INT); + sourceVector.setSafe(0, newIntHolder(5)); + sourceVector.setType(1, MinorType.INT); + sourceVector.setSafe(1, newIntHolder(10)); + sourceVector.setType(2, MinorType.INT); + sourceVector.setSafe(2, newIntHolder(15)); + sourceVector.setType(3, MinorType.INT); + sourceVector.setSafe(3, newIntHolder(20)); + sourceVector.setType(4, MinorType.INT); + sourceVector.setSafe(4, newIntHolder(25)); + sourceVector.setType(5, MinorType.INT); + sourceVector.setSafe(5, newIntHolder(30)); + sourceVector.setType(6, MinorType.INT); + sourceVector.setSafe(6, newIntHolder(35)); + sourceVector.setType(7, MinorType.INT); + sourceVector.setSafe(7, newIntHolder(40)); + sourceVector.setType(8, MinorType.INT); + sourceVector.setSafe(8, newIntHolder(45)); + sourceVector.setType(9, MinorType.INT); + sourceVector.setSafe(9, newIntHolder(50)); + sourceVector.setValueCount(10); /* check the vector output */ - assertEquals(10, sourceAccessor.getValueCount()); - assertEquals(false, sourceAccessor.isNull(0)); - assertEquals(5, sourceAccessor.getObject(0)); - assertEquals(false, sourceAccessor.isNull(1)); - assertEquals(10, sourceAccessor.getObject(1)); - assertEquals(false, sourceAccessor.isNull(2)); - assertEquals(15, sourceAccessor.getObject(2)); - assertEquals(false, sourceAccessor.isNull(3)); - assertEquals(20, sourceAccessor.getObject(3)); - assertEquals(false, sourceAccessor.isNull(4)); - assertEquals(25, sourceAccessor.getObject(4)); - assertEquals(false, sourceAccessor.isNull(5)); - assertEquals(30, sourceAccessor.getObject(5)); - assertEquals(false, sourceAccessor.isNull(6)); - assertEquals(35, sourceAccessor.getObject(6)); - assertEquals(false, sourceAccessor.isNull(7)); - assertEquals(40, sourceAccessor.getObject(7)); - assertEquals(false, sourceAccessor.isNull(8)); - assertEquals(45, sourceAccessor.getObject(8)); - assertEquals(false, sourceAccessor.isNull(9)); - assertEquals(50, sourceAccessor.getObject(9)); + assertEquals(10, sourceVector.getValueCount()); + assertEquals(false, sourceVector.isNull(0)); + assertEquals(5, sourceVector.getObject(0)); + assertEquals(false, sourceVector.isNull(1)); + assertEquals(10, sourceVector.getObject(1)); + assertEquals(false, sourceVector.isNull(2)); + assertEquals(15, sourceVector.getObject(2)); + assertEquals(false, sourceVector.isNull(3)); + assertEquals(20, sourceVector.getObject(3)); + assertEquals(false, sourceVector.isNull(4)); + assertEquals(25, sourceVector.getObject(4)); + assertEquals(false, sourceVector.isNull(5)); + assertEquals(30, sourceVector.getObject(5)); + assertEquals(false, sourceVector.isNull(6)); + assertEquals(35, sourceVector.getObject(6)); + assertEquals(false, sourceVector.isNull(7)); + assertEquals(40, sourceVector.getObject(7)); + assertEquals(false, sourceVector.isNull(8)); + assertEquals(45, sourceVector.getObject(8)); + assertEquals(false, sourceVector.isNull(9)); + assertEquals(50, sourceVector.getObject(9)); try (UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { final TransferPair transferPair = sourceVector.makeTransferPair(toVector); - final UnionVector.Accessor toAccessor = toVector.getAccessor(); final int[][] transferLengths = {{0, 3}, {3, 1}, @@ -199,8 +193,8 @@ public void testSplitAndTransfer() throws Exception { /* check the toVector output after doing the splitAndTransfer */ for (int i = 0; i < length; i++) { - assertEquals("Different data at indexes: " + (start + i) + "and " + i, sourceAccessor.getObject(start + i), - toAccessor.getObject(i)); + assertEquals("Different data at indexes: " + (start + i) + "and " + i, sourceVector.getObject(start + i), + toVector.getObject(i)); } } } @@ -210,70 +204,67 @@ public void testSplitAndTransfer() throws Exception { @Test public void testSplitAndTransferWithMixedVectors() throws Exception { try (UnionVector sourceVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { - final UnionVector.Mutator sourceMutator = sourceVector.getMutator(); - final UnionVector.Accessor sourceAccessor = sourceVector.getAccessor(); sourceVector.allocateNew(); /* populate the UnionVector */ - sourceMutator.setType(0, MinorType.INT); - sourceMutator.setSafe(0, newIntHolder(5)); + sourceVector.setType(0, MinorType.INT); + sourceVector.setSafe(0, newIntHolder(5)); - sourceMutator.setType(1, MinorType.FLOAT4); - sourceMutator.setSafe(1, newFloat4Holder(5.5f)); + sourceVector.setType(1, MinorType.FLOAT4); + sourceVector.setSafe(1, newFloat4Holder(5.5f)); - sourceMutator.setType(2, MinorType.INT); - sourceMutator.setSafe(2, newIntHolder(10)); + sourceVector.setType(2, MinorType.INT); + sourceVector.setSafe(2, newIntHolder(10)); - sourceMutator.setType(3, MinorType.FLOAT4); - sourceMutator.setSafe(3, newFloat4Holder(10.5f)); + sourceVector.setType(3, MinorType.FLOAT4); + sourceVector.setSafe(3, newFloat4Holder(10.5f)); - sourceMutator.setType(4, MinorType.INT); - sourceMutator.setSafe(4, newIntHolder(15)); + sourceVector.setType(4, MinorType.INT); + sourceVector.setSafe(4, newIntHolder(15)); - sourceMutator.setType(5, MinorType.FLOAT4); - sourceMutator.setSafe(5, newFloat4Holder(15.5f)); + sourceVector.setType(5, MinorType.FLOAT4); + sourceVector.setSafe(5, newFloat4Holder(15.5f)); - sourceMutator.setType(6, MinorType.INT); - sourceMutator.setSafe(6, newIntHolder(20)); + sourceVector.setType(6, MinorType.INT); + sourceVector.setSafe(6, newIntHolder(20)); - sourceMutator.setType(7, MinorType.FLOAT4); - sourceMutator.setSafe(7, newFloat4Holder(20.5f)); + sourceVector.setType(7, MinorType.FLOAT4); + sourceVector.setSafe(7, newFloat4Holder(20.5f)); - sourceMutator.setType(8, MinorType.INT); - sourceMutator.setSafe(8, newIntHolder(30)); + sourceVector.setType(8, MinorType.INT); + sourceVector.setSafe(8, newIntHolder(30)); - sourceMutator.setType(9, MinorType.FLOAT4); - sourceMutator.setSafe(9, newFloat4Holder(30.5f)); - sourceMutator.setValueCount(10); + sourceVector.setType(9, MinorType.FLOAT4); + sourceVector.setSafe(9, newFloat4Holder(30.5f)); + sourceVector.setValueCount(10); /* check the vector output */ - assertEquals(10, sourceAccessor.getValueCount()); - assertEquals(false, sourceAccessor.isNull(0)); - assertEquals(5, sourceAccessor.getObject(0)); - assertEquals(false, sourceAccessor.isNull(1)); - assertEquals(5.5f, sourceAccessor.getObject(1)); - assertEquals(false, sourceAccessor.isNull(2)); - assertEquals(10, sourceAccessor.getObject(2)); - assertEquals(false, sourceAccessor.isNull(3)); - assertEquals(10.5f, sourceAccessor.getObject(3)); - assertEquals(false, sourceAccessor.isNull(4)); - assertEquals(15, sourceAccessor.getObject(4)); - assertEquals(false, sourceAccessor.isNull(5)); - assertEquals(15.5f, sourceAccessor.getObject(5)); - assertEquals(false, sourceAccessor.isNull(6)); - assertEquals(20, sourceAccessor.getObject(6)); - assertEquals(false, sourceAccessor.isNull(7)); - assertEquals(20.5f, sourceAccessor.getObject(7)); - assertEquals(false, sourceAccessor.isNull(8)); - assertEquals(30, sourceAccessor.getObject(8)); - assertEquals(false, sourceAccessor.isNull(9)); - assertEquals(30.5f, sourceAccessor.getObject(9)); + assertEquals(10, sourceVector.getValueCount()); + assertEquals(false, sourceVector.isNull(0)); + assertEquals(5, sourceVector.getObject(0)); + assertEquals(false, sourceVector.isNull(1)); + assertEquals(5.5f, sourceVector.getObject(1)); + assertEquals(false, sourceVector.isNull(2)); + assertEquals(10, sourceVector.getObject(2)); + assertEquals(false, sourceVector.isNull(3)); + assertEquals(10.5f, sourceVector.getObject(3)); + assertEquals(false, sourceVector.isNull(4)); + assertEquals(15, sourceVector.getObject(4)); + assertEquals(false, sourceVector.isNull(5)); + assertEquals(15.5f, sourceVector.getObject(5)); + assertEquals(false, sourceVector.isNull(6)); + assertEquals(20, sourceVector.getObject(6)); + assertEquals(false, sourceVector.isNull(7)); + assertEquals(20.5f, sourceVector.getObject(7)); + assertEquals(false, sourceVector.isNull(8)); + assertEquals(30, sourceVector.getObject(8)); + assertEquals(false, sourceVector.isNull(9)); + assertEquals(30.5f, sourceVector.getObject(9)); try (UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { final TransferPair transferPair = sourceVector.makeTransferPair(toVector); - final UnionVector.Accessor toAccessor = toVector.getAccessor(); final int[][] transferLengths = {{0, 2}, {2, 1}, @@ -290,7 +281,7 @@ public void testSplitAndTransferWithMixedVectors() throws Exception { /* check the toVector output after doing the splitAndTransfer */ for (int i = 0; i < length; i++) { - assertEquals("Different values at index: " + i, sourceAccessor.getObject(start + i), toAccessor.getObject(i)); + assertEquals("Different values at index: " + i, sourceVector.getObject(start + i), toVector.getObject(i)); } } } @@ -300,37 +291,35 @@ public void testSplitAndTransferWithMixedVectors() throws Exception { @Test public void testGetBufferAddress() throws Exception { try (UnionVector vector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { - final UnionVector.Mutator mutator = vector.getMutator(); - final UnionVector.Accessor accessor = vector.getAccessor(); boolean error = false; vector.allocateNew(); /* populate the UnionVector */ - mutator.setType(0, MinorType.INT); - mutator.setSafe(0, newIntHolder(5)); + vector.setType(0, MinorType.INT); + vector.setSafe(0, newIntHolder(5)); - mutator.setType(1, MinorType.FLOAT4); - mutator.setSafe(1, newFloat4Holder(5.5f)); + vector.setType(1, MinorType.FLOAT4); + vector.setSafe(1, newFloat4Holder(5.5f)); - mutator.setType(2, MinorType.INT); - mutator.setSafe(2, newIntHolder(10)); + vector.setType(2, MinorType.INT); + vector.setSafe(2, newIntHolder(10)); - mutator.setType(3, MinorType.FLOAT4); - mutator.setSafe(3, newFloat4Holder(10.5f)); + vector.setType(3, MinorType.FLOAT4); + vector.setSafe(3, newFloat4Holder(10.5f)); - mutator.setValueCount(10); + vector.setValueCount(10); /* check the vector output */ - assertEquals(10, accessor.getValueCount()); - assertEquals(false, accessor.isNull(0)); - assertEquals(5, accessor.getObject(0)); - assertEquals(false, accessor.isNull(1)); - assertEquals(5.5f, accessor.getObject(1)); - assertEquals(false, accessor.isNull(2)); - assertEquals(10, accessor.getObject(2)); - assertEquals(false, accessor.isNull(3)); - assertEquals(10.5f, accessor.getObject(3)); + assertEquals(10, vector.getValueCount()); + assertEquals(false, vector.isNull(0)); + assertEquals(5, vector.getObject(0)); + assertEquals(false, vector.isNull(1)); + assertEquals(5.5f, vector.getObject(1)); + assertEquals(false, vector.isNull(2)); + assertEquals(10, vector.getObject(2)); + assertEquals(false, vector.isNull(3)); + assertEquals(10.5f, vector.getObject(3)); List buffers = vector.getFieldBuffers(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index b7f88c3babf..c7ee202f946 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -487,8 +487,6 @@ public void testNullableFixedType1() { // Create a new value vector for 1024 integers. try (final NullableUInt4Vector vector = newVector(NullableUInt4Vector.class, EMPTY_SCHEMA_PATH, new ArrowType.Int(32, false), allocator);) { - final NullableUInt4Vector.Mutator mutator = vector.getMutator(); - final NullableUInt4Vector.Accessor accessor = vector.getAccessor(); boolean error = false; int initialCapacity = 1024; @@ -500,33 +498,33 @@ public void testNullableFixedType1() { assertEquals(initialCapacity, vector.getValueCapacity()); // Put and set a few values - mutator.set(0, 100); - mutator.set(1, 101); - mutator.set(100, 102); - mutator.set(1022, 103); - mutator.set(1023, 104); + vector.set(0, 100); + vector.set(1, 101); + vector.set(100, 102); + vector.set(1022, 103); + vector.set(1023, 104); /* check vector contents */ - assertEquals(100, accessor.get(0)); - assertEquals(101, accessor.get(1)); - assertEquals(102, accessor.get(100)); - assertEquals(103, accessor.get(1022)); - assertEquals(104, accessor.get(1023)); + assertEquals(100, vector.get(0)); + assertEquals(101, vector.get(1)); + assertEquals(102, vector.get(100)); + assertEquals(103, vector.get(1022)); + assertEquals(104, vector.get(1023)); int val = 0; /* check unset bits/null values */ for (int i = 2, j = 101; i <= 99 || j <= 1021; i++, j++) { if (i <= 99) { - assertTrue(accessor.isNull(i)); + assertTrue(vector.isNull(i)); } if(j <= 1021) { - assertTrue(accessor.isNull(j)); + assertTrue(vector.isNull(j)); } } try { - mutator.set(1024, 10000); + vector.set(1024, 10000); } catch (IndexOutOfBoundsException ie) { error = true; @@ -537,7 +535,7 @@ public void testNullableFixedType1() { } try { - accessor.get(1024); + vector.get(1024); } catch (IndexOutOfBoundsException ie) { error = true; @@ -548,28 +546,28 @@ public void testNullableFixedType1() { } /* should trigger a realloc of the underlying bitvector and valuevector */ - mutator.setSafe(1024, 10000); + vector.setSafe(1024, 10000); /* check new capacity */ assertEquals(initialCapacity * 2, vector.getValueCapacity()); /* vector contents should still be intact after realloc */ - assertEquals(100, accessor.get(0)); - assertEquals(101, accessor.get(1)); - assertEquals(102, accessor.get(100)); - assertEquals(103, accessor.get(1022)); - assertEquals(104, accessor.get(1023)); - assertEquals(10000, accessor.get(1024)); + assertEquals(100, vector.get(0)); + assertEquals(101, vector.get(1)); + assertEquals(102, vector.get(100)); + assertEquals(103, vector.get(1022)); + assertEquals(104, vector.get(1023)); + assertEquals(10000, vector.get(1024)); val = 0; /* check unset bits/null values */ for (int i = 2, j = 101; i < 99 || j < 1021; i++, j++) { if (i <= 99) { - assertTrue(accessor.isNull(i)); + assertTrue(vector.isNull(i)); } if(j <= 1021) { - assertTrue(accessor.isNull(j)); + assertTrue(vector.isNull(j)); } } @@ -581,7 +579,7 @@ public void testNullableFixedType1() { /* vector data should be zeroed out */ for(int i = 0; i < (initialCapacity * 2); i++) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } } } @@ -590,8 +588,6 @@ public void testNullableFixedType1() { public void testNullableFixedType2() { // Create a new value vector for 1024 integers try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator);) { - final NullableFloat4Vector.Mutator mutator = vector.getMutator(); - final NullableFloat4Vector.Accessor accessor = vector.getAccessor(); boolean error = false; int initialCapacity = 16; @@ -603,17 +599,17 @@ public void testNullableFixedType2() { assertEquals(initialCapacity, vector.getValueCapacity()); /* populate the vector */ - mutator.set(0, 100.5f); - mutator.set(2, 201.5f); - mutator.set(4, 300.3f); - mutator.set(6, 423.8f); - mutator.set(8, 555.6f); - mutator.set(10, 66.6f); - mutator.set(12, 78.8f); - mutator.set(14, 89.5f); + vector.set(0, 100.5f); + vector.set(2, 201.5f); + vector.set(4, 300.3f); + vector.set(6, 423.8f); + vector.set(8, 555.6f); + vector.set(10, 66.6f); + vector.set(12, 78.8f); + vector.set(14, 89.5f); try { - mutator.set(16, 90.5f); + vector.set(16, 90.5f); } catch (IndexOutOfBoundsException ie) { error = true; @@ -624,25 +620,25 @@ public void testNullableFixedType2() { } /* check vector contents */ - assertEquals(100.5f, accessor.get(0), 0); - assertTrue(accessor.isNull(1)); - assertEquals(201.5f, accessor.get(2), 0); - assertTrue(accessor.isNull(3)); - assertEquals(300.3f, accessor.get(4), 0); - assertTrue(accessor.isNull(5)); - assertEquals(423.8f, accessor.get(6), 0); - assertTrue(accessor.isNull(7)); - assertEquals(555.6f, accessor.get(8), 0); - assertTrue(accessor.isNull(9)); - assertEquals(66.6f, accessor.get(10), 0); - assertTrue(accessor.isNull(11)); - assertEquals(78.8f, accessor.get(12), 0); - assertTrue(accessor.isNull(13)); - assertEquals(89.5f, accessor.get(14), 0); - assertTrue(accessor.isNull(15)); + assertEquals(100.5f, vector.get(0), 0); + assertTrue(vector.isNull(1)); + assertEquals(201.5f, vector.get(2), 0); + assertTrue(vector.isNull(3)); + assertEquals(300.3f, vector.get(4), 0); + assertTrue(vector.isNull(5)); + assertEquals(423.8f, vector.get(6), 0); + assertTrue(vector.isNull(7)); + assertEquals(555.6f, vector.get(8), 0); + assertTrue(vector.isNull(9)); + assertEquals(66.6f, vector.get(10), 0); + assertTrue(vector.isNull(11)); + assertEquals(78.8f, vector.get(12), 0); + assertTrue(vector.isNull(13)); + assertEquals(89.5f, vector.get(14), 0); + assertTrue(vector.isNull(15)); try { - accessor.get(16); + vector.get(16); } catch (IndexOutOfBoundsException ie) { error = true; @@ -653,29 +649,28 @@ public void testNullableFixedType2() { } /* this should trigger a realloc() */ - mutator.setSafe(16, 90.5f); + vector.setSafe(16, 90.5f); /* underlying buffer should now be able to store double the number of values */ assertEquals(initialCapacity * 2, vector.getValueCapacity()); /* vector data should still be intact after realloc */ - assertEquals(100.5f, accessor.get(0), 0); - assertTrue(accessor.isNull(1)); - assertEquals(201.5f, accessor.get(2), 0); - assertTrue(accessor.isNull(3)); - assertEquals(300.3f, accessor.get(4), 0); - assertTrue(accessor.isNull(5)); - assertEquals(423.8f, accessor.get(6), 0); - assertTrue(accessor.isNull(7)); - assertEquals(555.6f, accessor.get(8), 0); - assertTrue(accessor.isNull(9)); - assertEquals(66.6f, accessor.get(10), 0); - assertTrue(accessor.isNull(11)); - assertEquals(78.8f, accessor.get(12), 0); - assertTrue(accessor.isNull(13)); - assertEquals(89.5f, accessor.get(14), 0); - assertTrue(accessor.isNull(15)); - assertEquals(90.5f, accessor.get(16), 0); + assertEquals(100.5f, vector.get(0), 0); + assertTrue(vector.isNull(1)); + assertEquals(201.5f, vector.get(2), 0); + assertTrue(vector.isNull(3)); + assertEquals(300.3f, vector.get(4), 0); + assertTrue(vector.isNull(5)); + assertEquals(423.8f, vector.get(6), 0); + assertTrue(vector.isNull(7)); + assertEquals(555.6f, vector.get(8), 0); + assertTrue(vector.isNull(9)); + assertEquals(66.6f, vector.get(10), 0); + assertTrue(vector.isNull(11)); + assertEquals(78.8f, vector.get(12), 0); + assertTrue(vector.isNull(13)); + assertEquals(89.5f, vector.get(14), 0); + assertTrue(vector.isNull(15)); /* reset the vector */ vector.reset(); @@ -685,7 +680,7 @@ public void testNullableFixedType2() { /* vector data should be zeroed out */ for(int i = 0; i < (initialCapacity * 2); i++) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } } } @@ -928,32 +923,30 @@ public void testNullableVarType2() { // Create a new value vector for 1024 integers. try (final NullableVarBinaryVector vector = newNullableVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { - final NullableVarBinaryVector.Mutator m = vector.getMutator(); vector.allocateNew(1024 * 10, 1024); - m.set(0, STR1); - m.set(1, STR2); - m.set(2, STR3); - m.setSafe(3, STR3, 1, STR3.length - 1); - m.setSafe(4, STR3, 2, STR3.length - 2); + vector.set(0, STR1); + vector.set(1, STR2); + vector.set(2, STR3); + vector.setSafe(3, STR3, 1, STR3.length - 1); + vector.setSafe(4, STR3, 2, STR3.length - 2); ByteBuffer STR3ByteBuffer = ByteBuffer.wrap(STR3); - m.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1); - m.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2); + vector.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1); + vector.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2); // Check the sample strings. - final NullableVarBinaryVector.Accessor accessor = vector.getAccessor(); - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(3)); - assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(4)); - assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(5)); - assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(6)); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(3)); + assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(4)); + assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(5)); + assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(6)); // Ensure null value throws. boolean b = false; try { - vector.getAccessor().get(7); + vector.get(7); } catch (IllegalStateException e) { b = true; } finally { @@ -1064,8 +1057,6 @@ public void testReallocAfterVectorTransfer1() { @Test /* NullableFloat8Vector */ public void testReallocAfterVectorTransfer2() { try (final NullableFloat8Vector vector = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator)) { - final NullableFloat8Vector.Mutator mutator = vector.getMutator(); - final NullableFloat8Vector.Accessor accessor = vector.getAccessor(); final int initialDefaultCapacity = 4096; boolean error = false; @@ -1076,7 +1067,7 @@ public void testReallocAfterVectorTransfer2() { double baseValue = 100.375; for (int i = 0; i < initialDefaultCapacity; i++) { - mutator.setSafe(i, baseValue + (double)i); + vector.setSafe(i, baseValue + (double)i); } /* the above setSafe calls should not have triggered a realloc as @@ -1085,33 +1076,33 @@ public void testReallocAfterVectorTransfer2() { assertEquals(initialDefaultCapacity, vector.getValueCapacity()); for (int i = 0; i < initialDefaultCapacity; i++) { - double value = accessor.get(i); + double value = vector.get(i); assertEquals(baseValue + (double)i, value, 0); } /* this should trigger a realloc */ - mutator.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity); + vector.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity); assertEquals(initialDefaultCapacity * 2, vector.getValueCapacity()); for (int i = initialDefaultCapacity + 1; i < (initialDefaultCapacity * 2); i++) { - mutator.setSafe(i, baseValue + (double)i); + vector.setSafe(i, baseValue + (double)i); } for (int i = 0; i < (initialDefaultCapacity * 2); i++) { - double value = accessor.get(i); + double value = vector.get(i); assertEquals(baseValue + (double)i, value, 0); } /* this should trigger a realloc */ - mutator.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2)); + vector.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2)); assertEquals(initialDefaultCapacity * 4, vector.getValueCapacity()); for (int i = (initialDefaultCapacity * 2) + 1; i < (initialDefaultCapacity * 4); i++) { - mutator.setSafe(i, baseValue + (double)i); + vector.setSafe(i, baseValue + (double)i); } for (int i = 0; i < (initialDefaultCapacity * 4); i++) { - double value = accessor.get(i); + double value = vector.get(i); assertEquals(baseValue + (double)i, value, 0); } @@ -1123,12 +1114,11 @@ public void testReallocAfterVectorTransfer2() { transferPair.transfer(); NullableFloat8Vector toVector = (NullableFloat8Vector)transferPair.getTo(); - final NullableFloat8Vector.Accessor toAccessor = toVector.getAccessor(); /* check toVector contents before realloc */ for (int i = 0; i < (initialDefaultCapacity * 4); i++) { - assertFalse("unexpected null value at index: " + i, toAccessor.isNull(i)); - double value = toAccessor.get(i); + assertFalse("unexpected null value at index: " + i, toVector.isNull(i)); + double value = toVector.get(i); assertEquals("unexpected value at index: " + i, baseValue + (double)i, value, 0); } @@ -1138,12 +1128,12 @@ public void testReallocAfterVectorTransfer2() { for (int i = 0; i < (initialDefaultCapacity * 8); i++) { if (i < (initialDefaultCapacity * 4)) { - assertFalse("unexpected null value at index: " + i, toAccessor.isNull(i)); - double value = toAccessor.get(i); + assertFalse("unexpected null value at index: " + i, toVector.isNull(i)); + double value = toVector.get(i); assertEquals("unexpected value at index: " + i, baseValue + (double)i, value, 0); } else { - assertTrue("unexpected non-null value at index: " + i, toAccessor.isNull(i)); + assertTrue("unexpected non-null value at index: " + i, toVector.isNull(i)); } } @@ -1351,32 +1341,30 @@ public void testReallocAfterVectorTransfer4() { public void testReAllocNullableFixedWidthVector() { // Create a new value vector for 1024 integers try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator)) { - final NullableFloat4Vector.Mutator m = vector.getMutator(); vector.allocateNew(1024); assertEquals(1024, vector.getValueCapacity()); // Put values in indexes that fall within the initial allocation - m.setSafe(0, 100.1f); - m.setSafe(100, 102.3f); - m.setSafe(1023, 104.5f); + vector.setSafe(0, 100.1f); + vector.setSafe(100, 102.3f); + vector.setSafe(1023, 104.5f); // Now try to put values in space that falls beyond the initial allocation - m.setSafe(2000, 105.5f); + vector.setSafe(2000, 105.5f); // Check valueCapacity is more than initial allocation assertEquals(1024 * 2, vector.getValueCapacity()); - final NullableFloat4Vector.Accessor accessor = vector.getAccessor(); - assertEquals(100.1f, accessor.get(0), 0); - assertEquals(102.3f, accessor.get(100), 0); - assertEquals(104.5f, accessor.get(1023), 0); - assertEquals(105.5f, accessor.get(2000), 0); + assertEquals(100.1f, vector.get(0), 0); + assertEquals(102.3f, vector.get(100), 0); + assertEquals(104.5f, vector.get(1023), 0); + assertEquals(105.5f, vector.get(2000), 0); // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors // as we don't call setSafe for null values, but we do call setValueCount when all values are inserted into the // vector - m.setValueCount(vector.getValueCapacity() + 200); + vector.setValueCount(vector.getValueCapacity() + 200); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index 531a46c2734..293ffbfe192 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -28,6 +28,7 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.junit.After; import org.junit.Assert; @@ -115,7 +116,7 @@ public void testListType() { vector.reAlloc(); assertEquals(2047, vector.getValueCapacity()); // note: size - 1 - assertEquals(0, vector.getOffsetVector().getAccessor().get(2014)); + assertEquals(0, vector.getOffsetBuffer().getInt(2014 * ListVector.OFFSET_WIDTH)); } } @@ -138,7 +139,7 @@ public void testMapType() { vector.reAlloc(); assertEquals(1024, vector.getValueCapacity()); - assertNull(vector.getAccessor().getObject(513)); + assertNull(vector.getObject(513)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index 0b7928dca2c..3853eecac05 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -199,8 +199,10 @@ public void testLoadValidityBuffer() throws IOException { int count = 10; ArrowBuf[] values = new ArrowBuf[4]; for (int i = 0; i < 4; i+=2) { - ArrowBuf buf1 = allocator.buffer((int)Math.ceil(count / 8.0)); + ArrowBuf buf1 = allocator.buffer(BitVectorHelper.getValidityBufferSize(count)); ArrowBuf buf2 = allocator.buffer(count * 4); // integers + buf1.setZero(0, buf1.capacity()); + buf2.setZero(0, buf2.capacity()); values[i] = buf1; values[i+1] = buf2; for (int j = 0; j < count; j++) { @@ -314,7 +316,7 @@ public void testUnloadLoadDuplicates() throws IOException { public static VectorUnloader newVectorUnloader(FieldVector root) { Schema schema = new Schema(root.getField().getChildren()); - int valueCount = root.getAccessor().getValueCount(); + int valueCount = root.getValueCount(); List fields = root.getChildrenFromFields(); VectorSchemaRoot vsr = new VectorSchemaRoot(schema.getFields(), fields, valueCount); return new VectorUnloader(vsr); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 97efb7d5a6d..b0d6cf555e9 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -80,24 +80,23 @@ public void testPromoteToUnion() throws Exception { writer.end(); - container.getMutator().setValueCount(5); + container.setValueCount(5); final UnionVector uv = v.getChild("A", UnionVector.class); - final UnionVector.Accessor accessor = uv.getAccessor(); - assertFalse("0 shouldn't be null", accessor.isNull(0)); - assertEquals(false, accessor.getObject(0)); + assertFalse("0 shouldn't be null", uv.isNull(0)); + assertEquals(false, uv.getObject(0)); - assertFalse("1 shouldn't be null", accessor.isNull(1)); - assertEquals(true, accessor.getObject(1)); + assertFalse("1 shouldn't be null", uv.isNull(1)); + assertEquals(true, uv.getObject(1)); - assertFalse("2 shouldn't be null", accessor.isNull(2)); - assertEquals(10, accessor.getObject(2)); + assertFalse("2 shouldn't be null", uv.isNull(2)); + assertEquals(10, uv.getObject(2)); - assertTrue("3 should be null", accessor.isNull(3)); + assertTrue("3 should be null", uv.isNull(3)); - assertFalse("4 shouldn't be null", accessor.isNull(4)); - assertEquals(100, accessor.getObject(4)); + assertFalse("4 shouldn't be null", uv.isNull(4)); + assertEquals(100, uv.getObject(4)); container.clear(); container.allocateNew(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 856d60724b0..52defdc2e1b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -456,7 +456,7 @@ public void simpleUnion() { unionWriter.writeFloat4((float) i); } } - vector.getMutator().setValueCount(COUNT); + vector.setValueCount(COUNT); UnionReader unionReader = new UnionReader(vector); for (int i = 0; i < COUNT; i++) { unionReader.setPosition(i); @@ -834,7 +834,7 @@ public void complexCopierWithList() { TransferPair tp = mapVector.getTransferPair(allocator); tp.splitAndTransfer(0, 1); MapVector toMapVector = (MapVector) tp.getTo(); - JsonStringHashMap toMapValue = (JsonStringHashMap) toMapVector.getAccessor().getObject(0); + JsonStringHashMap toMapValue = (JsonStringHashMap) toMapVector.getObject(0); JsonStringArrayList object = (JsonStringArrayList) toMapValue.get("list"); assertEquals(1, object.get(0)); assertEquals(2, object.get(1)); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java index 60009b0a463..874ba99e20f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java @@ -97,14 +97,8 @@ public void tearDown() { protected void validateContent(int count, VectorSchemaRoot root) { for (int i = 0; i < count; i++) { - FieldVector fv = root.getVector("int"); - if (fv instanceof NullableIntVector) { - Assert.assertEquals(i, fv.getObject(i)); - Assert.assertEquals(Integer.valueOf(i), fv.getObject(i)); - } else { - Assert.assertEquals(i, fv.getAccessor().getObject(i)); - Assert.assertEquals(Long.valueOf(i), fv.getAccessor().getObject(i)); - } + Assert.assertEquals(i, root.getVector("int").getObject(i)); + Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); } } @@ -146,10 +140,9 @@ protected void writeComplexData(int count, MapVector parent) { public void printVectors(List vectors) { for (FieldVector vector : vectors) { LOGGER.debug(vector.getField().getName()); - Accessor accessor = vector.getAccessor(); - int valueCount = accessor.getValueCount(); + int valueCount = vector.getValueCount(); for (int i = 0; i < valueCount; i++) { - LOGGER.debug(String.valueOf(accessor.getObject(i))); + LOGGER.debug(String.valueOf(vector.getObject(i))); } } } @@ -159,14 +152,14 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) { printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { - Object intVal = root.getVector("int").getAccessor().getObject(i); + Object intVal = root.getVector("int").getObject(i); if (i % 5 != 3) { Assert.assertEquals(i, intVal); } else { Assert.assertNull(intVal); } - Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i)); - Assert.assertEquals(i % 3, ((List) root.getVector("list").getAccessor().getObject(i)).size()); + Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); + Assert.assertEquals(i % 3, ((List) root.getVector("list").getObject(i)).size()); NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder(); FieldReader mapReader = root.getVector("map").getReader(); mapReader.setPosition(i); @@ -209,15 +202,15 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) { Assert.assertEquals(count, root.getRowCount()); printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { - long dateVal = ((NullableDateMilliVector) root.getVector("date")).getAccessor().get(i); + long dateVal = ((NullableDateMilliVector) root.getVector("date")).get(i); LocalDateTime dt = makeDateTimeFromCount(i); LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay()); Assert.assertEquals(DateUtility.toMillis(dateExpected), dateVal); - long timeVal = ((NullableTimeMilliVector) root.getVector("time")).getAccessor().get(i); + long timeVal = ((NullableTimeMilliVector) root.getVector("time")).get(i); Assert.assertEquals(dt.getMillisOfDay(), timeVal); - Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i); + Object timestampMilliVal = root.getVector("timestamp-milli").getObject(i); Assert.assertEquals(dt, timestampMilliVal); - Object timestampMilliTZVal = root.getVector("timestamp-milliTZ").getAccessor().getObject(i); + Object timestampMilliTZVal = root.getVector("timestamp-milliTZ").getObject(i); Assert.assertEquals(DateUtility.toMillis(dt), timestampMilliTZVal); } } @@ -282,7 +275,7 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat List fields = ImmutableList.of(encodedVector1A.getField(), encodedVector1B.getField(), encodedVector2.getField()); List vectors = ImmutableList.of(encodedVector1A, encodedVector1B, encodedVector2); - return new VectorSchemaRoot(fields, vectors, encodedVector1A.getAccessor().getValueCount()); + return new VectorSchemaRoot(fields, vectors, encodedVector1A.getValueCount()); } protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider provider) { @@ -293,14 +286,13 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider Assert.assertNotNull(encoding1A); Assert.assertEquals(1L, encoding1A.getId()); - FieldVector.Accessor accessor = vector1A.getAccessor(); - Assert.assertEquals(6, accessor.getValueCount()); - Assert.assertEquals(0, accessor.getObject(0)); - Assert.assertEquals(1, accessor.getObject(1)); - Assert.assertEquals(null, accessor.getObject(2)); - Assert.assertEquals(2, accessor.getObject(3)); - Assert.assertEquals(1, accessor.getObject(4)); - Assert.assertEquals(2, accessor.getObject(5)); + Assert.assertEquals(6, vector1A.getValueCount()); + Assert.assertEquals(0, vector1A.getObject(0)); + Assert.assertEquals(1, vector1A.getObject(1)); + Assert.assertEquals(null, vector1A.getObject(2)); + Assert.assertEquals(2, vector1A.getObject(3)); + Assert.assertEquals(1, vector1A.getObject(4)); + Assert.assertEquals(2, vector1A.getObject(5)); FieldVector vector1B = root.getVector("varcharB"); Assert.assertNotNull(vector1B); @@ -310,14 +302,13 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider Assert.assertTrue(encoding1A.equals(encoding1B)); Assert.assertEquals(1L, encoding1B.getId()); - accessor = vector1B.getAccessor(); - Assert.assertEquals(6, accessor.getValueCount()); - Assert.assertEquals(2, accessor.getObject(0)); - Assert.assertEquals(1, accessor.getObject(1)); - Assert.assertEquals(2, accessor.getObject(2)); - Assert.assertEquals(null, accessor.getObject(3)); - Assert.assertEquals(1, accessor.getObject(4)); - Assert.assertEquals(0, accessor.getObject(5)); + Assert.assertEquals(6, vector1B.getValueCount()); + Assert.assertEquals(2, vector1B.getObject(0)); + Assert.assertEquals(1, vector1B.getObject(1)); + Assert.assertEquals(2, vector1B.getObject(2)); + Assert.assertEquals(null, vector1B.getObject(3)); + Assert.assertEquals(1, vector1B.getObject(4)); + Assert.assertEquals(0, vector1B.getObject(5)); FieldVector vector2 = root.getVector("sizes"); Assert.assertNotNull(vector2); @@ -326,30 +317,29 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider Assert.assertNotNull(encoding2); Assert.assertEquals(2L, encoding2.getId()); - accessor = vector2.getAccessor(); - Assert.assertEquals(6, accessor.getValueCount()); - Assert.assertEquals(null, accessor.getObject(0)); - Assert.assertEquals(2, accessor.getObject(1)); - Assert.assertEquals(1, accessor.getObject(2)); - Assert.assertEquals(1, accessor.getObject(3)); - Assert.assertEquals(2, accessor.getObject(4)); - Assert.assertEquals(null, accessor.getObject(5)); + Assert.assertEquals(6, vector2.getValueCount()); + Assert.assertEquals(null, vector2.getObject(0)); + Assert.assertEquals(2, vector2.getObject(1)); + Assert.assertEquals(1, vector2.getObject(2)); + Assert.assertEquals(1, vector2.getObject(3)); + Assert.assertEquals(2, vector2.getObject(4)); + Assert.assertEquals(null, vector2.getObject(5)); Dictionary dictionary1 = provider.lookup(1L); Assert.assertNotNull(dictionary1); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary1.getVector()).getAccessor(); - Assert.assertEquals(3, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); - Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2)); + NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary1.getVector()); + Assert.assertEquals(3, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); + Assert.assertEquals(new Text("baz"), dictionaryVector.getObject(2)); Dictionary dictionary2 = provider.lookup(2L); Assert.assertNotNull(dictionary2); - dictionaryAccessor = ((NullableVarCharVector) dictionary2.getVector()).getAccessor(); - Assert.assertEquals(3, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("micro"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("small"), dictionaryAccessor.getObject(1)); - Assert.assertEquals(new Text("large"), dictionaryAccessor.getObject(2)); + dictionaryVector = ((NullableVarCharVector) dictionary2.getVector()); + Assert.assertEquals(3, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("micro"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("small"), dictionaryVector.getObject(1)); + Assert.assertEquals(new Text("large"), dictionaryVector.getObject(2)); } protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAllocator, DictionaryProvider.MapDictionaryProvider provider) { @@ -359,7 +349,7 @@ protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAlloc dictionaryVector.allocateNewSafe(); dictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); dictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); - dictionaryVector.getMutator().setValueCount(2); + dictionaryVector.setValueCount(2); Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(2L, false, null)); provider.put(dictionary); @@ -398,18 +388,17 @@ protected void validateNestedDictionary(VectorSchemaRoot root, DictionaryProvide Assert.assertEquals(2L, encoding.getId()); Assert.assertEquals(new ArrowType.Int(32, true), encoding.getIndexType()); - FieldVector.Accessor accessor = vector.getAccessor(); - Assert.assertEquals(3, accessor.getValueCount()); - Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0)); - Assert.assertEquals(Arrays.asList(0), accessor.getObject(1)); - Assert.assertEquals(Arrays.asList(1), accessor.getObject(2)); + Assert.assertEquals(3, vector.getValueCount()); + Assert.assertEquals(Arrays.asList(0, 1), vector.getObject(0)); + Assert.assertEquals(Arrays.asList(0), vector.getObject(1)); + Assert.assertEquals(Arrays.asList(1), vector.getObject(2)); Dictionary dictionary = provider.lookup(2L); Assert.assertNotNull(dictionary); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor(); - Assert.assertEquals(2, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); + NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector()); + Assert.assertEquals(2, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); } protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) { @@ -423,14 +412,14 @@ protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) { decimalVector3.allocateNew(count); for (int i = 0; i < count; i++) { - decimalVector1.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i), 3)); - decimalVector2.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i * (1 << 10)), 2)); - decimalVector3.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), 8)); + decimalVector1.setSafe(i, new BigDecimal(BigInteger.valueOf(i), 3)); + decimalVector2.setSafe(i, new BigDecimal(BigInteger.valueOf(i * (1 << 10)), 2)); + decimalVector3.setSafe(i, new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), 8)); } - decimalVector1.getMutator().setValueCount(count); - decimalVector2.getMutator().setValueCount(count); - decimalVector3.getMutator().setValueCount(count); + decimalVector1.setValueCount(count); + decimalVector2.setValueCount(count); + decimalVector3.setValueCount(count); List fields = ImmutableList.of(decimalVector1.getField(), decimalVector2.getField(), decimalVector3.getField()); List vectors = ImmutableList.of(decimalVector1, decimalVector2, decimalVector3); @@ -446,19 +435,19 @@ protected void validateDecimalData(VectorSchemaRoot root) { for (int i = 0; i < count; i++) { // Verify decimal 1 vector - BigDecimal readValue = decimalVector1.getAccessor().getObject(i); + BigDecimal readValue = decimalVector1.getObject(i); ArrowType.Decimal type = (ArrowType.Decimal) decimalVector1.getField().getType(); BigDecimal genValue = new BigDecimal(BigInteger.valueOf(i), type.getScale()); Assert.assertEquals(genValue, readValue); // Verify decimal 2 vector - readValue = decimalVector2.getAccessor().getObject(i); + readValue = decimalVector2.getObject(i); type = (ArrowType.Decimal) decimalVector2.getField().getType(); genValue = new BigDecimal(BigInteger.valueOf(i * (1 << 10)), type.getScale()); Assert.assertEquals(genValue, readValue); // Verify decimal 3 vector - readValue = decimalVector3.getAccessor().getObject(i); + readValue = decimalVector3.getObject(i); type = (ArrowType.Decimal) decimalVector3.getField().getType(); genValue = new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), type.getScale()); Assert.assertEquals(genValue, readValue); @@ -571,7 +560,7 @@ protected void validateVarBinary(int count, VectorSchemaRoot root) { int numVarBinaryValues = 0; for (int i = 0; i < count; i++) { expectedArray[i] = (byte) i; - Object obj = listVector.getAccessor().getObject(i); + Object obj = listVector.getObject(i); List objList = (List) obj; if (i % 3 == 0) { Assert.assertTrue(objList.isEmpty()); @@ -586,10 +575,10 @@ protected void validateVarBinary(int count, VectorSchemaRoot root) { } // ListVector lastSet should be the index of last value + 1 - Assert.assertEquals(listVector.getMutator().getLastSet(), count); + Assert.assertEquals(listVector.getLastSet(), count); // NullableVarBinaryVector lastSet should be the index of last value NullableVarBinaryVector binaryVector = (NullableVarBinaryVector) listVector.getChildrenFromFields().get(0); - Assert.assertEquals(binaryVector.getMutator().getLastSet(), numVarBinaryValues - 1); + Assert.assertEquals(binaryVector.getLastSet(), numVarBinaryValues - 1); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java index feae08ea223..8559969a204 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java @@ -317,11 +317,11 @@ public void testWriteReadTiny() throws IOException { try (VectorSchemaRoot root = VectorSchemaRoot.create(MessageSerializerTest.testSchema(), allocator)) { root.getFieldVectors().get(0).allocateNew(); - NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator(); + NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); for (int i = 0; i < 16; i++) { - mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); + vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); } - mutator.setValueCount(16); + vector.setValueCount(16); root.setRowCount(16); // write file @@ -368,9 +368,9 @@ private void validateTinyData(VectorSchemaRoot root) { NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); for (int i = 0; i < 16; i++) { if (i < 8) { - Assert.assertEquals((byte) (i + 1), vector.getAccessor().get(i)); + Assert.assertEquals((byte) (i + 1), vector.get(i)); } else { - Assert.assertTrue(vector.getAccessor().isNull(i)); + Assert.assertTrue(vector.isNull(i)); } } } @@ -397,7 +397,7 @@ public void testWriteReadMetadata() throws IOException { try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) { vector.allocateNewSafe(); - vector.getMutator().setValueCount(0); + vector.setValueCount(0); List vectors = ImmutableList.of(vector); VectorSchemaRoot root = new VectorSchemaRoot(originalSchema, vectors, 0); @@ -578,13 +578,13 @@ public void testWriteReadFixedSizeList() throws IOException { parent.allocateNew(); for (int i = 0; i < 10; i++) { - tuples.getMutator().setNotNull(i); - floats.getMutator().set(i * 2, i + 0.1f); - floats.getMutator().set(i * 2 + 1, i + 10.1f); + tuples.setNotNull(i); + floats.set(i * 2, i + 0.1f); + floats.set(i * 2 + 1, i + 10.1f); ints.set(i, i); } - parent.getMutator().setValueCount(10); + parent.setValueCount(10); write(parent, file, stream); } @@ -600,8 +600,8 @@ public void testWriteReadFixedSizeList() throws IOException { arrowReader.loadRecordBatch(rbBlock); Assert.assertEquals(count, root.getRowCount()); for (int i = 0; i < 10; i++) { - Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getAccessor().getObject(i)); - Assert.assertEquals(i, root.getVector("ints").getAccessor().getObject(i)); + Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getObject(i)); + Assert.assertEquals(i, root.getVector("ints").getObject(i)); } } } @@ -616,8 +616,8 @@ public void testWriteReadFixedSizeList() throws IOException { arrowReader.loadNextBatch(); Assert.assertEquals(count, root.getRowCount()); for (int i = 0; i < 10; i++) { - Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getAccessor().getObject(i)); - Assert.assertEquals(i, root.getVector("ints").getAccessor().getObject(i)); + Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getObject(i)); + Assert.assertEquals(i, root.getVector("ints").getObject(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java index e2efabef009..c7e34191a3f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java @@ -70,11 +70,11 @@ public void testReadWrite() throws IOException { int numBatches = 1; root.getFieldVectors().get(0).allocateNew(); - NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator(); + NullableTinyIntVector vector = (NullableTinyIntVector)root.getFieldVectors().get(0); for (int i = 0; i < 16; i++) { - mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); + vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); } - mutator.setValueCount(16); + vector.setValueCount(16); root.setRowCount(16); ByteArrayOutputStream out = new ByteArrayOutputStream(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java index a19c3795fd5..b989f95be92 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java @@ -62,13 +62,13 @@ public void run() { writer.start(); for (int j = 0; j < numBatches; j++) { root.getFieldVectors().get(0).allocateNew(); - NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator(); + NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); // Send a changing batch id first - mutator.set(0, j); + vector.set(0, j); for (int i = 1; i < 16; i++) { - mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); + vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); } - mutator.setValueCount(16); + vector.setValueCount(16); root.setRowCount(16); writer.writeBatch(); @@ -117,12 +117,12 @@ public boolean loadNextBatch() throws IOException { VectorSchemaRoot root = getVectorSchemaRoot(); Assert.assertEquals(16, root.getRowCount()); NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); - Assert.assertEquals((byte) (batchesRead - 1), vector.getAccessor().get(0)); + Assert.assertEquals((byte) (batchesRead - 1), vector.get(0)); for (int i = 1; i < 16; i++) { if (i < 8) { - Assert.assertEquals((byte) (i + 1), vector.getAccessor().get(i)); + Assert.assertEquals((byte) (i + 1), vector.get(i)); } else { - Assert.assertTrue(vector.getAccessor().isNull(i)); + Assert.assertTrue(vector.isNull(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java index ee90d340d7c..5c4c48cd26b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java @@ -280,7 +280,7 @@ public void testSetStructLength() throws IOException { // initialize vectors try (VectorSchemaRoot root = reader.read();) { FieldVector vector = root.getVector("struct_nullable"); - Assert.assertEquals(7, vector.getAccessor().getValueCount()); + Assert.assertEquals(7, vector.getValueCount()); } } }