diff --git a/.gitignore b/.gitignore index 1f4e6dc48ee5..a4a5efdd670b 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,6 @@ sdist/ coverage.xml .pytest_cache/ spark/tmp/ -spark-warehouse/ spark/spark-warehouse/ spark2/spark-warehouse/ spark3/spark-warehouse/ diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/ConstantColumnVector.java b/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/ConstantColumnVector.java index c3acbc4f0d00..7e3d94a97375 100644 --- a/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/ConstantColumnVector.java +++ b/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/ConstantColumnVector.java @@ -59,37 +59,37 @@ public boolean isNullAt(int rowId) { @Override public boolean getBoolean(int rowId) { - return constant != null ? (boolean) constant : false; + return (boolean) constant; } @Override public byte getByte(int rowId) { - return constant != null ? (byte) constant : 0; + return (byte) constant; } @Override public short getShort(int rowId) { - return constant != null ? (short) constant : 0; + return (short) constant; } @Override public int getInt(int rowId) { - return constant != null ? (int) constant : 0; + return (int) constant; } @Override public long getLong(int rowId) { - return constant != null ? (long) constant : 0L; + return (long) constant; } @Override public float getFloat(int rowId) { - return constant != null ? (float) constant : 0.0F; + return (float) constant; } @Override public double getDouble(int rowId) { - return constant != null ? (double) constant : 0.0; + return (double) constant; } @Override diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkOrcReaders.java b/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkOrcReaders.java index 564fcfa0b3da..4d3d0ec78851 100644 --- a/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkOrcReaders.java +++ b/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkOrcReaders.java @@ -272,32 +272,27 @@ private static class PrimitiveOrcColumnVector extends BaseOrcColumnVector { @Override public boolean getBoolean(int rowId) { - Boolean value = (Boolean) primitiveValueReader.read(vector, rowId); - return value != null ? value : false; + return (Boolean) primitiveValueReader.read(vector, rowId); } @Override public int getInt(int rowId) { - Integer value = (Integer) primitiveValueReader.read(vector, rowId); - return value != null ? value : 0; + return (Integer) primitiveValueReader.read(vector, rowId); } @Override public long getLong(int rowId) { - Long value = (Long) primitiveValueReader.read(vector, rowId); - return value != null ? value : 0L; + return (Long) primitiveValueReader.read(vector, rowId); } @Override public float getFloat(int rowId) { - Float value = (Float) primitiveValueReader.read(vector, rowId); - return value != null ? value : 0.0F; + return (Float) primitiveValueReader.read(vector, rowId); } @Override public double getDouble(int rowId) { - Double value = (Double) primitiveValueReader.read(vector, rowId); - return value != null ? value : 0.0; + return (Double) primitiveValueReader.read(vector, rowId); } @Override @@ -335,12 +330,8 @@ public ColumnVector convert(org.apache.orc.storage.ql.exec.vector.ColumnVector v return new BaseOrcColumnVector(listType, batchSize, vector) { @Override public ColumnarArray getArray(int rowId) { - if (isNullAt(rowId)) { - return null; - } else { - int index = getRowIndex(rowId); - return new ColumnarArray(elementVector, (int) listVector.offsets[index], (int) listVector.lengths[index]); - } + int index = getRowIndex(rowId); + return new ColumnarArray(elementVector, (int) listVector.offsets[index], (int) listVector.lengths[index]); } }; } @@ -366,13 +357,9 @@ public ColumnVector convert(org.apache.orc.storage.ql.exec.vector.ColumnVector v return new BaseOrcColumnVector(mapType, batchSize, vector) { @Override public ColumnarMap getMap(int rowId) { - if (isNullAt(rowId)) { - return null; - } else { - int index = getRowIndex(rowId); - return new ColumnarMap(keyVector, valueVector, (int) mapVector.offsets[index], - (int) mapVector.lengths[index]); - } + int index = getRowIndex(rowId); + return new ColumnarMap(keyVector, valueVector, (int) mapVector.offsets[index], + (int) mapVector.lengths[index]); } }; } diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java b/spark/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java index aa0b24785cf0..e04099a0e026 100644 --- a/spark/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java +++ b/spark/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java @@ -649,7 +649,9 @@ private static void assertEquals(String context, ArrayType array, ArrayData expe expected.numElements(), actual.numElements()); DataType type = array.elementType(); for (int i = 0; i < actual.numElements(); i += 1) { - assertEquals(context + ".element", type, expected.get(i, type), actual.get(i, type)); + assertEquals(context + ".element", type, + expected.isNullAt(i) ? null : expected.get(i, type), + actual.isNullAt(i) ? null : actual.get(i, type)); } } @@ -667,9 +669,11 @@ private static void assertEquals(String context, MapType map, MapData expected, for (int i = 0; i < actual.numElements(); i += 1) { assertEquals(context + ".key", keyType, - expectedKeys.get(i, keyType), actualKeys.get(i, keyType)); + expectedKeys.isNullAt(i) ? null : expectedKeys.get(i, keyType), + actualKeys.isNullAt(i) ? null : actualKeys.get(i, keyType)); assertEquals(context + ".value", valueType, - expectedValues.get(i, valueType), actualValues.get(i, valueType)); + expectedValues.isNullAt(i) ? null : expectedValues.get(i, valueType), + actualValues.isNullAt(i) ? null : actualValues.get(i, valueType)); } } }