Skip to content

[MINOR][SQL] Deduplicate huge if statements in get between specialized getters #24016

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
HeartSaVioR wants to merge 4 commits into from
Closed

[MINOR][SQL] Deduplicate huge if statements in get between specialized getters #24016

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f75b497
[MINOR][SQL] Deduplicate huge if statements in get between specialize…
HeartSaVioR Mar 7, 2019
437dd08
Fix public -> private
HeartSaVioR Mar 7, 2019
d1f00ac
Address review comments
HeartSaVioR Mar 11, 2019
b7535df
Minor rearrangement
HeartSaVioR Mar 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 89 additions & 0 deletions ...yst/src/main/java/org/apache/spark/sql/catalyst/expressions/SpecializedGettersReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.spark.sql.catalyst.expressions;

import org.apache.spark.sql.types.*;

public final class SpecializedGettersReader {
private SpecializedGettersReader() {}

public static Object read(
SpecializedGetters obj,
int ordinal,
DataType dataType,
boolean handleNull,
boolean handleUserDefinedType) {
if (handleNull && (obj.isNullAt(ordinal) || dataType instanceof NullType)) {
return null;
}
if (dataType instanceof BooleanType) {
return obj.getBoolean(ordinal);
}
if (dataType instanceof ByteType) {
return obj.getByte(ordinal);
}
if (dataType instanceof ShortType) {
return obj.getShort(ordinal);
}
if (dataType instanceof IntegerType) {
return obj.getInt(ordinal);
}
if (dataType instanceof LongType) {
return obj.getLong(ordinal);
}
if (dataType instanceof FloatType) {
return obj.getFloat(ordinal);
}
if (dataType instanceof DoubleType) {
return obj.getDouble(ordinal);
}
if (dataType instanceof StringType) {
return obj.getUTF8String(ordinal);
}
if (dataType instanceof DecimalType) {
DecimalType dt = (DecimalType) dataType;
return obj.getDecimal(ordinal, dt.precision(), dt.scale());
}
if (dataType instanceof DateType) {
return obj.getInt(ordinal);
}
if (dataType instanceof TimestampType) {
return obj.getLong(ordinal);
}
if (dataType instanceof CalendarIntervalType) {
return obj.getInterval(ordinal);
}
if (dataType instanceof BinaryType) {
return obj.getBinary(ordinal);
}
if (dataType instanceof StructType) {
return obj.getStruct(ordinal, ((StructType) dataType).size());
}
if (dataType instanceof ArrayType) {
return obj.getArray(ordinal);
}
if (dataType instanceof MapType) {
return obj.getMap(ordinal);
}
if (handleUserDefinedType && dataType instanceof UserDefinedType) {
return obj.get(ordinal, ((UserDefinedType)dataType).sqlType());
}

throw new UnsupportedOperationException("Unsupported data type " + dataType.simpleString());
}
}
42 changes: 1 addition & 41 deletions sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
*/

public final class UnsafeArrayData extends ArrayData {

public static int calculateHeaderPortionInBytes(int numFields) {
return (int)calculateHeaderPortionInBytes((long)numFields);
}
Expand Down Expand Up @@ -137,46 +136,7 @@ public boolean isNullAt(int ordinal) {

@Override
public Object get(int ordinal, DataType dataType) {
if (isNullAt(ordinal) || dataType instanceof NullType) {
return null;
} else if (dataType instanceof BooleanType) {
return getBoolean(ordinal);
} else if (dataType instanceof ByteType) {
return getByte(ordinal);
} else if (dataType instanceof ShortType) {
return getShort(ordinal);
} else if (dataType instanceof IntegerType) {
return getInt(ordinal);
} else if (dataType instanceof LongType) {
return getLong(ordinal);
} else if (dataType instanceof FloatType) {
return getFloat(ordinal);
} else if (dataType instanceof DoubleType) {
return getDouble(ordinal);
} else if (dataType instanceof DecimalType) {
DecimalType dt = (DecimalType) dataType;
return getDecimal(ordinal, dt.precision(), dt.scale());
} else if (dataType instanceof DateType) {
return getInt(ordinal);
} else if (dataType instanceof TimestampType) {
return getLong(ordinal);
} else if (dataType instanceof BinaryType) {
return getBinary(ordinal);
} else if (dataType instanceof StringType) {
return getUTF8String(ordinal);
} else if (dataType instanceof CalendarIntervalType) {
return getInterval(ordinal);
} else if (dataType instanceof StructType) {
return getStruct(ordinal, ((StructType) dataType).size());
} else if (dataType instanceof ArrayType) {
return getArray(ordinal);
} else if (dataType instanceof MapType) {
return getMap(ordinal);
} else if (dataType instanceof UserDefinedType) {
return get(ordinal, ((UserDefinedType)dataType).sqlType());
} else {
throw new UnsupportedOperationException("Unsupported data type " + dataType.simpleString());
}
return SpecializedGettersReader.read(this, ordinal, dataType, true, true);
}

@Override
Expand Down
41 changes: 1 addition & 40 deletions sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,46 +299,7 @@ public void setDecimal(int ordinal, Decimal value, int precision) {

@Override
public Object get(int ordinal, DataType dataType) {
if (isNullAt(ordinal) || dataType instanceof NullType) {
return null;
} else if (dataType instanceof BooleanType) {
return getBoolean(ordinal);
} else if (dataType instanceof ByteType) {
return getByte(ordinal);
} else if (dataType instanceof ShortType) {
return getShort(ordinal);
} else if (dataType instanceof IntegerType) {
return getInt(ordinal);
} else if (dataType instanceof LongType) {
return getLong(ordinal);
} else if (dataType instanceof FloatType) {
return getFloat(ordinal);
} else if (dataType instanceof DoubleType) {
return getDouble(ordinal);
} else if (dataType instanceof DecimalType) {
DecimalType dt = (DecimalType) dataType;
return getDecimal(ordinal, dt.precision(), dt.scale());
} else if (dataType instanceof DateType) {
return getInt(ordinal);
} else if (dataType instanceof TimestampType) {
return getLong(ordinal);
} else if (dataType instanceof BinaryType) {
return getBinary(ordinal);
} else if (dataType instanceof StringType) {
return getUTF8String(ordinal);
} else if (dataType instanceof CalendarIntervalType) {
return getInterval(ordinal);
} else if (dataType instanceof StructType) {
return getStruct(ordinal, ((StructType) dataType).size());
} else if (dataType instanceof ArrayType) {
return getArray(ordinal);
} else if (dataType instanceof MapType) {
return getMap(ordinal);
} else if (dataType instanceof UserDefinedType) {
return get(ordinal, ((UserDefinedType)dataType).sqlType());
} else {
throw new UnsupportedOperationException("Unsupported data type " + dataType.simpleString());
}
return SpecializedGettersReader.read(this, ordinal, dataType, true, true);
}

@Override
Expand Down
38 changes: 2 additions & 36 deletions sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.apache.spark.sql.vectorized;

import org.apache.spark.annotation.Evolving;
import org.apache.spark.sql.catalyst.expressions.SpecializedGettersReader;
import org.apache.spark.sql.catalyst.expressions.UnsafeArrayData;
import org.apache.spark.sql.catalyst.util.ArrayData;
import org.apache.spark.sql.catalyst.util.GenericArrayData;
Expand Down Expand Up @@ -174,42 +175,7 @@ public ColumnarMap getMap(int ordinal) {

@Override
public Object get(int ordinal, DataType dataType) {
if (dataType instanceof BooleanType) {
return getBoolean(ordinal);
} else if (dataType instanceof ByteType) {
return getByte(ordinal);
} else if (dataType instanceof ShortType) {
return getShort(ordinal);
} else if (dataType instanceof IntegerType) {
return getInt(ordinal);
} else if (dataType instanceof LongType) {
return getLong(ordinal);
} else if (dataType instanceof FloatType) {
return getFloat(ordinal);
} else if (dataType instanceof DoubleType) {
return getDouble(ordinal);
} else if (dataType instanceof StringType) {
return getUTF8String(ordinal);
} else if (dataType instanceof BinaryType) {
return getBinary(ordinal);
} else if (dataType instanceof DecimalType) {
DecimalType t = (DecimalType) dataType;
return getDecimal(ordinal, t.precision(), t.scale());
} else if (dataType instanceof DateType) {
return getInt(ordinal);
} else if (dataType instanceof TimestampType) {
return getLong(ordinal);
} else if (dataType instanceof ArrayType) {
return getArray(ordinal);
} else if (dataType instanceof StructType) {
return getStruct(ordinal, ((StructType)dataType).fields().length);
} else if (dataType instanceof MapType) {
return getMap(ordinal);
} else if (dataType instanceof CalendarIntervalType) {
return getInterval(ordinal);
} else {
throw new UnsupportedOperationException("Datatype not supported " + dataType);
}
return SpecializedGettersReader.read(this, ordinal, dataType, false, false);
}

@Override
Expand Down