Skip to content

Add block position convention in all distinct from operators #11263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
gerlou wants to merge 1 commit into from
Closed

Add block position convention in all distinct from operators #11263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions presto-main/src/main/java/com/facebook/presto/metadata/FunctionRegistry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,27 +482,46 @@ public FunctionRegistry(TypeManager typeManager, BlockEncodingSerde blockEncodin
.scalars(JsonFunctions.class)
.scalars(ColorFunctions.class)
.scalars(ColorOperators.class)
.scalar(ColorOperators.ColorIsDistinctFrom.class)
.scalars(HyperLogLogFunctions.class)
.scalars(UnknownOperators.class)
.scalar(UnknownOperators.UnknownIsDistinctFrom.class)
.scalars(BooleanOperators.class)
.scalar(BooleanOperators.BooleanIsDistinctFrom.class)
.scalars(BigintOperators.class)
.scalar(BigintOperators.BigIntIsDistinctFrom.class)
.scalars(IntegerOperators.class)
.scalar(IntegerOperators.IntegerIsDistinctFrom.class)
.scalars(SmallintOperators.class)
.scalar(SmallintOperators.SmallIntIsDistinctFrom.class)
.scalars(TinyintOperators.class)
.scalar(TinyintOperators.TinyIntIsDistinctFrom.class)
.scalars(DoubleOperators.class)
.scalar(DoubleOperators.DoubleIsDistinctFrom.class)
.scalars(RealOperators.class)
.scalar(RealOperators.RealIsDistinctFrom.class)
.scalars(VarcharOperators.class)
.scalar(VarcharOperators.VarcharIsDistinctFrom.class)
.scalars(VarbinaryOperators.class)
.scalar(VarbinaryOperators.VarbinaryIsDistinctFrom.class)
.scalars(DateOperators.class)
.scalar(DateOperators.DateIsDistinctFrom.class)
.scalars(TimeOperators.class)
.scalar(TimeOperators.TimeIsDistinctFrom.class)
.scalars(TimestampOperators.class)
.scalar(TimestampOperators.TimestampIsDistinctFrom.class)
.scalars(IntervalDayTimeOperators.class)
.scalar(IntervalDayTimeOperators.IntervalDayTimeIsDistinctFrom.class)
.scalars(IntervalYearMonthOperators.class)
.scalar(IntervalYearMonthOperators.IntervalYearMonthIsDistinctFrom.class)
.scalars(TimeWithTimeZoneOperators.class)
.scalar(TimeWithTimeZoneOperators.TimeWithTimeZoneIsDistinctFrom.class)
.scalars(TimestampWithTimeZoneOperators.class)
.scalar(TimestampWithTimeZoneOperators.TimestampWithTimeZoneIsDistinctFrom.class)
.scalars(DateTimeOperators.class)
.scalars(HyperLogLogOperators.class)
.scalars(IpAddressOperators.class)
.scalar(IpAddressOperators.IpAddressIsDistinctFrom.class)
.scalars(LikeFunctions.class)
.scalars(ArrayFunctions.class)
.scalars(HmacFunctions.class)
Expand All @@ -512,10 +531,12 @@ public FunctionRegistry(TypeManager typeManager, BlockEncodingSerde blockEncodin
.scalar(ArrayPositionFunction.class)
.scalars(CombineHashFunction.class)
.scalars(JsonOperators.class)
.scalar(JsonOperators.JsonIsDistinctFrom.class)
.scalars(FailureFunction.class)
.scalars(JoniRegexpCasts.class)
.scalars(CharacterStringCasts.class)
.scalars(CharOperators.class)
.scalar(CharOperators.CharIsDistinctFrom.class)
.scalar(DecimalOperators.Negation.class)
.scalar(DecimalOperators.HashCode.class)
.scalar(DecimalOperators.Indeterminate.class)
Expand Down
43 changes: 25 additions & 18 deletions presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayDistinctFromOperator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
*/

import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.function.BlockIndex;
import com.facebook.presto.spi.function.BlockPosition;
import com.facebook.presto.spi.function.Convention;
import com.facebook.presto.spi.function.IsNull;
import com.facebook.presto.spi.function.OperatorDependency;
Expand All @@ -25,12 +27,10 @@

import java.lang.invoke.MethodHandle;

import static com.facebook.presto.spi.InvocationConvention.InvocationArgumentConvention.NULL_FLAG;
import static com.facebook.presto.spi.InvocationConvention.InvocationArgumentConvention.BLOCK_POSITION;
import static com.facebook.presto.spi.InvocationConvention.InvocationReturnConvention.FAIL_ON_NULL;
import static com.facebook.presto.spi.function.OperatorType.IS_DISTINCT_FROM;
import static com.facebook.presto.spi.type.TypeUtils.readNativeValue;
import static com.facebook.presto.util.Failures.internalError;
import static com.google.common.base.Defaults.defaultValue;

@ScalarOperator(IS_DISTINCT_FROM)
public final class ArrayDistinctFromOperator
Expand All @@ -44,7 +44,7 @@ public static boolean isDistinctFrom(
operator = IS_DISTINCT_FROM,
returnType = StandardTypes.BOOLEAN,
argumentTypes = {"E", "E"},
convention = @Convention(arguments = {NULL_FLAG, NULL_FLAG}, result = FAIL_ON_NULL)) MethodHandle function,
convention = @Convention(arguments = {BLOCK_POSITION, BLOCK_POSITION}, result = FAIL_ON_NULL)) MethodHandle function,
@TypeParameter("E") Type type,
@SqlType("array(E)") Block left,
@IsNull boolean leftNull,
Expand All @@ -61,22 +61,12 @@ public static boolean isDistinctFrom(
return true;
}
for (int i = 0; i < left.getPositionCount(); i++) {
Object leftValue = readNativeValue(type, left, i);
boolean leftValueNull = leftValue == null;
if (leftValueNull) {
leftValue = defaultValue(type.getJavaType());
}
Object rightValue = readNativeValue(type, right, i);
boolean rightValueNull = rightValue == null;
if (rightValueNull) {
rightValue = defaultValue(type.getJavaType());
}
try {
if ((boolean) function.invoke(
leftValue,
leftValueNull,
rightValue,
rightValueNull)) {
left,
i,
right,
i)) {
return true;
}
}
Expand All @@ -86,4 +76,21 @@ public static boolean isDistinctFrom(
}
return false;
}

@TypeParameter("E")
@SqlType(StandardTypes.BOOLEAN)
public static boolean isDistinctFrom(
@OperatorDependency(
operator = IS_DISTINCT_FROM,
returnType = StandardTypes.BOOLEAN,
argumentTypes = {"E", "E"},
convention = @Convention(arguments = {BLOCK_POSITION, BLOCK_POSITION}, result = FAIL_ON_NULL)) MethodHandle function,
@TypeParameter("array(E)") Type type,
@BlockPosition @SqlType(value = "array(E)", nativeContainerType = Block.class) Block left,
@BlockIndex int leftPosition,
@BlockPosition @SqlType(value = "array(E)", nativeContainerType = Block.class) Block right,
@BlockIndex int rightPosition)
{
return isDistinctFrom(function, type, (Block) type.getObject(left, leftPosition), left.isNull(leftPosition), (Block) type.getObject(right, rightPosition), right.isNull(rightPosition));
}
}
41 changes: 34 additions & 7 deletions presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonOperators.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@

import com.facebook.presto.spi.ConnectorSession;
import com.facebook.presto.spi.PrestoException;
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.function.BlockIndex;
import com.facebook.presto.spi.function.BlockPosition;
import com.facebook.presto.spi.function.IsNull;
import com.facebook.presto.spi.function.LiteralParameters;
import com.facebook.presto.spi.function.ScalarOperator;
Expand Down Expand Up @@ -389,15 +392,39 @@ public static boolean notEqual(@SqlType(JSON) Slice leftJson, @SqlType(JSON) Sli
}

@ScalarOperator(IS_DISTINCT_FROM)
@SqlType(BOOLEAN)
public static boolean isDistinctFrom(@SqlType(JSON) Slice leftJson, @IsNull boolean leftNull, @SqlType(JSON) Slice rightJson, @IsNull boolean rightNull)
public static class JsonIsDistinctFrom
{
if (leftNull != rightNull) {
return true;
@SqlType(BOOLEAN)
public static boolean isDistinctFrom(@SqlType(JSON) Slice leftJson, @IsNull boolean leftNull, @SqlType(JSON) Slice rightJson, @IsNull boolean rightNull)
{
if (leftNull != rightNull) {
return true;
}
if (leftNull) {
return false;
}
return notEqual(leftJson, rightJson);
}
if (leftNull) {
return false;

@SqlType(BOOLEAN)
public static boolean isDistinctFrom(
@BlockPosition @SqlType(value = JSON, nativeContainerType = Slice.class) Block left,
@BlockIndex int leftPosition,
@BlockPosition @SqlType(value = JSON, nativeContainerType = Slice.class) Block right,
@BlockIndex int rightPosition)
{
if (left.isNull(leftPosition) != right.isNull(rightPosition)) {
return true;
}
if (left.isNull(leftPosition)) {
return false;
}
int leftLength = left.getSliceLength(leftPosition);
int rightLength = right.getSliceLength(rightPosition);
if (leftLength != rightLength) {
return true;
}
return !left.equals(leftPosition, 0, right, rightPosition, 0, leftLength);
}
return notEqual(leftJson, rightJson);
}
}
44 changes: 29 additions & 15 deletions presto-main/src/main/java/com/facebook/presto/operator/scalar/MapDistinctFromOperator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,25 @@
*/

import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.function.BlockIndex;
import com.facebook.presto.spi.function.BlockPosition;
import com.facebook.presto.spi.function.Convention;
import com.facebook.presto.spi.function.IsNull;
import com.facebook.presto.spi.function.OperatorDependency;
import com.facebook.presto.spi.function.ScalarOperator;
import com.facebook.presto.spi.function.SqlType;
import com.facebook.presto.spi.function.TypeParameter;
import com.facebook.presto.spi.type.MapType;
import com.facebook.presto.spi.type.StandardTypes;
import com.facebook.presto.spi.type.Type;

import java.lang.invoke.MethodHandle;

import static com.facebook.presto.spi.InvocationConvention.InvocationArgumentConvention.BLOCK_POSITION;
import static com.facebook.presto.spi.InvocationConvention.InvocationReturnConvention.FAIL_ON_NULL;
import static com.facebook.presto.spi.function.OperatorType.EQUAL;
import static com.facebook.presto.spi.function.OperatorType.HASH_CODE;
import static com.facebook.presto.spi.function.OperatorType.IS_DISTINCT_FROM;
import static com.facebook.presto.spi.type.TypeUtils.readNativeValue;

@ScalarOperator(IS_DISTINCT_FROM)
public final class MapDistinctFromOperator
Expand All @@ -42,10 +47,9 @@ public static boolean isDistinctFrom(
MethodHandle keyEqualsFunction,
@OperatorDependency(operator = HASH_CODE, returnType = StandardTypes.BIGINT, argumentTypes = {"K"})
MethodHandle keyHashcodeFunction,
@OperatorDependency(operator = IS_DISTINCT_FROM, returnType = StandardTypes.BOOLEAN, argumentTypes = {"V", "V"})
@OperatorDependency(operator = IS_DISTINCT_FROM, returnType = StandardTypes.BOOLEAN, argumentTypes = {"V", "V"}, convention = @Convention(arguments = {BLOCK_POSITION, BLOCK_POSITION}, result = FAIL_ON_NULL))
MethodHandle valueDistinctFromFunction,
@TypeParameter("K") Type keyType,
@TypeParameter("V") Type valueType,
@TypeParameter("map(K, V)") Type mapType,
@SqlType("map(K,V)") Block leftMapBlock,
@IsNull boolean leftMapNull,
@SqlType("map(K,V)") Block rightMapBlock,
Expand All @@ -59,18 +63,28 @@ public static boolean isDistinctFrom(
}
// Note that we compare to NOT distinct here and so negate the result.
return !MapGenericEquality.genericEqual(
keyType,
((MapType) mapType).getKeyType(),
leftMapBlock,
rightMapBlock,
(leftMapIndex, rightMapIndex) -> {
Object leftValue = readNativeValue(valueType, leftMapBlock, leftMapIndex);
Object rightValue = readNativeValue(valueType, rightMapBlock, rightMapIndex);
boolean leftNull = leftValue == null;
boolean rightNull = rightValue == null;
if (leftNull || rightNull) {
return leftNull == rightNull;
}
return !(boolean) valueDistinctFromFunction.invoke(leftValue, leftNull, rightValue, rightNull);
});
(leftMapIndex, rightMapIndex) -> !(boolean) valueDistinctFromFunction.invoke(leftMapBlock, leftMapIndex, rightMapBlock, rightMapIndex));
}

@TypeParameter("K")
@TypeParameter("V")
@SqlType(StandardTypes.BOOLEAN)
public static boolean isDistinctFrom(
@OperatorDependency(operator = EQUAL, returnType = StandardTypes.BOOLEAN, argumentTypes = {"K", "K"})
MethodHandle keyEqualsFunction,
@OperatorDependency(operator = HASH_CODE, returnType = StandardTypes.BIGINT, argumentTypes = {"K"})
MethodHandle keyHashcodeFunction,
@OperatorDependency(operator = IS_DISTINCT_FROM, returnType = StandardTypes.BOOLEAN, argumentTypes = {"V", "V"}, convention = @Convention(arguments = {BLOCK_POSITION, BLOCK_POSITION}, result = FAIL_ON_NULL))
MethodHandle valueDistinctFromFunction,
@TypeParameter("map(K, V)") Type mapType,
@BlockPosition @SqlType(value = "map(K,V)", nativeContainerType = Block.class) Block left,
@BlockIndex int leftPosition,
@BlockPosition @SqlType(value = "map(K,V)", nativeContainerType = Block.class) Block right,
@BlockIndex int rightPosition)
{
return isDistinctFrom(keyEqualsFunction, keyHashcodeFunction, valueDistinctFromFunction, mapType, (Block) mapType.getObject(left, leftPosition), left.isNull(leftPosition), (Block) mapType.getObject(right, rightPosition), right.isNull(rightPosition));
}
}
22 changes: 18 additions & 4 deletions presto-main/src/main/java/com/facebook/presto/operator/scalar/RowDistinctFromOperator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.facebook.presto.metadata.FunctionRegistry;
import com.facebook.presto.metadata.Signature;
import com.facebook.presto.metadata.SqlOperator;
import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ScalarImplementationChoice;
import com.facebook.presto.spi.InvocationConvention;
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.type.StandardTypes;
Expand All @@ -31,6 +32,7 @@

import static com.facebook.presto.metadata.Signature.comparableWithVariadicBound;
import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty;
import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION;
import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_NULL_FLAG;
import static com.facebook.presto.spi.InvocationConvention.InvocationArgumentConvention.NULL_FLAG;
import static com.facebook.presto.spi.function.OperatorType.IS_DISTINCT_FROM;
Expand All @@ -45,6 +47,7 @@ public class RowDistinctFromOperator
{
public static final RowDistinctFromOperator ROW_DISTINCT_FROM = new RowDistinctFromOperator();
private static final MethodHandle METHOD_HANDLE = methodHandle(RowDistinctFromOperator.class, "isDistinctFrom", Type.class, List.class, Block.class, boolean.class, Block.class, boolean.class);
private static final MethodHandle METHOD_HANDLE_BLOCK_POSITION = methodHandle(RowDistinctFromOperator.class, "isDistinctFrom", Type.class, List.class, Block.class, int.class, Block.class, int.class);

private RowDistinctFromOperator()
{
Expand All @@ -71,11 +74,17 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in
argumentMethods.add(functionInvoker.methodHandle());
}
return new ScalarFunctionImplementation(
false,
ImmutableList.of(
valueTypeArgumentProperty(USE_NULL_FLAG),
valueTypeArgumentProperty(USE_NULL_FLAG)),
METHOD_HANDLE.bindTo(type).bindTo(argumentMethods.build()),
new ScalarImplementationChoice(
false,
ImmutableList.of(valueTypeArgumentProperty(USE_NULL_FLAG), valueTypeArgumentProperty(USE_NULL_FLAG)),
METHOD_HANDLE.bindTo(type).bindTo(argumentMethods.build()),
Optional.empty()),
new ScalarImplementationChoice(
false,
ImmutableList.of(valueTypeArgumentProperty(BLOCK_AND_POSITION), valueTypeArgumentProperty(BLOCK_AND_POSITION)),
METHOD_HANDLE_BLOCK_POSITION.bindTo(type).bindTo(argumentMethods.build()),
Optional.empty())),
isDeterministic());
}

Expand Down Expand Up @@ -115,4 +124,9 @@ public static boolean isDistinctFrom(Type rowType, List<MethodHandle> argumentMe
}
return false;
}

public static boolean isDistinctFrom(Type rowType, List<MethodHandle> argumentMethods, Block leftRow, int leftPosition, Block rightRow, int rightPosition)
{
return isDistinctFrom(rowType, argumentMethods, (Block) rowType.getObject(leftRow, leftPosition), leftRow.isNull(leftPosition), (Block) rowType.getObject(rightRow, rightPosition), rightRow.isNull(rightPosition));
}
}
Loading