From 8ddb9fcb0ac514a77095994b03652c78580cc8bf Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Mon, 27 Jun 2022 17:14:49 -0700 Subject: [PATCH 01/19] WIP: Plugin implementation of array_intersect using setIntersect, sort still needed Signed-off-by: Navin Kumar --- .../src/main/python/array_test.py | 22 ++++++++ .../nvidia/spark/rapids/GpuOverrides.scala | 19 ++++++- .../sql/rapids/collectionOperations.scala | 52 +++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index 2a766439936..21e3d2f23ee 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -394,3 +394,25 @@ def test_array_max_q1(): def q1(spark): return spark.sql('SELECT ARRAY_MAX(TRANSFORM(ARRAY_REPEAT(STRUCT(1, 2), 0), s -> s.col2))') assert_gpu_and_cpu_are_equal_collect(q1) + + +@pytest.mark.parametrize('data_gen', all_basic_gens_no_null, ids=idfn) +def test_array_intersect(data_gen): + gen = StructGen( + [('a', ArrayGen(data_gen, nullable=False)), + ('b', ArrayGen(data_gen, nullable=False))], + nullable=False) + literal = gen_scalar(data_gen, force_no_nulls=True) + + assert_gpu_and_cpu_are_equal_collect( + lambda spark: gen_df(spark, gen).selectExpr( + # 'array_intersect(a, b)', + # 'array_intersect(a, array())', + # 'array_intersect(array(), b)', + # 'array_intersect(a, null)', + # 'array_intersect(null, b)', + 'array_intersect(a, a)', + # 'array_intersect(a, array({}))', + ) + ) + \ No newline at end of file diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index ae82390baeb..617295c78f7 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -2907,7 +2907,24 @@ object GpuOverrides extends Logging { } } ), - + expr[ArrayIntersect]( + "Returns an array of the elements in the intersection of array1 and array2, without" + + " duplicates", + ExprChecks.binaryProject( + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), + TypeSig.ARRAY.nested(TypeSig.all), + ("array1", + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), + TypeSig.ARRAY.nested(TypeSig.all)), + ("array2", + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), + TypeSig.ARRAY.nested(TypeSig.all))), + (in, conf, p, r) => new BinaryExprMeta[ArrayIntersect](in, conf, p, r) { + override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = { + GpuArrayIntersect(lhs, rhs) + } + } + ), expr[TransformKeys]( "Transform keys in a map using a transform function", ExprChecks.projectOnly(TypeSig.MAP.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index 7521103835e..bbe4609db9a 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -695,6 +695,58 @@ case class GpuArraysZip(children: Seq[Expression]) extends GpuExpression with Sh } } +case class GpuArrayIntersect(left: Expression, right: Expression) + extends GpuBinaryExpression with ExpectsInputTypes { + + override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, ArrayType) + + override def checkInputDataTypes(): TypeCheckResult = + (left.dataType, right.dataType) match { + case (ArrayType(ldt, _), ArrayType(rdt, _)) => + if (ldt.sameType(rdt)) { + TypeCheckResult.TypeCheckSuccess + } else { + TypeCheckResult.TypeCheckFailure( + s"Array_intersect requires both array params to have the same subType: $ldt != $rdt") + } + case dt => + TypeCheckResult.TypeCheckFailure(s"$prettyName only supports array input, but found $dt") + } + + override def dataType: DataType = left.dataType + + override def nullable: Boolean = false + + + override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { + // Unlike set intersection, array intersection must preserve the order the items + // from both arrays + withResource(ColumnView.setIntersect(lhs.getBase, rhs.getBase)) { setResult => + setResult + } + } + + override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { + withResource(GpuColumnVector.from(lhs, rhs.getRowCount.toInt, lhs.dataType)) { left => + doColumnar(left, rhs) + } + } + + override def doColumnar(lhs: GpuColumnVector, rhs: GpuScalar): ColumnVector = { + withResource(GpuColumnVector.from(rhs, lhs.getRowCount.toInt, rhs.dataType)) { right => + doColumnar(lhs, right) + } + } + + override def doColumnar(numRows: Int, lhs: GpuScalar, rhs: GpuScalar): ColumnVector = { + withResource(GpuColumnVector.from(lhs, numRows, lhs.dataType)) { left => + withResource(GpuColumnVector.from(rhs, numRows, rhs.dataType)) { right => + doColumnar(left, right) + } + } + } +} + class GpuSequenceMeta( expr: Sequence, conf: RapidsConf, From 0367a78f969d5b2c2f204a876cb31d6e9193d3d0 Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Wed, 29 Jun 2022 14:52:38 -0700 Subject: [PATCH 02/19] WIP: array_intersect Signed-off-by: Navin Kumar --- .../spark/sql/rapids/collectionOperations.scala | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index bbe4609db9a..a89743b0d83 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -717,13 +717,18 @@ case class GpuArrayIntersect(left: Expression, right: Expression) override def nullable: Boolean = false + private def arrayIntersect(array1: ColumnVector, array2: ColumnVector): ColumnVector = { + withResource(ColumnView.setIntersect(array1, array2)) { setResult => + withResource(array1.listIndexOf(setResult, ColumnView.FindOptions.FIND_FIRST)) { indices => + array1.extractListElement(indices) + } + } + } override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { // Unlike set intersection, array intersection must preserve the order the items // from both arrays - withResource(ColumnView.setIntersect(lhs.getBase, rhs.getBase)) { setResult => - setResult - } + arrayIntersect(lhs.getBase, rhs.getBase) } override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { From 7743f45d4815b96972fba2875cd9a0eae9988c4d Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Thu, 30 Jun 2022 15:35:18 -0700 Subject: [PATCH 03/19] array_intersect implementation Signed-off-by: Navin Kumar --- .../src/main/python/array_test.py | 33 ++++++++--- .../sql/rapids/collectionOperations.scala | 57 ++++++++----------- 2 files changed, 49 insertions(+), 41 deletions(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index 21e3d2f23ee..fe592d148fb 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -396,7 +396,8 @@ def q1(spark): assert_gpu_and_cpu_are_equal_collect(q1) -@pytest.mark.parametrize('data_gen', all_basic_gens_no_null, ids=idfn) +@pytest.mark.parametrize('data_gen', [byte_gen, short_gen, int_gen, long_gen, + FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) def test_array_intersect(data_gen): gen = StructGen( [('a', ArrayGen(data_gen, nullable=False)), @@ -406,13 +407,27 @@ def test_array_intersect(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark: gen_df(spark, gen).selectExpr( - # 'array_intersect(a, b)', - # 'array_intersect(a, array())', - # 'array_intersect(array(), b)', - # 'array_intersect(a, null)', - # 'array_intersect(null, b)', - 'array_intersect(a, a)', - # 'array_intersect(a, array({}))', + 'sort_array(array_intersect(a, b))', + 'sort_array(array_intersect(a, b))', + 'array_intersect(a, array())', + 'array_intersect(array(), b)', + 'sort_array(array_intersect(a, a))', ) ) - \ No newline at end of file + +def test_array_union(data_gen): + gen = StructGen( + [('a', ArrayGen(data_gen, nullable=False)), + ('b', ArrayGen(data_gen, nullable=False))], + nullable=False) + literal = gen_scalar(data_gen, force_no_nulls=True) + + assert_gpu_and_cpu_are_equal_collect( + lambda spark: gen_df(spark, gen).selectExpr( + 'sort_array(array_union(a, b))', + 'sort_array(array_union(a, b))', + 'array_union(a, array())', + 'array_union(array(), b)', + 'sort_array(array_union(a, a))', + ) + ) \ No newline at end of file diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index a89743b0d83..384056f73ff 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -695,8 +695,31 @@ case class GpuArraysZip(children: Seq[Expression]) extends GpuExpression with Sh } } +trait GpuArraySetOperation extends GpuBinaryExpression { + + override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { + withResource(GpuColumnVector.from(lhs, rhs.getRowCount.toInt, lhs.dataType)) { left => + doColumnar(left, rhs) + } + } + + override def doColumnar(lhs: GpuColumnVector, rhs: GpuScalar): ColumnVector = { + withResource(GpuColumnVector.from(rhs, lhs.getRowCount.toInt, rhs.dataType)) { right => + doColumnar(lhs, right) + } + } + + override def doColumnar(numRows: Int, lhs: GpuScalar, rhs: GpuScalar): ColumnVector = { + withResource(GpuColumnVector.from(lhs, numRows, lhs.dataType)) { left => + withResource(GpuColumnVector.from(rhs, numRows, rhs.dataType)) { right => + doColumnar(left, right) + } + } + } +} + case class GpuArrayIntersect(left: Expression, right: Expression) - extends GpuBinaryExpression with ExpectsInputTypes { + extends GpuArraySetOperation with ExpectsInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, ArrayType) @@ -717,38 +740,8 @@ case class GpuArrayIntersect(left: Expression, right: Expression) override def nullable: Boolean = false - private def arrayIntersect(array1: ColumnVector, array2: ColumnVector): ColumnVector = { - withResource(ColumnView.setIntersect(array1, array2)) { setResult => - withResource(array1.listIndexOf(setResult, ColumnView.FindOptions.FIND_FIRST)) { indices => - array1.extractListElement(indices) - } - } - } - override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { - // Unlike set intersection, array intersection must preserve the order the items - // from both arrays - arrayIntersect(lhs.getBase, rhs.getBase) - } - - override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { - withResource(GpuColumnVector.from(lhs, rhs.getRowCount.toInt, lhs.dataType)) { left => - doColumnar(left, rhs) - } - } - - override def doColumnar(lhs: GpuColumnVector, rhs: GpuScalar): ColumnVector = { - withResource(GpuColumnVector.from(rhs, lhs.getRowCount.toInt, rhs.dataType)) { right => - doColumnar(lhs, right) - } - } - - override def doColumnar(numRows: Int, lhs: GpuScalar, rhs: GpuScalar): ColumnVector = { - withResource(GpuColumnVector.from(lhs, numRows, lhs.dataType)) { left => - withResource(GpuColumnVector.from(rhs, numRows, rhs.dataType)) { right => - doColumnar(left, right) - } - } + ColumnView.setIntersect(lhs.getBase, rhs.getBase) } } From 64946b28e7e2ade378021aa23b045ecf88af9c20 Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Mon, 4 Jul 2022 15:10:20 -0700 Subject: [PATCH 04/19] implementation of 3 other array set operations Signed-off-by: Navin Kumar --- .../nvidia/spark/rapids/GpuOverrides.scala | 53 ++++++++++++ .../sql/rapids/collectionOperations.scala | 85 ++++++++++++++++++- 2 files changed, 136 insertions(+), 2 deletions(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 617295c78f7..671224fa82e 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -2907,6 +2907,23 @@ object GpuOverrides extends Logging { } } ), + expr[ArrayExcept]( + "Returns an array of the elements in array1 but not in array2, without duplicates", + ExprChecks.binaryProject( + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), + TypeSig.ARRAY.nested(TypeSig.all), + ("array1", + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), + TypeSig.ARRAY.nested(TypeSig.all)), + ("array2", + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), + TypeSig.ARRAY.nested(TypeSig.all))), + (in, conf, p, r) => new BinaryExprMeta[ArrayExcept](in, conf, p, r) { + override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = { + GpuArrayExcept(lhs, rhs) + } + } + ), expr[ArrayIntersect]( "Returns an array of the elements in the intersection of array1 and array2, without" + " duplicates", @@ -2925,6 +2942,42 @@ object GpuOverrides extends Logging { } } ), + expr[ArrayUnion]( + "Returns an array of the elements in the union of array1 and array2, without duplicates.", + ExprChecks.binaryProject( + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), + TypeSig.ARRAY.nested(TypeSig.all), + ("array1", + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), + TypeSig.ARRAY.nested(TypeSig.all)), + ("array2", + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), + TypeSig.ARRAY.nested(TypeSig.all))), + (in, conf, p, r) => new BinaryExprMeta[ArrayUnion](in, conf, p, r) { + override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = { + GpuArrayUnion(lhs, rhs) + } + } + ), + expr[ArraysOverlap]( + "Returns true if a1 contains at least a non-null element present also in a2. If the arrays " + + "have no common element and they are both non-empty and either of them contains a null " + + "element null is returned, false otherwise.", + ExprChecks.binaryProject( + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), + TypeSig.ARRAY.nested(TypeSig.all), + ("array1", + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), + TypeSig.ARRAY.nested(TypeSig.all)), + ("array2", + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), + TypeSig.ARRAY.nested(TypeSig.all))), + (in, conf, p, r) => new BinaryExprMeta[ArraysOverlap](in, conf, p, r) { + override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = { + GpuArraysOverlap(lhs, rhs) + } + } + ), expr[TransformKeys]( "Transform keys in a map using a transform function", ExprChecks.projectOnly(TypeSig.MAP.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index 384056f73ff..379128413f5 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -718,17 +718,44 @@ trait GpuArraySetOperation extends GpuBinaryExpression { } } +case class GpuArrayExcept(left: Expression, right: Expression) + extends GpuArraySetOperation with ExpectsInputTypes { + + override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, ArrayType) + + override def checkInputDataTypes(): TypeCheckResult = + (left.dataType, right.dataType) match { + case (ArrayType(ldt, _), ArrayType(rdt, _)) => + if (ldt.sameType(rdt)) { + TypeCheckResult.TypeCheckSuccess + } else { + TypeCheckResult.TypeCheckFailure( + s"Array_intersect requires both array params to have the same subType: $ldt != $rdt") + } + case dt => + TypeCheckResult.TypeCheckFailure(s"$prettyName only supports array input, but found $dt") + } + + override def dataType: DataType = left.dataType + + override def nullable: Boolean = false + + override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { + ColumnView.setDifference(lhs.getBase, rhs.getBase) + } +} + case class GpuArrayIntersect(left: Expression, right: Expression) extends GpuArraySetOperation with ExpectsInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, ArrayType) - override def checkInputDataTypes(): TypeCheckResult = + override def checkInputDataTypes(): TypeCheckResult = (left.dataType, right.dataType) match { case (ArrayType(ldt, _), ArrayType(rdt, _)) => if (ldt.sameType(rdt)) { TypeCheckResult.TypeCheckSuccess - } else { + } else { TypeCheckResult.TypeCheckFailure( s"Array_intersect requires both array params to have the same subType: $ldt != $rdt") } @@ -745,6 +772,60 @@ case class GpuArrayIntersect(left: Expression, right: Expression) } } +case class GpuArrayUnion(left: Expression, right: Expression) + extends GpuArraySetOperation with ExpectsInputTypes { + + override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, ArrayType) + + override def checkInputDataTypes(): TypeCheckResult = + (left.dataType, right.dataType) match { + case (ArrayType(ldt, _), ArrayType(rdt, _)) => + if (ldt.sameType(rdt)) { + TypeCheckResult.TypeCheckSuccess + } else { + TypeCheckResult.TypeCheckFailure( + s"Array_union requires both array params to have the same subType: $ldt != $rdt") + } + case dt => + TypeCheckResult.TypeCheckFailure(s"$prettyName only supports array input, but found $dt") + } + + override def dataType: DataType = left.dataType + + override def nullable: Boolean = false + + override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { + ColumnView.setUnion(lhs.getBase, rhs.getBase) + } +} + +case class GpuArraysOverlap(left: Expression, right: Expression) + extends GpuArraySetOperation with ExpectsInputTypes { + + override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, ArrayType) + + override def checkInputDataTypes(): TypeCheckResult = + (left.dataType, right.dataType) match { + case (ArrayType(ldt, _), ArrayType(rdt, _)) => + if (ldt.sameType(rdt)) { + TypeCheckResult.TypeCheckSuccess + } else { + TypeCheckResult.TypeCheckFailure( + s"Array_union requires both array params to have the same subType: $ldt != $rdt") + } + case dt => + TypeCheckResult.TypeCheckFailure(s"$prettyName only supports array input, but found $dt") + } + + override def dataType: DataType = left.dataType + + override def nullable: Boolean = false + + override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { + ColumnView.setOverlap(lhs.getBase, rhs.getBase) + } +} + class GpuSequenceMeta( expr: Sequence, conf: RapidsConf, From ef54c68778f85e9c10bdedb8a2d76a97d9be0f91 Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Mon, 4 Jul 2022 22:33:22 -0700 Subject: [PATCH 05/19] add integration tests Signed-off-by: Navin Kumar --- .../src/main/python/array_test.py | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index fe592d148fb..d5242f92545 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -403,7 +403,6 @@ def test_array_intersect(data_gen): [('a', ArrayGen(data_gen, nullable=False)), ('b', ArrayGen(data_gen, nullable=False))], nullable=False) - literal = gen_scalar(data_gen, force_no_nulls=True) assert_gpu_and_cpu_are_equal_collect( lambda spark: gen_df(spark, gen).selectExpr( @@ -415,12 +414,13 @@ def test_array_intersect(data_gen): ) ) +@pytest.mark.parametrize('data_gen', [byte_gen, short_gen, int_gen, long_gen, + FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) def test_array_union(data_gen): gen = StructGen( [('a', ArrayGen(data_gen, nullable=False)), ('b', ArrayGen(data_gen, nullable=False))], nullable=False) - literal = gen_scalar(data_gen, force_no_nulls=True) assert_gpu_and_cpu_are_equal_collect( lambda spark: gen_df(spark, gen).selectExpr( @@ -430,4 +430,40 @@ def test_array_union(data_gen): 'array_union(array(), b)', 'sort_array(array_union(a, a))', ) + ) + +@pytest.mark.parametrize('data_gen', [byte_gen, short_gen, int_gen, long_gen, + FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) +def test_array_difference(data_gen): + gen = StructGen( + [('a', ArrayGen(data_gen, nullable=False)), + ('b', ArrayGen(data_gen, nullable=False))], + nullable=False) + + assert_gpu_and_cpu_are_equal_collect( + lambda spark: gen_df(spark, gen).selectExpr( + 'sort_array(array_difference(a, b))', + 'sort_array(array_difference(a, b))', + 'array_difference(a, array())', + 'array_difference(array(), b)', + 'sort_array(array_difference(a, a))', + ) + ) + +@pytest.mark.parametrize('data_gen', [byte_gen, short_gen, int_gen, long_gen, + FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) +def test_arrays_overlap(data_gen): + gen = StructGen( + [('a', ArrayGen(data_gen, nullable=False)), + ('b', ArrayGen(data_gen, nullable=False))], + nullable=False) + + assert_gpu_and_cpu_are_equal_collect( + lambda spark: gen_df(spark, gen).selectExpr( + 'arrays_overlap(a, b)', + 'arrays_overlap(a, b)', + 'arrays_overlap(a, array())', + 'arrays_overlap(array(), b)', + 'arrays_overlap(a, a)', + ) ) \ No newline at end of file From 3362c084d011929f2cb007287e6d13fd4bb0a7c6 Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Tue, 5 Jul 2022 15:10:50 -0700 Subject: [PATCH 06/19] working versions of array_difference and arrays_overlap Signed-off-by: Navin Kumar --- .../src/main/python/array_test.py | 34 ++++++++-------- .../nvidia/spark/rapids/GpuOverrides.scala | 4 +- .../sql/rapids/collectionOperations.scala | 39 +++++++++++++++++-- 3 files changed, 55 insertions(+), 22 deletions(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index d5242f92545..2c88c6d1e6c 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -407,9 +407,9 @@ def test_array_intersect(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark: gen_df(spark, gen).selectExpr( 'sort_array(array_intersect(a, b))', - 'sort_array(array_intersect(a, b))', - 'array_intersect(a, array())', - 'array_intersect(array(), b)', + 'sort_array(array_intersect(b, a))', + 'sort_array(array_intersect(a, array()))', + 'sort_array(array_intersect(array(), b))', 'sort_array(array_intersect(a, a))', ) ) @@ -425,16 +425,16 @@ def test_array_union(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark: gen_df(spark, gen).selectExpr( 'sort_array(array_union(a, b))', - 'sort_array(array_union(a, b))', - 'array_union(a, array())', - 'array_union(array(), b)', + 'sort_array(array_union(b, a))', + 'sort_array(array_union(a, array()))', + 'sort_array(array_union(array(), b))', 'sort_array(array_union(a, a))', ) ) @pytest.mark.parametrize('data_gen', [byte_gen, short_gen, int_gen, long_gen, FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) -def test_array_difference(data_gen): +def test_array_except(data_gen): gen = StructGen( [('a', ArrayGen(data_gen, nullable=False)), ('b', ArrayGen(data_gen, nullable=False))], @@ -442,11 +442,11 @@ def test_array_difference(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark: gen_df(spark, gen).selectExpr( - 'sort_array(array_difference(a, b))', - 'sort_array(array_difference(a, b))', - 'array_difference(a, array())', - 'array_difference(array(), b)', - 'sort_array(array_difference(a, a))', + 'sort_array(array_except(a, b))', + 'sort_array(array_except(b, a))', + 'sort_array(array_except(a, array()))', + 'sort_array(array_except(array(), b))', + 'sort_array(array_except(a, a))', ) ) @@ -460,10 +460,12 @@ def test_arrays_overlap(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark: gen_df(spark, gen).selectExpr( + 'a', + 'b', 'arrays_overlap(a, b)', - 'arrays_overlap(a, b)', - 'arrays_overlap(a, array())', - 'arrays_overlap(array(), b)', - 'arrays_overlap(a, a)', + # 'arrays_overlap(b, a)', + # 'arrays_overlap(a, array())', + # 'arrays_overlap(array(), b)', + # 'arrays_overlap(a, a)', ) ) \ No newline at end of file diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 671224fa82e..92f02a04d36 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -2963,9 +2963,7 @@ object GpuOverrides extends Logging { "Returns true if a1 contains at least a non-null element present also in a2. If the arrays " + "have no common element and they are both non-empty and either of them contains a null " + "element null is returned, false otherwise.", - ExprChecks.binaryProject( - TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), - TypeSig.ARRAY.nested(TypeSig.all), + ExprChecks.binaryProject(TypeSig.BOOLEAN, TypeSig.BOOLEAN, ("array1", TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL), TypeSig.ARRAY.nested(TypeSig.all)), diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index 379128413f5..74348bb81ea 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -817,12 +817,45 @@ case class GpuArraysOverlap(left: Expression, right: Expression) TypeCheckResult.TypeCheckFailure(s"$prettyName only supports array input, but found $dt") } - override def dataType: DataType = left.dataType + override def dataType: DataType = BooleanType - override def nullable: Boolean = false + override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { - ColumnView.setOverlap(lhs.getBase, rhs.getBase) + // Spark's arrays_overlap() function will return null if either array is null, so + // we check if either array is null + val bothNonEmpty = withResource(lhs.getBase.countElements) { leftCount => + withResource(rhs.getBase.countElements) { rightCount => + withResource(Scalar.fromInt(0)) { zero => + withResource(leftCount.greaterThan(zero)) { leftNonEmpty => + withResource(rightCount.greaterThan(zero)) { rightNonEmpty => + leftNonEmpty.and(rightNonEmpty) + } + } + } + } + } + val eitherHasNulls = withResource(lhs.getBase.listContainsNulls) { leftHasNulls => + withResource(rhs.getBase.listContainsNulls) { rightHasNulls => + leftHasNulls.or(rightHasNulls) + } + } + withResource(bothNonEmpty) { _ => + withResource(eitherHasNulls) { _ => + withResource(ColumnView.listOverlap(lhs.getBase, rhs.getBase)) { overlaps => + withResource(overlaps.not) { notOverlap => + withResource(bothNonEmpty.and(eitherHasNulls)) { nonEmptyAndHasNull => + withResource(nonEmptyAndHasNull.and(notOverlap)) { returnNull => + returnNull.ifElse( + GpuColumnVector.columnVectorFromNull(lhs.getRowCount.toInt, BooleanType), + overlaps + ) + } + } + } + } + } + } } } From 728f15a7a34df5156e9eb74f883e558786609343 Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Tue, 5 Jul 2022 16:08:05 -0700 Subject: [PATCH 07/19] array_union passing tests but for one strange bug in GPU partitioning Signed-off-by: Navin Kumar --- integration_tests/src/main/python/array_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index 2c88c6d1e6c..4977f80effd 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -427,7 +427,7 @@ def test_array_union(data_gen): 'sort_array(array_union(a, b))', 'sort_array(array_union(b, a))', 'sort_array(array_union(a, array()))', - 'sort_array(array_union(array(), b))', + # 'sort_array(array_union(array(), b))', 'sort_array(array_union(a, a))', ) ) From 3bfe106e917327170f34b9b88857d9613c07921e Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Tue, 5 Jul 2022 16:19:56 -0700 Subject: [PATCH 08/19] cleanup arrays_overlap integration test Signed-off-by: Navin Kumar --- integration_tests/src/main/python/array_test.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index 4977f80effd..34ddfaeee2f 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -460,12 +460,10 @@ def test_arrays_overlap(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark: gen_df(spark, gen).selectExpr( - 'a', - 'b', 'arrays_overlap(a, b)', - # 'arrays_overlap(b, a)', - # 'arrays_overlap(a, array())', - # 'arrays_overlap(array(), b)', - # 'arrays_overlap(a, a)', + 'arrays_overlap(b, a)', + 'arrays_overlap(a, array())', + 'arrays_overlap(array(), b)', + 'arrays_overlap(a, a)', ) ) \ No newline at end of file From c8e21b68ebe18aff69bb873b83ab44bf25064c94 Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Wed, 6 Jul 2022 11:49:35 -0700 Subject: [PATCH 09/19] Add reference to issue filed regarding partitioning and collect() Signed-off-by: Navin Kumar --- integration_tests/src/main/python/array_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index 34ddfaeee2f..10ed0379e26 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -422,12 +422,16 @@ def test_array_union(data_gen): ('b', ArrayGen(data_gen, nullable=False))], nullable=False) + # The 4th item in this integration test here is left commmented out here + # There is an issue with running that item with collect(), which affects + # the integration test here. + # See this issue (https://github.com/NVIDIA/spark-rapids/issues/5957) assert_gpu_and_cpu_are_equal_collect( lambda spark: gen_df(spark, gen).selectExpr( 'sort_array(array_union(a, b))', 'sort_array(array_union(b, a))', 'sort_array(array_union(a, array()))', - # 'sort_array(array_union(array(), b))', + # 'sort_array(array_union(array(), b))', # see https://github.com/NVIDIA/spark-rapids/issues/5957 'sort_array(array_union(a, a))', ) ) From fab15a1ce98216f7a8ff7e75971034e8c1d8acd7 Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Wed, 6 Jul 2022 13:23:25 -0700 Subject: [PATCH 10/19] Add clarifying comment for GpuArraysOverlap Signed-off-by: Navin Kumar --- .../org/apache/spark/sql/rapids/collectionOperations.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index a429579f481..1499283c778 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -806,8 +806,9 @@ case class GpuArraysOverlap(left: Expression, right: Expression) override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { - // Spark's arrays_overlap() function will return null if either array is null, so - // we check if either array is null + // Spark's arrays_overlap() function will return null when both arrays are non-empty, + // they have no-non null overlapping elements and either array contains at least one null + // value, so we have to handle that here. val bothNonEmpty = withResource(lhs.getBase.countElements) { leftCount => withResource(rhs.getBase.countElements) { rightCount => withResource(Scalar.fromInt(0)) { zero => From f1003c43b95a4ac82cceef62eb6fd30fca969633 Mon Sep 17 00:00:00 2001 From: NVnavkumar <97137715+NVnavkumar@users.noreply.github.com> Date: Wed, 6 Jul 2022 16:32:37 -0700 Subject: [PATCH 11/19] Update integration_tests/src/main/python/array_test.py fix typo Co-authored-by: Nghia Truong --- integration_tests/src/main/python/array_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index 10ed0379e26..0cc1466de27 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -422,7 +422,7 @@ def test_array_union(data_gen): ('b', ArrayGen(data_gen, nullable=False))], nullable=False) - # The 4th item in this integration test here is left commmented out here + # The 4th item in this integration test here is left commented out here # There is an issue with running that item with collect(), which affects # the integration test here. # See this issue (https://github.com/NVIDIA/spark-rapids/issues/5957) From eb5e6f4c9788e8fc666813683a6fce2f57854f25 Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Thu, 7 Jul 2022 12:57:36 -0700 Subject: [PATCH 12/19] make generated arrays nullable in integration tests Signed-off-by: Navin Kumar --- integration_tests/src/main/python/array_test.py | 16 ++++++++-------- .../spark/sql/rapids/collectionOperations.scala | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index 0cc1466de27..924866f0ca5 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -400,8 +400,8 @@ def q1(spark): FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) def test_array_intersect(data_gen): gen = StructGen( - [('a', ArrayGen(data_gen, nullable=False)), - ('b', ArrayGen(data_gen, nullable=False))], + [('a', ArrayGen(data_gen, nullable=True)), + ('b', ArrayGen(data_gen, nullable=True))], nullable=False) assert_gpu_and_cpu_are_equal_collect( @@ -418,8 +418,8 @@ def test_array_intersect(data_gen): FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) def test_array_union(data_gen): gen = StructGen( - [('a', ArrayGen(data_gen, nullable=False)), - ('b', ArrayGen(data_gen, nullable=False))], + [('a', ArrayGen(data_gen, nullable=True)), + ('b', ArrayGen(data_gen, nullable=True))], nullable=False) # The 4th item in this integration test here is left commented out here @@ -440,8 +440,8 @@ def test_array_union(data_gen): FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) def test_array_except(data_gen): gen = StructGen( - [('a', ArrayGen(data_gen, nullable=False)), - ('b', ArrayGen(data_gen, nullable=False))], + [('a', ArrayGen(data_gen, nullable=True)), + ('b', ArrayGen(data_gen, nullable=True))], nullable=False) assert_gpu_and_cpu_are_equal_collect( @@ -458,8 +458,8 @@ def test_array_except(data_gen): FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) def test_arrays_overlap(data_gen): gen = StructGen( - [('a', ArrayGen(data_gen, nullable=False)), - ('b', ArrayGen(data_gen, nullable=False))], + [('a', ArrayGen(data_gen, nullable=True)), + ('b', ArrayGen(data_gen, nullable=True))], nullable=False) assert_gpu_and_cpu_are_equal_collect( diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index 1499283c778..c106376e380 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -722,7 +722,7 @@ case class GpuArrayExcept(left: Expression, right: Expression) override def dataType: DataType = left.dataType - override def nullable: Boolean = false + override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { ColumnView.setDifference(lhs.getBase, rhs.getBase) @@ -749,7 +749,7 @@ case class GpuArrayIntersect(left: Expression, right: Expression) override def dataType: DataType = left.dataType - override def nullable: Boolean = false + override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { ColumnView.setIntersect(lhs.getBase, rhs.getBase) @@ -776,7 +776,7 @@ case class GpuArrayUnion(left: Expression, right: Expression) override def dataType: DataType = left.dataType - override def nullable: Boolean = false + override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { ColumnView.setUnion(lhs.getBase, rhs.getBase) From 06af5ebd73a1cd7450ee4391b8fb12ea415abad4 Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Thu, 7 Jul 2022 17:02:24 -0700 Subject: [PATCH 13/19] remove the plugin code for null return value handling since this is now down in cudf Signed-off-by: Navin Kumar --- .../sql/rapids/collectionOperations.scala | 36 +------------------ 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index c106376e380..8878cabe0f0 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -806,41 +806,7 @@ case class GpuArraysOverlap(left: Expression, right: Expression) override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { - // Spark's arrays_overlap() function will return null when both arrays are non-empty, - // they have no-non null overlapping elements and either array contains at least one null - // value, so we have to handle that here. - val bothNonEmpty = withResource(lhs.getBase.countElements) { leftCount => - withResource(rhs.getBase.countElements) { rightCount => - withResource(Scalar.fromInt(0)) { zero => - withResource(leftCount.greaterThan(zero)) { leftNonEmpty => - withResource(rightCount.greaterThan(zero)) { rightNonEmpty => - leftNonEmpty.and(rightNonEmpty) - } - } - } - } - } - val eitherHasNulls = withResource(lhs.getBase.listContainsNulls) { leftHasNulls => - withResource(rhs.getBase.listContainsNulls) { rightHasNulls => - leftHasNulls.or(rightHasNulls) - } - } - withResource(bothNonEmpty) { _ => - withResource(eitherHasNulls) { _ => - withResource(ColumnView.listOverlap(lhs.getBase, rhs.getBase)) { overlaps => - withResource(overlaps.not) { notOverlap => - withResource(bothNonEmpty.and(eitherHasNulls)) { nonEmptyAndHasNull => - withResource(nonEmptyAndHasNull.and(notOverlap)) { returnNull => - returnNull.ifElse( - GpuColumnVector.columnVectorFromNull(lhs.getRowCount.toInt, BooleanType), - overlaps - ) - } - } - } - } - } - } + ColumnView.listOverlap(lhs.getBase, rhs.getBase) } } From cd6beb286d1b45532d61e118d13bf2d3dbf0c1f4 Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Fri, 22 Jul 2022 10:51:10 -0700 Subject: [PATCH 14/19] Refactor to using GpuComplexTypeMergingExpression Signed-off-by: Navin Kumar --- .../src/main/python/array_test.py | 24 ++-- .../sql/rapids/collectionOperations.scala | 136 ++++++++++++++---- 2 files changed, 121 insertions(+), 39 deletions(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index 924866f0ca5..ed116a4927c 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -34,6 +34,12 @@ array_item_test_gens = array_gens_sample + [array_all_null_gen, ArrayGen(MapGen(StringGen(pattern='key_[0-9]', nullable=False), StringGen(), max_length=10), max_length=10)] +no_neg_zero_all_basic_gens = [byte_gen, short_gen, int_gen, long_gen, + # -0.0 cannot work because of -0.0 == 0.0 in cudf for distinct and + # Spark fixed ordering of 0.0 and -0.0 in Spark 3.1 in the ordering + FloatGen(special_cases=[]), DoubleGen(special_cases=[]), + string_gen, boolean_gen, date_gen, timestamp_gen] + # Merged "test_nested_array_item" with this one since arrays as literals is supported @pytest.mark.parametrize('data_gen', array_item_test_gens, ids=idfn) def test_array_item(data_gen): @@ -396,8 +402,7 @@ def q1(spark): assert_gpu_and_cpu_are_equal_collect(q1) -@pytest.mark.parametrize('data_gen', [byte_gen, short_gen, int_gen, long_gen, - FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) +@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens, ids=idfn) def test_array_intersect(data_gen): gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), @@ -414,30 +419,24 @@ def test_array_intersect(data_gen): ) ) -@pytest.mark.parametrize('data_gen', [byte_gen, short_gen, int_gen, long_gen, - FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) +@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens, ids=idfn) def test_array_union(data_gen): gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), ('b', ArrayGen(data_gen, nullable=True))], nullable=False) - # The 4th item in this integration test here is left commented out here - # There is an issue with running that item with collect(), which affects - # the integration test here. - # See this issue (https://github.com/NVIDIA/spark-rapids/issues/5957) assert_gpu_and_cpu_are_equal_collect( lambda spark: gen_df(spark, gen).selectExpr( 'sort_array(array_union(a, b))', 'sort_array(array_union(b, a))', 'sort_array(array_union(a, array()))', - # 'sort_array(array_union(array(), b))', # see https://github.com/NVIDIA/spark-rapids/issues/5957 + 'sort_array(array_union(array(), b))', 'sort_array(array_union(a, a))', ) ) -@pytest.mark.parametrize('data_gen', [byte_gen, short_gen, int_gen, long_gen, - FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) +@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens, ids=idfn) def test_array_except(data_gen): gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), @@ -454,8 +453,7 @@ def test_array_except(data_gen): ) ) -@pytest.mark.parametrize('data_gen', [byte_gen, short_gen, int_gen, long_gen, - FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) +@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens, ids=idfn) def test_arrays_overlap(data_gen): gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index 8878cabe0f0..21a6affb3e3 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -28,7 +28,7 @@ import com.nvidia.spark.rapids.RapidsPluginImplicits._ import com.nvidia.spark.rapids.shims.ShimExpression import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion} -import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, ImplicitCastInputTypes, NamedExpression, RowOrdering, Sequence, TimeZoneAwareExpression} +import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, ImplicitCastInputTypes, NamedExpression, NullIntolerant, RowOrdering, Sequence, TimeZoneAwareExpression} import org.apache.spark.sql.catalyst.util.GenericArrayData import org.apache.spark.sql.rapids.shims.RapidsErrorUtils import org.apache.spark.sql.types._ @@ -679,31 +679,41 @@ case class GpuArraysZip(children: Seq[Expression]) extends GpuExpression with Sh } } -trait GpuArraySetOperation extends GpuBinaryExpression { +// Base class for GpuArrayExcept, GpuArrayUnion, GpuArrayIntersect +trait GpuArrayBinaryLike extends GpuComplexTypeMergingExpression with NullIntolerant { + val left: Expression + val right: Expression - override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { - withResource(GpuColumnVector.from(lhs, rhs.getRowCount.toInt, lhs.dataType)) { left => - doColumnar(left, rhs) - } - } + @transient override final lazy val children: Seq[Expression] = IndexedSeq(left, right) - override def doColumnar(lhs: GpuColumnVector, rhs: GpuScalar): ColumnVector = { - withResource(GpuColumnVector.from(rhs, lhs.getRowCount.toInt, rhs.dataType)) { right => - doColumnar(lhs, right) - } - } + def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector + def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector + def doColumnar(lhs: GpuColumnVector, rhs: GpuScalar): ColumnVector + def doColumnar(numRows: Int, lhs: GpuScalar, rhs: GpuScalar): ColumnVector - override def doColumnar(numRows: Int, lhs: GpuScalar, rhs: GpuScalar): ColumnVector = { - withResource(GpuColumnVector.from(lhs, numRows, lhs.dataType)) { left => - withResource(GpuColumnVector.from(rhs, numRows, rhs.dataType)) { right => - doColumnar(left, right) + override def columnarEval(batch: ColumnarBatch): Any = { + withResourceIfAllowed(left.columnarEval(batch)) { lhs => + withResourceIfAllowed(right.columnarEval(batch)) { rhs => + (lhs, rhs) match { + case (l: GpuColumnVector, r: GpuColumnVector) => + GpuColumnVector.from(doColumnar(l, r), dataType) + case (l: GpuScalar, r: GpuColumnVector) => + GpuColumnVector.from(doColumnar(l, r), dataType) + case (l: GpuColumnVector, r: GpuScalar) => + GpuColumnVector.from(doColumnar(l, r), dataType) + case (l: GpuScalar, r: GpuScalar) => + GpuColumnVector.from(doColumnar(batch.numRows(), l, r), dataType) + case (l, r) => + throw new UnsupportedOperationException(s"Unsupported data '($l: " + + s"${l.getClass}, $r: ${r.getClass})' for GPU binary expression.") + } } } } } case class GpuArrayExcept(left: Expression, right: Expression) - extends GpuArraySetOperation with ExpectsInputTypes { + extends GpuArrayBinaryLike with ExpectsInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, ArrayType) @@ -720,17 +730,35 @@ case class GpuArrayExcept(left: Expression, right: Expression) TypeCheckResult.TypeCheckFailure(s"$prettyName only supports array input, but found $dt") } - override def dataType: DataType = left.dataType - override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { ColumnView.setDifference(lhs.getBase, rhs.getBase) } + + override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { + withResource(GpuColumnVector.from(lhs, rhs.getRowCount.toInt, lhs.dataType)) { left => + doColumnar(left, rhs) + } + } + + override def doColumnar(lhs: GpuColumnVector, rhs: GpuScalar): ColumnVector = { + withResource(GpuColumnVector.from(rhs, lhs.getRowCount.toInt, rhs.dataType)) { right => + doColumnar(lhs, right) + } + } + + override def doColumnar(numRows: Int, lhs: GpuScalar, rhs: GpuScalar): ColumnVector = { + withResource(GpuColumnVector.from(lhs, numRows, lhs.dataType)) { left => + withResource(GpuColumnVector.from(rhs, numRows, rhs.dataType)) { right => + doColumnar(left, right) + } + } + } } case class GpuArrayIntersect(left: Expression, right: Expression) - extends GpuArraySetOperation with ExpectsInputTypes { + extends GpuArrayBinaryLike with ExpectsInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, ArrayType) @@ -747,17 +775,35 @@ case class GpuArrayIntersect(left: Expression, right: Expression) TypeCheckResult.TypeCheckFailure(s"$prettyName only supports array input, but found $dt") } - override def dataType: DataType = left.dataType - override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { ColumnView.setIntersect(lhs.getBase, rhs.getBase) } + + override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { + withResource(GpuColumnVector.from(lhs, rhs.getRowCount.toInt, lhs.dataType)) { left => + doColumnar(left, rhs) + } + } + + override def doColumnar(lhs: GpuColumnVector, rhs: GpuScalar): ColumnVector = { + withResource(GpuColumnVector.from(rhs, lhs.getRowCount.toInt, rhs.dataType)) { right => + doColumnar(lhs, right) + } + } + + override def doColumnar(numRows: Int, lhs: GpuScalar, rhs: GpuScalar): ColumnVector = { + withResource(GpuColumnVector.from(lhs, numRows, lhs.dataType)) { left => + withResource(GpuColumnVector.from(rhs, numRows, rhs.dataType)) { right => + doColumnar(left, right) + } + } + } } case class GpuArrayUnion(left: Expression, right: Expression) - extends GpuArraySetOperation with ExpectsInputTypes { + extends GpuArrayBinaryLike with ExpectsInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, ArrayType) @@ -774,17 +820,35 @@ case class GpuArrayUnion(left: Expression, right: Expression) TypeCheckResult.TypeCheckFailure(s"$prettyName only supports array input, but found $dt") } - override def dataType: DataType = left.dataType - override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { ColumnView.setUnion(lhs.getBase, rhs.getBase) } + + override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { + withResource(GpuColumnVector.from(lhs, rhs.getRowCount.toInt, lhs.dataType)) { left => + doColumnar(left, rhs) + } + } + + override def doColumnar(lhs: GpuColumnVector, rhs: GpuScalar): ColumnVector = { + withResource(GpuColumnVector.from(rhs, lhs.getRowCount.toInt, rhs.dataType)) { right => + doColumnar(lhs, right) + } + } + + override def doColumnar(numRows: Int, lhs: GpuScalar, rhs: GpuScalar): ColumnVector = { + withResource(GpuColumnVector.from(lhs, numRows, lhs.dataType)) { left => + withResource(GpuColumnVector.from(rhs, numRows, rhs.dataType)) { right => + doColumnar(left, right) + } + } + } } case class GpuArraysOverlap(left: Expression, right: Expression) - extends GpuArraySetOperation with ExpectsInputTypes { + extends GpuBinaryExpression with ExpectsInputTypes with NullIntolerant { override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, ArrayType) @@ -808,6 +872,26 @@ case class GpuArraysOverlap(left: Expression, right: Expression) override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { ColumnView.listOverlap(lhs.getBase, rhs.getBase) } + + override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { + withResource(GpuColumnVector.from(lhs, rhs.getRowCount.toInt, lhs.dataType)) { left => + doColumnar(left, rhs) + } + } + + override def doColumnar(lhs: GpuColumnVector, rhs: GpuScalar): ColumnVector = { + withResource(GpuColumnVector.from(rhs, lhs.getRowCount.toInt, rhs.dataType)) { right => + doColumnar(lhs, right) + } + } + + override def doColumnar(numRows: Int, lhs: GpuScalar, rhs: GpuScalar): ColumnVector = { + withResource(GpuColumnVector.from(lhs, numRows, lhs.dataType)) { left => + withResource(GpuColumnVector.from(rhs, numRows, rhs.dataType)) { right => + doColumnar(left, right) + } + } + } } class GpuSequenceMeta( From 4587238570aef6f45632bb3dfb3dfad7bd2a060d Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Fri, 22 Jul 2022 10:58:54 -0700 Subject: [PATCH 15/19] Add decimal_gens to testing here Signed-off-by: Navin Kumar --- integration_tests/src/main/python/array_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index ed116a4927c..081ac2df7ee 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -35,8 +35,8 @@ ArrayGen(MapGen(StringGen(pattern='key_[0-9]', nullable=False), StringGen(), max_length=10), max_length=10)] no_neg_zero_all_basic_gens = [byte_gen, short_gen, int_gen, long_gen, - # -0.0 cannot work because of -0.0 == 0.0 in cudf for distinct and - # Spark fixed ordering of 0.0 and -0.0 in Spark 3.1 in the ordering + # -0.0 cannot work because of -0.0 == 0.0 in cudf for distinct + # but nans do work FloatGen(special_cases=[]), DoubleGen(special_cases=[]), string_gen, boolean_gen, date_gen, timestamp_gen] @@ -402,7 +402,7 @@ def q1(spark): assert_gpu_and_cpu_are_equal_collect(q1) -@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens, ids=idfn) +@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) def test_array_intersect(data_gen): gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), @@ -419,7 +419,7 @@ def test_array_intersect(data_gen): ) ) -@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens, ids=idfn) +@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) def test_array_union(data_gen): gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), @@ -436,7 +436,7 @@ def test_array_union(data_gen): ) ) -@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens, ids=idfn) +@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) def test_array_except(data_gen): gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), @@ -453,7 +453,7 @@ def test_array_except(data_gen): ) ) -@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens, ids=idfn) +@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) def test_arrays_overlap(data_gen): gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), From d2c8911fab3d6e8dfd3b84f5b78b4cbb68fced0c Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Fri, 22 Jul 2022 13:33:12 -0700 Subject: [PATCH 16/19] ensure all other special cases for floats and doubles are here except for -0.0 Signed-off-by: Navin Kumar --- .../src/main/python/array_test.py | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index 081ac2df7ee..5f2c3ea863c 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -34,10 +34,38 @@ array_item_test_gens = array_gens_sample + [array_all_null_gen, ArrayGen(MapGen(StringGen(pattern='key_[0-9]', nullable=False), StringGen(), max_length=10), max_length=10)] + +_non_neg_zero_float_special_cases = [ + FLOAT_MIN, + FLOAT_MAX, + -1.0, + 1.0, + 0.0, + float('inf'), + float('-inf'), + float('nan'), + NEG_FLOAT_NAN_MAX_VALUE +] + +_non_neg_zero_double_special_cases = [ + DoubleGen.make_from(1, DOUBLE_MAX_EXP, DOUBLE_MAX_FRACTION), + DoubleGen.make_from(0, DOUBLE_MAX_EXP, DOUBLE_MAX_FRACTION), + DoubleGen.make_from(1, DOUBLE_MIN_EXP, DOUBLE_MAX_FRACTION), + DoubleGen.make_from(0, DOUBLE_MIN_EXP, DOUBLE_MAX_FRACTION), + -1.0, + 1.0, + 0.0, + float('inf'), + float('-inf'), + float('nan'), + NEG_DOUBLE_NAN_MAX_VALUE +] + no_neg_zero_all_basic_gens = [byte_gen, short_gen, int_gen, long_gen, # -0.0 cannot work because of -0.0 == 0.0 in cudf for distinct - # but nans do work - FloatGen(special_cases=[]), DoubleGen(special_cases=[]), + # but nans and other default special cases do work + FloatGen(special_cases=_non_neg_zero_float_special_cases), + DoubleGen(special_cases=_non_neg_zero_double_special_cases), string_gen, boolean_gen, date_gen, timestamp_gen] # Merged "test_nested_array_item" with this one since arrays as literals is supported From 893f7d00c5a8c71c539ed9ddf811b24b343bf6df Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Mon, 25 Jul 2022 14:12:53 -0700 Subject: [PATCH 17/19] Updated docs and xfail Double and Float tests on Spark versions < 3.1.3 Signed-off-by: Navin Kumar --- docs/configs.md | 4 + docs/supported_ops.md | 1338 ++++++++++------- .../src/main/python/array_test.py | 10 +- .../src/main/python/spark_session.py | 3 + tools/src/main/resources/operatorsScore.csv | 4 + tools/src/main/resources/supportedExprs.csv | 12 + 6 files changed, 850 insertions(+), 521 deletions(-) diff --git a/docs/configs.md b/docs/configs.md index 8276c1a3ef4..b8c32558898 100644 --- a/docs/configs.md +++ b/docs/configs.md @@ -161,11 +161,15 @@ Name | SQL Function(s) | Description | Default Value | Notes spark.rapids.sql.expression.And|`and`|Logical AND|true|None| spark.rapids.sql.expression.AnsiCast| |Convert a column of one type of data into another type|true|None| spark.rapids.sql.expression.ArrayContains|`array_contains`|Returns a boolean if the array contains the passed in key|true|None| +spark.rapids.sql.expression.ArrayExcept|`array_except`|Returns an array of the elements in array1 but not in array2, without duplicates|true|None| spark.rapids.sql.expression.ArrayExists|`exists`|Return true if any element satisfies the predicate LambdaFunction|true|None| +spark.rapids.sql.expression.ArrayIntersect|`array_intersect`|Returns an array of the elements in the intersection of array1 and array2, without duplicates|true|None| spark.rapids.sql.expression.ArrayMax|`array_max`|Returns the maximum value in the array|true|None| spark.rapids.sql.expression.ArrayMin|`array_min`|Returns the minimum value in the array|true|None| spark.rapids.sql.expression.ArrayRepeat|`array_repeat`|Returns the array containing the given input value (left) count (right) times|true|None| spark.rapids.sql.expression.ArrayTransform|`transform`|Transform elements in an array using the transform function. This is similar to a `map` in functional programming|true|None| +spark.rapids.sql.expression.ArrayUnion|`array_union`|Returns an array of the elements in the union of array1 and array2, without duplicates.|true|None| +spark.rapids.sql.expression.ArraysOverlap|`arrays_overlap`|Returns true if a1 contains at least a non-null element present also in a2. If the arrays have no common element and they are both non-empty and either of them contains a null element null is returned, false otherwise.|true|None| spark.rapids.sql.expression.ArraysZip|`arrays_zip`|Returns a merged array of structs in which the N-th struct contains all N-th values of input arrays.|true|None| spark.rapids.sql.expression.Asin|`asin`|Inverse sine|true|None| spark.rapids.sql.expression.Asinh|`asinh`|Inverse hyperbolic sine|true|None| diff --git a/docs/supported_ops.md b/docs/supported_ops.md index f3b20edf50c..39baa5c3c1c 100644 --- a/docs/supported_ops.md +++ b/docs/supported_ops.md @@ -2030,12 +2030,12 @@ are limited. -ArrayExists -`exists` -Return true if any element satisfies the predicate LambdaFunction +ArrayExcept +`array_except` +Returns an array of the elements in array1 but not in array2, without duplicates None project -argument +array1 @@ -2050,14 +2050,13 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT -function -S +array2 @@ -2072,13 +2071,13 @@ are limited. +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT result -S @@ -2093,6 +2092,7 @@ are limited. +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT @@ -2124,6 +2124,142 @@ are limited. UDT +ArrayExists +`exists` +Return true if any element satisfies the predicate LambdaFunction +None +project +argument + + + + + + + + + + + + + + +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT + + + + + +function +S + + + + + + + + + + + + + + + + + + + +result +S + + + + + + + + + + + + + + + + + + + +ArrayIntersect +`array_intersect` +Returns an array of the elements in the intersection of array1 and array2, without duplicates +None +project +array1 + + + + + + + + + + + + + + +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT + + + + + +array2 + + + + + + + + + + + + + + +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT + + + + + +result + + + + + + + + + + + + + + +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT + + + + + ArrayMax `array_max` Returns the maximum value in the array @@ -2354,6 +2490,168 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + +ArrayUnion +`array_union` +Returns an array of the elements in the union of array1 and array2, without duplicates. +None +project +array1 + + + + + + + + + + + + + + +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT + + + + + +array2 + + + + + + + + + + + + + + +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT + + + + + +result + + + + + + + + + + + + + + +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT + + + + + +ArraysOverlap +`arrays_overlap` +Returns true if a1 contains at least a non-null element present also in a2. If the arrays have no common element and they are both non-empty and either of them contains a null element null is returned, false otherwise. +None +project +array1 + + + + + + + + + + + + + + +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT + + + + + +array2 + + + + + + + + + + + + + + +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT + + + + + +result +S + + + + + + + + + + + + + + + + + + + ArraysZip `arrays_zip` Returns a merged array of structs in which the N-th struct contains all N-th values of input arrays. @@ -2491,32 +2789,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Asinh `asinh` Inverse hyperbolic sine @@ -2607,6 +2879,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + AtLeastNNonNulls Checks if number of non null/Nan values is greater than a given value @@ -2882,32 +3180,6 @@ are limited. NS -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - BRound `bround` Round an expression to d decimal places using HALF_EVEN rounding mode @@ -2976,6 +3248,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + BitLength `bit_length` The bit length of string data @@ -3245,32 +3543,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - BitwiseOr `\|` Returns the bitwise OR of the operands @@ -3403,6 +3675,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + BitwiseXor `^` Returns the bitwise XOR of the operands @@ -3603,32 +3901,6 @@ are limited. NS -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Cbrt `cbrt` Cube root @@ -3766,6 +4038,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + CheckOverflow CheckOverflow after arithmetic operations between DecimalType data @@ -3998,54 +4296,28 @@ are limited. - - - -result -S - - - - - - - - - - - - - - - - - - - -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT + + + +result +S + + + + + + + + + + + + + + + + + Cos @@ -4138,6 +4410,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Cosh `cosh` Hyperbolic cosine @@ -4412,32 +4710,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - CreateNamedStruct `named_struct`, `struct` Creates a struct with the given field names and values @@ -4506,6 +4778,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + CurrentRow$ Special boundary for a window frame, indicating stopping at the current row @@ -4804,32 +5102,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - DateSub `date_sub` Returns the date that is num_days before start_date @@ -4898,6 +5170,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + DayOfMonth `dayofmonth`, `day` Returns the day of the month from a date or timestamp @@ -5222,32 +5520,6 @@ are limited. NS -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - EndsWith Ends with @@ -5316,6 +5588,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + EqualNullSafe `<=>` Check if the values are equal including nulls <=> @@ -5606,32 +5904,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Explode `explode`, `explode_outer` Given an input array produces a sequence of rows for each value in the array @@ -5679,6 +5951,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Expm1 `expm1` Euler's number e raised to a power minus 1 @@ -5999,32 +6297,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - GetJsonObject `get_json_object` Extracts a json object from path @@ -6093,6 +6365,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + GetMapValue Gets Value from a Map based on a key @@ -6408,32 +6706,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - GreaterThanOrEqual `>=` >= operator @@ -6566,6 +6838,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Greatest `greatest` Returns the greatest value of all parameters, skipping null values @@ -6817,32 +7115,6 @@ are limited. NS -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - In `in` IN operator @@ -6958,6 +7230,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + InitCap `initcap` Returns str with the first letter of each word in uppercase. All other letters are in lowercase @@ -7198,32 +7496,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - IsNotNull `isnotnull` Checks if a value is not null @@ -7365,6 +7637,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + KnownNotNull Tag an expression as known to not be null @@ -7545,54 +7843,28 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -NS - - -result -S -S -S -S -S -S -S -S -PS
UTC is only supported TZ for TIMESTAMP -S -S -S -NS -NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -NS - - -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT +NS + + +result +S +S +S +S +S +S +S +S +PS
UTC is only supported TZ for TIMESTAMP +S +S +S +NS +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +NS LastDay @@ -7731,6 +8003,32 @@ are limited. NS +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Least `least` Returns the least value of all parameters, skipping null values @@ -7957,32 +8255,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - LessThanOrEqual `<=` <= operator @@ -8115,6 +8387,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Like `like` Like @@ -8325,32 +8623,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Log1p `log1p` Natural log 1 + expr @@ -8513,6 +8785,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Lower `lower`, `lcase` String lowercase operator @@ -8701,32 +8999,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - MapFilter `map_filter` Filters entries in a map using the function @@ -8889,6 +9161,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Md5 `md5` MD5 hash operator @@ -9188,32 +9486,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Murmur3Hash `hash` Murmur3 hash operator @@ -9261,6 +9533,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + NaNvl `nanvl` Evaluates to `left` iff left is not NaN, `right` otherwise diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index 5f2c3ea863c..88027bda016 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -16,7 +16,7 @@ from asserts import assert_gpu_and_cpu_are_equal_collect, assert_gpu_and_cpu_are_equal_sql, assert_gpu_and_cpu_error, assert_gpu_fallback_collect from data_gen import * -from spark_session import is_before_spark_330, is_databricks104_or_later +from spark_session import is_before_spark_313, is_before_spark_330, is_databricks104_or_later from pyspark.sql.types import * from pyspark.sql.types import IntegralType from pyspark.sql.functions import array_contains, col, element_at, lit @@ -432,6 +432,8 @@ def q1(spark): @pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) def test_array_intersect(data_gen): + if is_before_spark_313() and isinstance(data_gen, (FloatGen, DoubleGen)): + pytest.xfail("array_intersect tests on floating point types will fail because of NaN inequalilty in Spark 3.1.2 or earlier") gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), ('b', ArrayGen(data_gen, nullable=True))], @@ -449,6 +451,8 @@ def test_array_intersect(data_gen): @pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) def test_array_union(data_gen): + if is_before_spark_313() and isinstance(data_gen, (FloatGen, DoubleGen)): + pytest.xfail("array_union tests on floating point types will fail because of NaN inequality in Spark 3.1.2 or earlier") gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), ('b', ArrayGen(data_gen, nullable=True))], @@ -466,6 +470,8 @@ def test_array_union(data_gen): @pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) def test_array_except(data_gen): + if is_before_spark_313() and isinstance(data_gen, (FloatGen, DoubleGen)): + pytest.xfail("array_except tests on floating point types will fail because of NaN inequality in Spark 3.1.2 or earlier") gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), ('b', ArrayGen(data_gen, nullable=True))], @@ -483,6 +489,8 @@ def test_array_except(data_gen): @pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) def test_arrays_overlap(data_gen): + if is_before_spark_313() and isinstance(data_gen, (FloatGen, DoubleGen)): + pytest.xfail("arrays_overlap tests on floating point types will fail because of NaN inequality in Spark 3.1.2 or earlier") gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), ('b', ArrayGen(data_gen, nullable=True))], diff --git a/integration_tests/src/main/python/spark_session.py b/integration_tests/src/main/python/spark_session.py index d6feb691222..e22fb3f4a3b 100644 --- a/integration_tests/src/main/python/spark_session.py +++ b/integration_tests/src/main/python/spark_session.py @@ -134,6 +134,9 @@ def with_gpu_session(func, conf={}): def is_before_spark_312(): return spark_version() < "3.1.2" +def is_before_spark_313(): + return spark_version() < "3.1.3" + def is_before_spark_314(): return spark_version() < "3.1.4" diff --git a/tools/src/main/resources/operatorsScore.csv b/tools/src/main/resources/operatorsScore.csv index a8353a9e396..88394801d5d 100644 --- a/tools/src/main/resources/operatorsScore.csv +++ b/tools/src/main/resources/operatorsScore.csv @@ -43,11 +43,15 @@ Alias,4 And,4 ApproximatePercentile,4 ArrayContains,4 +ArrayExcept,4 ArrayExists,4 +ArrayIntersect,4 ArrayMax,4 ArrayMin,4 ArrayRepeat,4 ArrayTransform,4 +ArrayUnion,4 +ArraysOverlap,4 ArraysZip,4 Asin,4 Asinh,4 diff --git a/tools/src/main/resources/supportedExprs.csv b/tools/src/main/resources/supportedExprs.csv index 955f8ce9af3..02811a55766 100644 --- a/tools/src/main/resources/supportedExprs.csv +++ b/tools/src/main/resources/supportedExprs.csv @@ -30,9 +30,15 @@ And,S,`and`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA ArrayContains,S,`array_contains`,None,project,array,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArrayContains,S,`array_contains`,None,project,key,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS ArrayContains,S,`array_contains`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ArrayExcept,S,`array_except`,None,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayExcept,S,`array_except`,None,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayExcept,S,`array_except`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArrayExists,S,`exists`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArrayExists,S,`exists`,None,project,function,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA ArrayExists,S,`exists`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ArrayIntersect,S,`array_intersect`,None,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayIntersect,S,`array_intersect`,None,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayIntersect,S,`array_intersect`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArrayMax,S,`array_max`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArrayMax,S,`array_max`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS ArrayMin,S,`array_min`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA @@ -43,6 +49,12 @@ ArrayRepeat,S,`array_repeat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,N ArrayTransform,S,`transform`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArrayTransform,S,`transform`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS ArrayTransform,S,`transform`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayUnion,S,`array_union`,None,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayUnion,S,`array_union`,None,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayUnion,S,`array_union`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArraysOverlap,S,`arrays_overlap`,None,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArraysOverlap,S,`arrays_overlap`,None,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArraysOverlap,S,`arrays_overlap`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA ArraysZip,S,`arrays_zip`,None,project,children,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArraysZip,S,`arrays_zip`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA Asin,S,`asin`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA From 3a832576065011b555e7ac5b8b65ec7623266a41 Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Tue, 26 Jul 2022 16:50:52 -0700 Subject: [PATCH 18/19] Updated references to methods in cudf and added some scalar tests Signed-off-by: Navin Kumar --- docs/supported_ops.md | 104 +++++++++--------- .../src/main/python/array_test.py | 13 ++- .../sql/rapids/collectionOperations.scala | 8 +- 3 files changed, 65 insertions(+), 60 deletions(-) diff --git a/docs/supported_ops.md b/docs/supported_ops.md index 71f4fc2b0f2..c5b8989edbb 100644 --- a/docs/supported_ops.md +++ b/docs/supported_ops.md @@ -9858,32 +9858,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Or `or` Logical OR @@ -10016,6 +9990,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + PercentRank `percent_rank` Window function that returns the percent rank value within the aggregation window @@ -10310,32 +10310,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - PreciseTimestampConversion Expression used internally to convert the TimestampType to Long and back without losing precision, i.e. in microseconds. Used in time windowing @@ -10383,6 +10357,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + PromotePrecision PromotePrecision before arithmetic operations between DecimalType data diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index 88027bda016..f2271f40083 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -446,7 +446,8 @@ def test_array_intersect(data_gen): 'sort_array(array_intersect(a, array()))', 'sort_array(array_intersect(array(), b))', 'sort_array(array_intersect(a, a))', - ) + 'sort_array(array_intersect(array(1), array(1, 2, 3)))', + 'sort_array(array_intersect(array(), array(1, 2, 3)))') ) @pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) @@ -465,7 +466,8 @@ def test_array_union(data_gen): 'sort_array(array_union(a, array()))', 'sort_array(array_union(array(), b))', 'sort_array(array_union(a, a))', - ) + 'sort_array(array_union(array(1), array(1, 2, 3)))', + 'sort_array(array_union(array(), array(1, 2, 3)))') ) @pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) @@ -484,7 +486,8 @@ def test_array_except(data_gen): 'sort_array(array_except(a, array()))', 'sort_array(array_except(array(), b))', 'sort_array(array_except(a, a))', - ) + 'sort_array(array_except(array(1, 2, 3), array(1, 2, 3)))', + 'sort_array(array_except(array(1), array(1, 2, 3)))') ) @pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) @@ -503,5 +506,7 @@ def test_arrays_overlap(data_gen): 'arrays_overlap(a, array())', 'arrays_overlap(array(), b)', 'arrays_overlap(a, a)', - ) + 'arrays_overlap(array(1), array(1, 2))', + 'arrays_overlap(array(3, 4), array(1, 2))', + 'arrays_overlap(array(), array(1, 2))') ) \ No newline at end of file diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index 21a6affb3e3..aa5bc939179 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -733,7 +733,7 @@ case class GpuArrayExcept(left: Expression, right: Expression) override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { - ColumnView.setDifference(lhs.getBase, rhs.getBase) + ColumnView.listsDifferenceDistinct(lhs.getBase, rhs.getBase) } override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { @@ -778,7 +778,7 @@ case class GpuArrayIntersect(left: Expression, right: Expression) override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { - ColumnView.setIntersect(lhs.getBase, rhs.getBase) + ColumnView.listsIntersectDistinct(lhs.getBase, rhs.getBase) } override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { @@ -823,7 +823,7 @@ case class GpuArrayUnion(left: Expression, right: Expression) override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { - ColumnView.setUnion(lhs.getBase, rhs.getBase) + ColumnView.listsUnionDistinct(lhs.getBase, rhs.getBase) } override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { @@ -870,7 +870,7 @@ case class GpuArraysOverlap(left: Expression, right: Expression) override def nullable: Boolean = true override def doColumnar(lhs: GpuColumnVector, rhs: GpuColumnVector): ColumnVector = { - ColumnView.listOverlap(lhs.getBase, rhs.getBase) + ColumnView.listsHaveOverlap(lhs.getBase, rhs.getBase) } override def doColumnar(lhs: GpuScalar, rhs: GpuColumnVector): ColumnVector = { From 5736e1e20629cffadde8e1a76fd116fdf06561ba Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Mon, 1 Aug 2022 15:34:46 -0700 Subject: [PATCH 19/19] Add incompat documentation, and update tests to handle pre Spark 3.1.3 case as separate test Signed-off-by: Navin Kumar --- docs/configs.md | 8 +- docs/supported_ops.md | 8 +- .../src/main/python/array_test.py | 110 ++++++++++++++++-- .../nvidia/spark/rapids/GpuOverrides.scala | 24 +++- tools/src/main/resources/supportedExprs.csv | 24 ++-- 5 files changed, 140 insertions(+), 34 deletions(-) diff --git a/docs/configs.md b/docs/configs.md index 9110b8c48a2..a10257c08ae 100644 --- a/docs/configs.md +++ b/docs/configs.md @@ -161,15 +161,15 @@ Name | SQL Function(s) | Description | Default Value | Notes spark.rapids.sql.expression.And|`and`|Logical AND|true|None| spark.rapids.sql.expression.AnsiCast| |Convert a column of one type of data into another type|true|None| spark.rapids.sql.expression.ArrayContains|`array_contains`|Returns a boolean if the array contains the passed in key|true|None| -spark.rapids.sql.expression.ArrayExcept|`array_except`|Returns an array of the elements in array1 but not in array2, without duplicates|true|None| +spark.rapids.sql.expression.ArrayExcept|`array_except`|Returns an array of the elements in array1 but not in array2, without duplicates|true|This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+| spark.rapids.sql.expression.ArrayExists|`exists`|Return true if any element satisfies the predicate LambdaFunction|true|None| -spark.rapids.sql.expression.ArrayIntersect|`array_intersect`|Returns an array of the elements in the intersection of array1 and array2, without duplicates|true|None| +spark.rapids.sql.expression.ArrayIntersect|`array_intersect`|Returns an array of the elements in the intersection of array1 and array2, without duplicates|true|This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+| spark.rapids.sql.expression.ArrayMax|`array_max`|Returns the maximum value in the array|true|None| spark.rapids.sql.expression.ArrayMin|`array_min`|Returns the minimum value in the array|true|None| spark.rapids.sql.expression.ArrayRepeat|`array_repeat`|Returns the array containing the given input value (left) count (right) times|true|None| spark.rapids.sql.expression.ArrayTransform|`transform`|Transform elements in an array using the transform function. This is similar to a `map` in functional programming|true|None| -spark.rapids.sql.expression.ArrayUnion|`array_union`|Returns an array of the elements in the union of array1 and array2, without duplicates.|true|None| -spark.rapids.sql.expression.ArraysOverlap|`arrays_overlap`|Returns true if a1 contains at least a non-null element present also in a2. If the arrays have no common element and they are both non-empty and either of them contains a null element null is returned, false otherwise.|true|None| +spark.rapids.sql.expression.ArrayUnion|`array_union`|Returns an array of the elements in the union of array1 and array2, without duplicates.|true|This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+| +spark.rapids.sql.expression.ArraysOverlap|`arrays_overlap`|Returns true if a1 contains at least a non-null element present also in a2. If the arrays have no common element and they are both non-empty and either of them contains a null element null is returned, false otherwise.|true|This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+| spark.rapids.sql.expression.ArraysZip|`arrays_zip`|Returns a merged array of structs in which the N-th struct contains all N-th values of input arrays.|true|None| spark.rapids.sql.expression.Asin|`asin`|Inverse sine|true|None| spark.rapids.sql.expression.Asinh|`asinh`|Inverse hyperbolic sine|true|None| diff --git a/docs/supported_ops.md b/docs/supported_ops.md index c5b8989edbb..3ce1f276b06 100644 --- a/docs/supported_ops.md +++ b/docs/supported_ops.md @@ -2033,7 +2033,7 @@ are limited. ArrayExcept `array_except` Returns an array of the elements in array1 but not in array2, without duplicates -None +This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+ project array1 @@ -2195,7 +2195,7 @@ are limited. ArrayIntersect `array_intersect` Returns an array of the elements in the intersection of array1 and array2, without duplicates -None +This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+ project array1 @@ -2519,7 +2519,7 @@ are limited. ArrayUnion `array_union` Returns an array of the elements in the union of array1 and array2, without duplicates. -None +This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+ project array1 @@ -2587,7 +2587,7 @@ are limited. ArraysOverlap `arrays_overlap` Returns true if a1 contains at least a non-null element present also in a2. If the arrays have no common element and they are both non-empty and either of them contains a null element null is returned, false otherwise. -None +This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal, but the CPU implementation currently does not (see SPARK-39845). Also, Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+ project array1 diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index f2271f40083..6f3e94f3632 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -16,6 +16,7 @@ from asserts import assert_gpu_and_cpu_are_equal_collect, assert_gpu_and_cpu_are_equal_sql, assert_gpu_and_cpu_error, assert_gpu_fallback_collect from data_gen import * +from marks import incompat from spark_session import is_before_spark_313, is_before_spark_330, is_databricks104_or_later from pyspark.sql.types import * from pyspark.sql.types import IntegralType @@ -35,6 +36,8 @@ ArrayGen(MapGen(StringGen(pattern='key_[0-9]', nullable=False), StringGen(), max_length=10), max_length=10)] +# Need these for set-based operations +# See https://issues.apache.org/jira/browse/SPARK-39845 _non_neg_zero_float_special_cases = [ FLOAT_MIN, FLOAT_MAX, @@ -68,6 +71,12 @@ DoubleGen(special_cases=_non_neg_zero_double_special_cases), string_gen, boolean_gen, date_gen, timestamp_gen] +no_neg_zero_all_basic_gens_no_nans = [byte_gen, short_gen, int_gen, long_gen, + # -0.0 cannot work because of -0.0 == 0.0 in cudf for distinct + FloatGen(special_cases=[], no_nans=True), + DoubleGen(special_cases=[], no_nans=True), + string_gen, boolean_gen, date_gen, timestamp_gen] + # Merged "test_nested_array_item" with this one since arrays as literals is supported @pytest.mark.parametrize('data_gen', array_item_test_gens, ids=idfn) def test_array_item(data_gen): @@ -430,10 +439,10 @@ def q1(spark): assert_gpu_and_cpu_are_equal_collect(q1) +@incompat @pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) +@pytest.mark.skipif(is_before_spark_313(), reason="NaN equality is only handled in Spark 3.1.3+") def test_array_intersect(data_gen): - if is_before_spark_313() and isinstance(data_gen, (FloatGen, DoubleGen)): - pytest.xfail("array_intersect tests on floating point types will fail because of NaN inequalilty in Spark 3.1.2 or earlier") gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), ('b', ArrayGen(data_gen, nullable=True))], @@ -449,11 +458,31 @@ def test_array_intersect(data_gen): 'sort_array(array_intersect(array(1), array(1, 2, 3)))', 'sort_array(array_intersect(array(), array(1, 2, 3)))') ) - + +@incompat +@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens_no_nans + decimal_gens, ids=idfn) +@pytest.mark.skipif(not is_before_spark_313(), reason="NaN equality is only handled in Spark 3.1.3+") +def test_array_intersect_before_spark313(data_gen): + gen = StructGen( + [('a', ArrayGen(data_gen, nullable=True)), + ('b', ArrayGen(data_gen, nullable=True))], + nullable=False) + + assert_gpu_and_cpu_are_equal_collect( + lambda spark: gen_df(spark, gen).selectExpr( + 'sort_array(array_intersect(a, b))', + 'sort_array(array_intersect(b, a))', + 'sort_array(array_intersect(a, array()))', + 'sort_array(array_intersect(array(), b))', + 'sort_array(array_intersect(a, a))', + 'sort_array(array_intersect(array(1), array(1, 2, 3)))', + 'sort_array(array_intersect(array(), array(1, 2, 3)))') + ) + +@incompat @pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) +@pytest.mark.skipif(is_before_spark_313(), reason="NaN equality is only handled in Spark 3.1.3+") def test_array_union(data_gen): - if is_before_spark_313() and isinstance(data_gen, (FloatGen, DoubleGen)): - pytest.xfail("array_union tests on floating point types will fail because of NaN inequality in Spark 3.1.2 or earlier") gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), ('b', ArrayGen(data_gen, nullable=True))], @@ -470,10 +499,30 @@ def test_array_union(data_gen): 'sort_array(array_union(array(), array(1, 2, 3)))') ) +@incompat +@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens_no_nans + decimal_gens, ids=idfn) +@pytest.mark.skipif(not is_before_spark_313(), reason="NaN equality is only handled in Spark 3.1.3+") +def test_array_union_before_spark313(data_gen): + gen = StructGen( + [('a', ArrayGen(data_gen, nullable=True)), + ('b', ArrayGen(data_gen, nullable=True))], + nullable=False) + + assert_gpu_and_cpu_are_equal_collect( + lambda spark: gen_df(spark, gen).selectExpr( + 'sort_array(array_union(a, b))', + 'sort_array(array_union(b, a))', + 'sort_array(array_union(a, array()))', + 'sort_array(array_union(array(), b))', + 'sort_array(array_union(a, a))', + 'sort_array(array_union(array(1), array(1, 2, 3)))', + 'sort_array(array_union(array(), array(1, 2, 3)))') + ) + +@incompat @pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) +@pytest.mark.skipif(is_before_spark_313(), reason="NaN equality is only handled in Spark 3.1.3+") def test_array_except(data_gen): - if is_before_spark_313() and isinstance(data_gen, (FloatGen, DoubleGen)): - pytest.xfail("array_except tests on floating point types will fail because of NaN inequality in Spark 3.1.2 or earlier") gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), ('b', ArrayGen(data_gen, nullable=True))], @@ -490,10 +539,30 @@ def test_array_except(data_gen): 'sort_array(array_except(array(1), array(1, 2, 3)))') ) +@incompat +@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens_no_nans + decimal_gens, ids=idfn) +@pytest.mark.skipif(not is_before_spark_313(), reason="NaN equality is only handled in Spark 3.1.3+") +def test_array_except_before_spark313(data_gen): + gen = StructGen( + [('a', ArrayGen(data_gen, nullable=True)), + ('b', ArrayGen(data_gen, nullable=True))], + nullable=False) + + assert_gpu_and_cpu_are_equal_collect( + lambda spark: gen_df(spark, gen).selectExpr( + 'sort_array(array_except(a, b))', + 'sort_array(array_except(b, a))', + 'sort_array(array_except(a, array()))', + 'sort_array(array_except(array(), b))', + 'sort_array(array_except(a, a))', + 'sort_array(array_except(array(1, 2, 3), array(1, 2, 3)))', + 'sort_array(array_except(array(1), array(1, 2, 3)))') + ) + +@incompat @pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens + decimal_gens, ids=idfn) +@pytest.mark.skipif(is_before_spark_313(), reason="NaN equality is only handled in Spark 3.1.3+") def test_arrays_overlap(data_gen): - if is_before_spark_313() and isinstance(data_gen, (FloatGen, DoubleGen)): - pytest.xfail("arrays_overlap tests on floating point types will fail because of NaN inequality in Spark 3.1.2 or earlier") gen = StructGen( [('a', ArrayGen(data_gen, nullable=True)), ('b', ArrayGen(data_gen, nullable=True))], @@ -509,4 +578,25 @@ def test_arrays_overlap(data_gen): 'arrays_overlap(array(1), array(1, 2))', 'arrays_overlap(array(3, 4), array(1, 2))', 'arrays_overlap(array(), array(1, 2))') - ) \ No newline at end of file + ) + +@incompat +@pytest.mark.parametrize('data_gen', no_neg_zero_all_basic_gens_no_nans + decimal_gens, ids=idfn) +@pytest.mark.skipif(not is_before_spark_313(), reason="NaN equality is only handled in Spark 3.1.3+") +def test_arrays_overlap_before_spark313(data_gen): + gen = StructGen( + [('a', ArrayGen(data_gen, nullable=True)), + ('b', ArrayGen(data_gen, nullable=True))], + nullable=False) + + assert_gpu_and_cpu_are_equal_collect( + lambda spark: gen_df(spark, gen).selectExpr( + 'arrays_overlap(a, b)', + 'arrays_overlap(b, a)', + 'arrays_overlap(a, array())', + 'arrays_overlap(array(), b)', + 'arrays_overlap(a, a)', + 'arrays_overlap(array(1), array(1, 2))', + 'arrays_overlap(array(3, 4), array(1, 2))', + 'arrays_overlap(array(), array(1, 2))') + ) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 96940ac9eb4..c1949b8d334 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -2942,7 +2942,11 @@ object GpuOverrides extends Logging { GpuArrayExcept(lhs, rhs) } } - ), + ).incompat("the GPU implementation treats -0.0 and 0.0 as equal, but the CPU " + + "implementation currently does not (see SPARK-39845). Also, Apache Spark " + + "3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were " + + "not treated as being equal. We have chosen to break with compatibility for " + + "the older versions of Spark in this instance and handle NaNs the same as 3.1.3+"), expr[ArrayIntersect]( "Returns an array of the elements in the intersection of array1 and array2, without" + " duplicates", @@ -2960,7 +2964,11 @@ object GpuOverrides extends Logging { GpuArrayIntersect(lhs, rhs) } } - ), + ).incompat("the GPU implementation treats -0.0 and 0.0 as equal, but the CPU " + + "implementation currently does not (see SPARK-39845). Also, Apache Spark " + + "3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were " + + "not treated as being equal. We have chosen to break with compatibility for " + + "the older versions of Spark in this instance and handle NaNs the same as 3.1.3+"), expr[ArrayUnion]( "Returns an array of the elements in the union of array1 and array2, without duplicates.", ExprChecks.binaryProject( @@ -2977,7 +2985,11 @@ object GpuOverrides extends Logging { GpuArrayUnion(lhs, rhs) } } - ), + ).incompat("the GPU implementation treats -0.0 and 0.0 as equal, but the CPU " + + "implementation currently does not (see SPARK-39845). Also, Apache Spark " + + "3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were " + + "not treated as being equal. We have chosen to break with compatibility for " + + "the older versions of Spark in this instance and handle NaNs the same as 3.1.3+"), expr[ArraysOverlap]( "Returns true if a1 contains at least a non-null element present also in a2. If the arrays " + "have no common element and they are both non-empty and either of them contains a null " + @@ -2994,7 +3006,11 @@ object GpuOverrides extends Logging { GpuArraysOverlap(lhs, rhs) } } - ), + ).incompat("the GPU implementation treats -0.0 and 0.0 as equal, but the CPU " + + "implementation currently does not (see SPARK-39845). Also, Apache Spark " + + "3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were " + + "not treated as being equal. We have chosen to break with compatibility for " + + "the older versions of Spark in this instance and handle NaNs the same as 3.1.3+"), expr[TransformKeys]( "Transform keys in a map using a transform function", ExprChecks.projectOnly(TypeSig.MAP.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + diff --git a/tools/src/main/resources/supportedExprs.csv b/tools/src/main/resources/supportedExprs.csv index 9a38ee227a8..9f86c960f49 100644 --- a/tools/src/main/resources/supportedExprs.csv +++ b/tools/src/main/resources/supportedExprs.csv @@ -30,15 +30,15 @@ And,S,`and`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA ArrayContains,S,`array_contains`,None,project,array,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArrayContains,S,`array_contains`,None,project,key,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS ArrayContains,S,`array_contains`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ArrayExcept,S,`array_except`,None,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayExcept,S,`array_except`,None,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayExcept,S,`array_except`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayExcept,S,`array_except`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayExcept,S,`array_except`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayExcept,S,`array_except`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArrayExists,S,`exists`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArrayExists,S,`exists`,None,project,function,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA ArrayExists,S,`exists`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ArrayIntersect,S,`array_intersect`,None,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayIntersect,S,`array_intersect`,None,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayIntersect,S,`array_intersect`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayIntersect,S,`array_intersect`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayIntersect,S,`array_intersect`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayIntersect,S,`array_intersect`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArrayMax,S,`array_max`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArrayMax,S,`array_max`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS ArrayMin,S,`array_min`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA @@ -49,12 +49,12 @@ ArrayRepeat,S,`array_repeat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,N ArrayTransform,S,`transform`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArrayTransform,S,`transform`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS ArrayTransform,S,`transform`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayUnion,S,`array_union`,None,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayUnion,S,`array_union`,None,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayUnion,S,`array_union`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArraysOverlap,S,`arrays_overlap`,None,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArraysOverlap,S,`arrays_overlap`,None,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArraysOverlap,S,`arrays_overlap`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ArrayUnion,S,`array_union`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayUnion,S,`array_union`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArrayUnion,S,`array_union`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA ArraysZip,S,`arrays_zip`,None,project,children,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA ArraysZip,S,`arrays_zip`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA Asin,S,`asin`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA