Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -3862,16 +3862,23 @@ object GpuOverrides extends Logging {

override def aggBufferAttribute: AttributeReference = {
val aggBuffer = c.aggBufferAttributes.head
aggBuffer.copy(dataType = c.dataType)(aggBuffer.exprId, aggBuffer.qualifier)
// Match Spark 4.2+ CollectSet buffer layout for float/double (normalized bit keys).
val ignoreNulls = TypeUtilsShims.collectSetIgnoreNulls(c)
val bufferElementType =
TypeUtilsShims.collectSetCpuBufferElementType(c.child.dataType)
aggBuffer.copy(dataType = ArrayType(bufferElementType, !ignoreNulls))(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add a Spark 4.2 mixed CPU/GPU test for Float/Double collect_set(... RESPECT NULLS) here? This buffer schema derives containsNull from ignoreNulls, but the mixed-stage tests only exercise default IGNORE NULLS; the existing RESPECT NULLS case uses integral input and does not cross a CPU/GPU aggregate boundary. Please cover both CPU-to-GPU and GPU-to-CPU conversion with mixed-null and all-null groups.

aggBuffer.exprId, aggBuffer.qualifier)
}

override def createCpuToGpuBufferConverter(): CpuToGpuAggregateBufferConverter =
new CpuToGpuCollectSetBufferConverter(c.child.dataType,
!TypeUtilsShims.collectSetIgnoreNulls(c))
override def createCpuToGpuBufferConverter(): CpuToGpuAggregateBufferConverter = {
val ignoreNulls = TypeUtilsShims.collectSetIgnoreNulls(c)
new CpuToGpuCollectBufferConverter(
TypeUtilsShims.collectSetCpuBufferElementType(c.child.dataType),
!ignoreNulls)
}

override def createGpuToCpuBufferConverter(): GpuToCpuAggregateBufferConverter =
new GpuToCpuCollectSetBufferConverter(c.child.dataType,
!TypeUtilsShims.collectSetIgnoreNulls(c))
new GpuToCpuCollectBufferConverter()

override val supportBufferConversion: Boolean = true

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import com.nvidia.spark.rapids._
import com.nvidia.spark.rapids.Arm.withResource
import com.nvidia.spark.rapids.RapidsPluginImplicits.ReallyAGpuExpression
import com.nvidia.spark.rapids.jni.Aggregation64Utils
import com.nvidia.spark.rapids.shims.{GpuDeterministicFirstLastCollectShim, ShimExpression, TypeUtilsShims}
import com.nvidia.spark.rapids.shims.{GpuDeterministicFirstLastCollectShim, HashUtils, ShimExpression,
TypeUtilsShims}
import com.nvidia.spark.rapids.window._

import org.apache.spark.sql.catalyst.InternalRow
Expand Down Expand Up @@ -1924,23 +1925,32 @@ trait GpuCollectBase
def child: Expression
protected def arrayContainsNull: Boolean = false

/**
* Element type stored in the aggregation buffer. Defaults to the result element type.
* Spark 4.2+ CollectSet overrides this for float/double to store normalized bit keys.
*/
protected def bufferElementType: DataType = child.dataType

protected final def bufferDataType: DataType =
ArrayType(bufferElementType, containsNull = arrayContainsNull)

override def nullable: Boolean = false

override def dataType: DataType = ArrayType(child.dataType, containsNull = arrayContainsNull)

override def children: Seq[Expression] = child :: Nil

// WINDOW FUNCTION
override val windowInputProjection: Seq[Expression] = Seq(child)
override lazy val windowInputProjection: Seq[Expression] = Seq(child)

override val initialValues: Seq[Expression] = {
Seq(GpuLiteral.create(new GenericArrayData(Array.empty[Any]), dataType))
override lazy val initialValues: Seq[Expression] = {
Seq(GpuLiteral.create(new GenericArrayData(Array.empty[Any]), bufferDataType))
}

override val inputProjection: Seq[Expression] = Seq(child)
override lazy val inputProjection: Seq[Expression] = Seq(child)

protected final lazy val outputBuf: AttributeReference =
AttributeReference("inputBuf", dataType)()
AttributeReference("inputBuf", bufferDataType)()
}

/**
Expand Down Expand Up @@ -1985,6 +1995,11 @@ case class GpuCollectList(
*
* The two 'offset' parameters are not used by GPU version, but are here for the compatibility
* with the CPU version and automated checks.
*
* On Spark 4.2+, CollectSet stores float/double aggregation buffers as normalized bit patterns
* (INT/LONG) so HashSet can treat NaNs and signed zeros as equal. GPU CollectSet mirrors that
* by normalizing and bit-keying in [[inputProjection]], so mixed CPU/GPU aggregate stages share
* the same buffer layout without host-side per-row float↔bits converters.
*/
case class GpuCollectSet(
child: Expression,
Expand All @@ -1998,10 +2013,36 @@ case class GpuCollectSet(

override protected def arrayContainsNull: Boolean = !ignoreNulls

// Spark 4.2+ float/double buffers use normalized bit keys (see TypeUtilsShims).
override protected def bufferElementType: DataType =
TypeUtilsShims.collectSetCpuBufferElementType(child.dataType)

private lazy val useNormalizedBitKeys: Boolean = bufferElementType != child.dataType

override lazy val inputProjection: Seq[Expression] = {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Blocking: for Spark 4.2 Float/Double, this becomes GpuCollectSetNormalizedBitKey(child), while GpuCollectSet still opts into GpuUnboundedToUnboundedAggWindowExec. That executor selects every marker implementation, but its stage builder accepts only GpuBoundReference/GpuLiteral, so the existing fully-unbounded Float/Double window IT reaches IllegalStateException("Unexpected expression"). It also exposes aggregate result types directly and never applies evaluateExpression, so simply accepting this unary projection would still omit GpuCollectSetBitKeysToValues. Please either guard this path back to regular GpuWindowExec or add both projection and finalizer support, with a Spark 4.2 regression.

if (useNormalizedBitKeys) {
Seq(GpuCollectSetNormalizedBitKey(child))
} else {
Seq(child)
}
}

// Window collect_set emits the result array directly from rolling aggregation, so normalize
// floats in-place rather than switching the window input to bit keys.
override lazy val windowInputProjection: Seq[Expression] = {
if (useNormalizedBitKeys) {
Seq(GpuNormalizeFloatingPointForCollectSet(child))
} else {
Seq(child)
}
}

override lazy val updateAggregates: Seq[CudfAggregate] =
Seq(new CudfCollectSet(dataType, nullPolicy))
override lazy val mergeAggregates: Seq[CudfAggregate] = Seq(new CudfMergeSets(dataType))
override lazy val evaluateExpression: Expression = outputBuf
Seq(new CudfCollectSet(bufferDataType, nullPolicy))
override lazy val mergeAggregates: Seq[CudfAggregate] = Seq(new CudfMergeSets(bufferDataType))
override lazy val evaluateExpression: Expression =
if (useNormalizedBitKeys) GpuCollectSetBitKeysToValues(outputBuf) else outputBuf

override def aggBufferAttributes: Seq[AttributeReference] = outputBuf :: Nil

override def prettyName: String = "collect_set"
Expand All @@ -2027,6 +2068,97 @@ case class GpuCollectSet(
override def getMinPeriods: Int = 0
}

/**
* Helpers matching Spark's NormalizeFloatingNumbers for CollectSet float/double keys:
* canonicalize NaN payloads and map -0.0 to +0.0.
*/
object CollectSetFloatingNormalize {
def normalize(cv: ColumnVector): ColumnVector = {
val dtype = cv.getType
require(dtype == DType.FLOAT32 || dtype == DType.FLOAT64,
s"CollectSet floating normalize expects FLOAT32/FLOAT64, found $dtype")
// Canonicalize NaN payloads first (Spark FLOAT_NORMALIZER / DOUBLE_NORMALIZER), then
// reuse HashUtils for signed-zero normalization (-0.0 -> +0.0).
val withCanonNan = withResource(cv.isNan) { isNan =>

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could this reuse the existing GpuNormalizeNaNAndZero / cuDF normalizeNANsAndZeros() path? The current sequence materializes isNan + ifElse, followed by another equality mask + ifElse in HashUtils.normalizeInput, for every hash/window input batch. The existing native operation performs the same NaN/signed-zero normalization in one transform and avoids duplicate normalization logic and temporaries.

withResource(nanScalar(dtype)) { nan =>
isNan.ifElse(nan, cv)
}
}
withResource(withCanonNan) { normalizedNan =>
HashUtils.normalizeInput(normalizedNan)
}
}

private def nanScalar(dtype: DType): Scalar =
if (dtype == DType.FLOAT32) {
Scalar.fromFloat(Float.NaN)
} else {
Scalar.fromDouble(Double.NaN)
}
}

/** Normalize float/double values for CollectSet without changing the logical type. */
case class GpuNormalizeFloatingPointForCollectSet(child: Expression) extends GpuUnaryExpression {
override def dataType: DataType = child.dataType

override def doColumnar(input: GpuColumnVector): ColumnVector =
CollectSetFloatingNormalize.normalize(input.getBase)
}

/**
* Normalize float/double and bit-cast to the Spark 4.2 CollectSet buffer key type
* (Float -> Int, Double -> Long).
*/
case class GpuCollectSetNormalizedBitKey(child: Expression) extends GpuUnaryExpression {
override def dataType: DataType = {
val keyType = TypeUtilsShims.collectSetCpuBufferElementType(child.dataType)
if (keyType == child.dataType) {
throw new IllegalStateException(
s"CollectSet bit-key conversion only applies when buffer keys differ, found $keyType")
}
keyType
}

override def doColumnar(input: GpuColumnVector): ColumnVector = {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add deterministic Spark 4.2 Float/Double regression data for this normalization and round trip? The current RepeatSeqGen(FloatGen/DoubleGen) data samples before repeating, so it does not guarantee that +0/-0 and distinct NaN payloads land in the same group. Please include +0, -0, multiple NaN payloads, infinities, null, both mixed replacement directions, and an empty typed Float/Double aggregation.

withResource(CollectSetFloatingNormalize.normalize(input.getBase)) { normalized =>
val bitsType = GpuColumnVector.getNonNestedRapidsType(dataType)
withResource(normalized.bitCastTo(bitsType)) { bits =>
bits.copyToColumnVector()
}
}
}
}

/**
* Convert a CollectSet aggregation buffer of normalized bit keys back to float/double values.
* Int keys -> Float, Long keys -> Double.
*/
case class GpuCollectSetBitKeysToValues(child: Expression) extends GpuUnaryExpression {
override def dataType: DataType = child.dataType match {
case ArrayType(IntegerType, containsNull) => ArrayType(FloatType, containsNull)
case ArrayType(LongType, containsNull) => ArrayType(DoubleType, containsNull)
case t =>
throw new IllegalStateException(
s"CollectSet bit-key buffer must be ArrayType(IntegerType|LongType), found $t")
}

override def doColumnar(input: GpuColumnVector): ColumnVector = {
val list = input.getBase
withResource(list.getChildColumnView(0)) { bitsView =>
val outDType = bitsView.getType match {
case DType.INT32 => DType.FLOAT32
case DType.INT64 => DType.FLOAT64
case t => throw new IllegalStateException(s"Unexpected CollectSet bit-key cudf type $t")
}
withResource(bitsView.bitCastTo(outDType)) { valuesView =>
withResource(list.replaceListChild(valuesView)) { replaced =>
replaced.copyToColumnVector()
}
}
}
}
}

class CpuToGpuCollectBufferConverter(
elementType: DataType,
containsNull: Boolean = false) extends CpuToGpuAggregateBufferConverter {
Expand Down Expand Up @@ -2079,163 +2211,6 @@ case class GpuToCpuCollectBufferTransition(
}
}

/**
* CollectSet buffer converters.
*
* Spark 4.2 changed CollectSet's CPU agg buffer for FloatType/DoubleType to store normalized
* bit patterns (IntegerType/LongType) so HashSet can treat NaNs and signed zeros as equal.
* GPU CollectSet still stores the logical float/double values, so mixed CPU/GPU aggregation
* stages must convert between the two buffer layouts.
*/
class CpuToGpuCollectSetBufferConverter(
elementType: DataType,
containsNull: Boolean = false) extends CpuToGpuAggregateBufferConverter {
def createExpression(child: Expression): CpuToGpuBufferTransition = {
CpuToGpuCollectSetBufferTransition(child, elementType, containsNull)
}
}

case class CpuToGpuCollectSetBufferTransition(
override val child: Expression,
private val elementType: DataType,
private val containsNull: Boolean) extends CpuToGpuBufferTransition {

private lazy val row = new UnsafeRow(1)
private lazy val cpuElementType: DataType =
TypeUtilsShims.collectSetCpuBufferElementType(elementType)

override def dataType: DataType = ArrayType(elementType, containsNull)

override protected def nullSafeEval(input: Any): ArrayData = {
val bytes = input.asInstanceOf[Array[Byte]]
row.pointTo(bytes, bytes.length)
val cpuArray = row.getArray(0)
if (cpuElementType == elementType) {
cpuArray.copy()
} else {
CollectSetBufferConversions.cpuBitsToGpuValues(cpuArray, elementType, containsNull)
}
}
}

class GpuToCpuCollectSetBufferConverter(
elementType: DataType,
containsNull: Boolean = false) extends GpuToCpuAggregateBufferConverter {
def createExpression(child: Expression): GpuToCpuBufferTransition = {
GpuToCpuCollectSetBufferTransition(child, elementType, containsNull)
}
}

case class GpuToCpuCollectSetBufferTransition(
override val child: Expression,
private val elementType: DataType,
private val containsNull: Boolean) extends GpuToCpuBufferTransition {

private lazy val cpuElementType: DataType =
TypeUtilsShims.collectSetCpuBufferElementType(elementType)
private lazy val cpuBufferType: DataType = ArrayType(cpuElementType, containsNull)
private lazy val projection = UnsafeProjection.create(Array[DataType](cpuBufferType))

override protected def nullSafeEval(input: Any): Array[Byte] = {
val arrayData = input.asInstanceOf[ArrayData]
val cpuArray = if (cpuElementType == elementType) {
arrayData
} else {
CollectSetBufferConversions.gpuValuesToCpuBits(arrayData, elementType, containsNull)
}
projection.apply(InternalRow.apply(cpuArray)).getBytes
}
}

object CollectSetBufferConversions {
// Matches Spark's NormalizeFloatingNumbers.FLOAT_NORMALIZER / DOUBLE_NORMALIZER.
private def normalizeFloat(f: Float): Float = {
if (f.isNaN) {
Float.NaN
} else if (f == -0.0f) {
0.0f
} else {
f
}
}

private def normalizeDouble(d: Double): Double = {
if (d.isNaN) {
Double.NaN
} else if (d == -0.0d) {
0.0d
} else {
d
}
}

def gpuValuesToCpuBits(
arrayData: ArrayData,
elementType: DataType,
containsNull: Boolean): ArrayData = {
val n = arrayData.numElements()
val out = new Array[Any](n)
var i = 0
elementType match {
case FloatType =>
while (i < n) {
if (containsNull && arrayData.isNullAt(i)) {
out(i) = null
} else {
out(i) = java.lang.Float.floatToIntBits(normalizeFloat(arrayData.getFloat(i)))
}
i += 1
}
case DoubleType =>
while (i < n) {
if (containsNull && arrayData.isNullAt(i)) {
out(i) = null
} else {
out(i) = java.lang.Double.doubleToLongBits(normalizeDouble(arrayData.getDouble(i)))
}
i += 1
}
case other =>
throw new IllegalStateException(
s"Unexpected CollectSet GPU-to-CPU buffer conversion for $other")
}
new GenericArrayData(out)
}

def cpuBitsToGpuValues(
arrayData: ArrayData,
elementType: DataType,
containsNull: Boolean): ArrayData = {
val n = arrayData.numElements()
val out = new Array[Any](n)
var i = 0
elementType match {
case FloatType =>
while (i < n) {
if (containsNull && arrayData.isNullAt(i)) {
out(i) = null
} else {
out(i) = java.lang.Float.intBitsToFloat(arrayData.getInt(i))
}
i += 1
}
case DoubleType =>
while (i < n) {
if (containsNull && arrayData.isNullAt(i)) {
out(i) = null
} else {
out(i) = java.lang.Double.longBitsToDouble(arrayData.getLong(i))
}
i += 1
}
case other =>
throw new IllegalStateException(
s"Unexpected CollectSet CPU-to-GPU buffer conversion for $other")
}
new GenericArrayData(out)
}
}

/**
* Base class for overriding standard deviation and variance aggregations.
* This is also a GPU-based implementation of 'CentralMomentAgg' aggregation class in Spark with
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ object TypeUtilsShims {
val collectSetFloatNanEquality: NaNEquality = NaNEquality.ALL_EQUAL

// Spark 4.2 CollectSet keys float/double by normalized bit patterns in the agg buffer.
// GPU CollectSet uses the same buffer element types so mixed CPU/GPU stages match.
def collectSetCpuBufferElementType(childType: DataType): DataType = childType match {
case FloatType => IntegerType
case DoubleType => LongType
Expand Down
Loading