Skip to content
Closed
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,7 @@ final class ShuffleBlockFetcherIterator(
hostLocalDirManager.getHostLocalDirs(host, port, bmIds.map(_.executorId)) {
case Success(dirsByExecId) =>
fetchMultipleHostLocalBlocks(
hostLocalBlocksWithMissingDirs.filterKeys(bmIds.contains),
hostLocalBlocksWithMissingDirs.filterKeys(bmIds.contains).toMap,
dirsByExecId,
cached = false)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ object Analyzer {
.exists(_._2.map(_._2.exprId).distinct.length > 1),
"Found duplicate rewrite attributes")

val attributeRewrites = AttributeMap(attrMapping)
val attributeRewrites = AttributeMap(attrMapping.toSeq)
// Using attrMapping from the children plans to rewrite their parent node.
// Note that we shouldn't rewrite a node using attrMapping from its sibling nodes.
val p = newPlan.transformExpressions {
Expand All @@ -174,7 +174,7 @@ object Analyzer {
}
attrMapping ++= plan.output.zip(p.output)
.filter { case (a1, a2) => a1.exprId != a2.exprId }
p -> attrMapping
p -> attrMapping.toSeq
} else {
// Just passes through unresolved nodes
plan.mapChildren {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -842,7 +842,7 @@ case class MapObjects private(
val array = ctx.freshName("array")
val determineCollectionType = inputData.dataType match {
case ObjectType(cls) if cls == classOf[Object] =>
val seqClass = classOf[Seq[_]].getName
val seqClass = classOf[scala.collection.Seq[_]].getName
s"""
$seqClass $seq = null;
$elementJavaType[] $array = null;
Expand Down Expand Up @@ -1755,7 +1755,7 @@ case class ValidateExternalType(child: Expression, expected: DataType)
Seq(classOf[java.math.BigDecimal], classOf[scala.math.BigDecimal], classOf[Decimal])
.map(cls => s"$obj instanceof ${cls.getName}").mkString(" || ")
case _: ArrayType =>
s"$obj.getClass().isArray() || $obj instanceof ${classOf[Seq[_]].getName}"
s"$obj.getClass().isArray() || $obj instanceof ${classOf[scala.collection.Seq[_]].getName}"
case _ =>
s"$obj instanceof ${CodeGenerator.boxedType(dataType)}"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@ import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}

class GenericArrayData(val array: Array[Any]) extends ArrayData {

def this(seq: Seq[Any]) = this(seq.toArray)
// Specified this as`scala.collection.Seq` because seqOrArray can be
// `mutable.ArraySeq` in Scala 2.13
def this(seq: scala.collection.Seq[Any]) = this(seq.toArray)
def this(list: java.util.List[Any]) = this(list.asScala.toSeq)

// TODO: This is boxing. We should specialize.
Expand All @@ -38,7 +40,9 @@ class GenericArrayData(val array: Array[Any]) extends ArrayData {
def this(primitiveArray: Array[Boolean]) = this(primitiveArray.toSeq)

def this(seqOrArray: Any) = this(seqOrArray match {
case seq: Seq[Any] => seq.toArray
// Specified this as`scala.collection.Seq` because seqOrArray can be
// `mutable.ArraySeq` in Scala 2.13
case seq: scala.collection.Seq[Any] => seq.toArray
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The entrance is

object ArrayData {
def toArrayData(input: Any): ArrayData = input match {
case a: Array[Boolean] => UnsafeArrayData.fromPrimitiveArray(a)
case a: Array[Byte] => UnsafeArrayData.fromPrimitiveArray(a)
case a: Array[Short] => UnsafeArrayData.fromPrimitiveArray(a)
case a: Array[Int] => UnsafeArrayData.fromPrimitiveArray(a)
case a: Array[Long] => UnsafeArrayData.fromPrimitiveArray(a)
case a: Array[Float] => UnsafeArrayData.fromPrimitiveArray(a)
case a: Array[Double] => UnsafeArrayData.fromPrimitiveArray(a)
case other => new GenericArrayData(other)
}

not easy to call toSeq, so I changed it here

case array: Array[Any] => array // array of objects, so no need to convert
case array: Array[_] => array.toSeq.toArray[Any] // array of primitives, so box them
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -478,15 +478,15 @@ final class Decimal extends Ordered[Decimal] with Serializable {
if (decimalVal.eq(null) && that.decimalVal.eq(null) && scale == that.scale) {
Decimal(longVal + that.longVal, Math.max(precision, that.precision), scale)
} else {
Decimal(toBigDecimal + that.toBigDecimal)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In Scala 2.13, + method is

def +  (that: BigDecimal): BigDecimal = new BigDecimal(this.bigDecimal.add(that.bigDecimal, mc), mc)

and in Scala 2.12 + method is

def +  (that: BigDecimal): BigDecimal = new BigDecimal(this.bigDecimal add that.bigDecimal, mc)

There are some differences in accuracy.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we want to set a MathContext here anyway?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you mean we need change to use methods with MathContext ? Like BigDecimal add(BigDecimal augend, MathContext mc) ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry , I think I don't fully understand this comments ....

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the change is OK here, because we actually do not want to modify the rounding, right?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, you are right ~

Decimal(toBigDecimal.bigDecimal.add(that.toBigDecimal.bigDecimal))
}
}

def - (that: Decimal): Decimal = {
if (decimalVal.eq(null) && that.decimalVal.eq(null) && scale == that.scale) {
Decimal(longVal - that.longVal, Math.max(precision, that.precision), scale)
} else {
Decimal(toBigDecimal - that.toBigDecimal)
Decimal(toBigDecimal.bigDecimal.subtract(that.toBigDecimal.bigDecimal))
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,11 @@ class RelationalGroupedDataset protected[sql](

private[this] def toDF(aggExprs: Seq[Expression]): DataFrame = {
val aggregates = if (df.sparkSession.sessionState.conf.dataFrameRetainGroupColumns) {
groupingExprs ++ aggExprs
groupingExprs match {
// call `toList` because `Stream` can't serialize in scala 2.13
case s: Stream[Expression] => s.toList ++ aggExprs
case other => other ++ aggExprs
}
} else {
aggExprs
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ case class GenerateExec(
if (outer && outputRows.isEmpty) {
joinedRow.withRight(generatorNullRow) :: Nil
} else {
outputRows.map(joinedRow.withRight)
outputRows.toIterator.map(joinedRow.withRight)
}
} ++ LazyIterator(() => boundGenerator.terminate()).map { row =>
// we leave the left side as the last element of its child output
Expand Down
Loading