diff --git a/pom.xml b/pom.xml
index 127f24cd6..0d94d0870 100644
--- a/pom.xml
+++ b/pom.xml
@@ -18,7 +18,7 @@
${scala.major.version}
4.4.0
- 3.2.0
+ 3.2.1
deequ
@@ -74,6 +74,14 @@
${scala.version}
+
+
+ org.scala-lang
+ scala-reflect
+ ${scala.version}
+
+
+
org.apache.spark
spark-core_${scala.major.version}
diff --git a/src/main/scala/com/amazon/deequ/analyzers/catalyst/StatefulApproxQuantile.scala b/src/main/scala/com/amazon/deequ/analyzers/catalyst/StatefulApproxQuantile.scala
index 32388f01a..cd5a3a43b 100644
--- a/src/main/scala/com/amazon/deequ/analyzers/catalyst/StatefulApproxQuantile.scala
+++ b/src/main/scala/com/amazon/deequ/analyzers/catalyst/StatefulApproxQuantile.scala
@@ -1,19 +1,18 @@
/**
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"). You may not
- * use this file except in compliance with the License. A copy of the License
- * is located at
- *
- * http://aws.amazon.com/apache2.0/
- *
- * or in the "license" file accompanying this file. This file is distributed on
- * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- *
- */
-
+ * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not
+ * use this file except in compliance with the License. A copy of the License
+ * is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
package org.apache.spark.sql.catalyst.expressions.aggregate
import org.apache.spark.sql.catalyst.InternalRow
@@ -26,12 +25,13 @@ import org.apache.spark.sql.types._
/** Adjusted version of org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile
* (github tag v2.2.0) */
-private[sql] case class StatefulApproxQuantile(
- child: Expression,
- accuracyExpression: Expression,
- override val mutableAggBufferOffset: Int,
- override val inputAggBufferOffset: Int)
- extends TypedImperativeAggregate[PercentileDigest] with ImplicitCastInputTypes with BinaryLike[Expression] {
+private[sql] case class StatefulApproxQuantile(child: Expression,
+ accuracyExpression: Expression,
+ override val mutableAggBufferOffset: Int,
+ override val inputAggBufferOffset: Int)
+ extends TypedImperativeAggregate[PercentileDigest]
+ with ImplicitCastInputTypes
+ with BinaryLike[Expression] {
def this(child: Expression, accuracyExpression: Expression) = {
this(child, accuracyExpression, 0, 0)
@@ -111,11 +111,12 @@ private[sql] case class StatefulApproxQuantile(
}
override def left: Expression = child
+
override def right: Expression = accuracyExpression
// override def third: Expression = accuracyExpression
- protected def withNewChildrenInternal(
- newFirst: Expression, newSecond: Expression): StatefulApproxQuantile =
+ protected def withNewChildrenInternal(newFirst: Expression,
+ newSecond: Expression): StatefulApproxQuantile =
copy(child = newFirst, accuracyExpression = newSecond)
// protected def withNewChildrenInternal(
diff --git a/src/main/scala/com/amazon/deequ/analyzers/catalyst/StatefulCorrelation.scala b/src/main/scala/com/amazon/deequ/analyzers/catalyst/StatefulCorrelation.scala
index 5e6e82a34..cea644604 100644
--- a/src/main/scala/com/amazon/deequ/analyzers/catalyst/StatefulCorrelation.scala
+++ b/src/main/scala/com/amazon/deequ/analyzers/catalyst/StatefulCorrelation.scala
@@ -50,4 +50,8 @@ private[sql] class StatefulCorrelation(
val state = Seq(super.hashCode(), evaluateExpression)
state.map { _.hashCode() }.foldLeft(0) {(a, b) => 31 * a + b }
}
+
+ override protected def withNewChildrenInternal(newLeft: Expression,
+ newRight: Expression): StatefulCorrelation =
+ new StatefulCorrelation(newLeft, newRight, nullOnDivideByZero)
}
diff --git a/src/test/scala/com/amazon/deequ/SparkContextSpec.scala b/src/test/scala/com/amazon/deequ/SparkContextSpec.scala
index 43e9cb588..007598d97 100644
--- a/src/test/scala/com/amazon/deequ/SparkContextSpec.scala
+++ b/src/test/scala/com/amazon/deequ/SparkContextSpec.scala
@@ -78,6 +78,7 @@ trait SparkContextSpec {
.appName("test")
.config("spark.ui.enabled", "false")
.config("spark.sql.shuffle.partitions", 2.toString)
+ .config("spark.sql.adaptive.enabled", value = false)
.getOrCreate()
session.sparkContext.setCheckpointDir(System.getProperty("java.io.tmpdir"))
session