[SPARK-28519][SQL] Use StrictMath log, pow functions for platform independence

docs/sql-migration-guide-upgrade.md

@@ -155,6 +155,8 @@ license: |
 
   - Since Spark 3.0, 0-argument Java UDF is executed in the executor side identically with other UDFs. In Spark version 2.4 and earlier, 0-argument Java UDF alone was executed in the driver side, and the result was propagated to executors, which might be more performant in some cases but caused inconsistency with a correctness issue in some cases.
 
+  - The result of `java.lang.Math`'s `log`, `log1p`, `exp`, `expm1`, and `pow` may vary across platforms. In Spark 3.0, the result of the equivalent SQL functions (including related SQL functions like `LOG10`) return values consistent with `java.lang.StrictMath`. In virtually all cases this makes no difference in the return value, and the difference is very small, but may not exactly match `java.lang.Math` on x86 platforms in cases like, for example, `log(3.0)`, whose value varies between `Math.log()` and `StrictMath.log()`.
+
 ## Upgrading from Spark SQL 2.4 to 2.4.1
 
   - The value of `spark.executor.heartbeatInterval`, when specified without units like "30" rather than "30s", was

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala

@@ -96,7 +96,7 @@ abstract class UnaryLogExpression(f: Double => Double, name: String)
         if ($c <= $yAsymptote) {
           ${ev.isNull} = true;
         } else {
-          ${ev.value} = java.lang.Math.${funcName}($c);
+          ${ev.value} = java.lang.StrictMath.${funcName}($c);
         }
       """
     )
@@ -291,10 +291,6 @@ case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH"
   usage = """
     _FUNC_(expr) - Returns inverse hyperbolic cosine of `expr`.
   """,
-  arguments = """
-    Arguments:
-      * expr - hyperbolic angle
-  """,
   examples = """
     Examples:
       > SELECT _FUNC_(1);
@@ -304,9 +300,10 @@ case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH"
   """,
   since = "3.0.0")
 case class Acosh(child: Expression)
-  extends UnaryMathExpression((x: Double) => math.log(x + math.sqrt(x * x - 1.0)), "ACOSH") {
+  extends UnaryMathExpression((x: Double) => StrictMath.log(x + math.sqrt(x * x - 1.0)), "ACOSH") {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    defineCodeGen(ctx, ev, c => s"java.lang.Math.log($c + java.lang.Math.sqrt($c * $c - 1.0))")
+    defineCodeGen(ctx, ev,
+      c => s"java.lang.StrictMath.log($c + java.lang.Math.sqrt($c * $c - 1.0))")
   }
 }
 
@@ -361,7 +358,11 @@ case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expre
       > SELECT _FUNC_(0);
        1.0
   """)
-case class Exp(child: Expression) extends UnaryMathExpression(math.exp, "EXP")
+case class Exp(child: Expression) extends UnaryMathExpression(StrictMath.exp, "EXP") {
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    defineCodeGen(ctx, ev, c => s"java.lang.StrictMath.exp($c)")
+  }
+}
 
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns exp(`expr`) - 1.",
@@ -370,7 +371,11 @@ case class Exp(child: Expression) extends UnaryMathExpression(math.exp, "EXP")
       > SELECT _FUNC_(0);
        0.0
   """)
-case class Expm1(child: Expression) extends UnaryMathExpression(math.expm1, "EXPM1")
+case class Expm1(child: Expression) extends UnaryMathExpression(StrictMath.expm1, "EXPM1") {
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    defineCodeGen(ctx, ev, c => s"java.lang.StrictMath.expm1($c)")
+  }
+}
 
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns the largest integer not greater than `expr`.",
@@ -486,7 +491,7 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas
       > SELECT _FUNC_(1);
        0.0
   """)
-case class Log(child: Expression) extends UnaryLogExpression(math.log, "LOG")
+case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LOG")
 
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 2.",
@@ -496,14 +501,14 @@ case class Log(child: Expression) extends UnaryLogExpression(math.log, "LOG")
        1.0
   """)
 case class Log2(child: Expression)
-  extends UnaryLogExpression((x: Double) => math.log(x) / math.log(2), "LOG2") {
+  extends UnaryLogExpression((x: Double) => StrictMath.log(x) / StrictMath.log(2), "LOG2") {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     nullSafeCodeGen(ctx, ev, c =>
       s"""
         if ($c <= $yAsymptote) {
           ${ev.isNull} = true;
         } else {
-          ${ev.value} = java.lang.Math.log($c) / java.lang.Math.log(2);
+          ${ev.value} = java.lang.StrictMath.log($c) / java.lang.StrictMath.log(2);
         }
       """
     )
@@ -517,7 +522,7 @@ case class Log2(child: Expression)
       > SELECT _FUNC_(10);
        1.0
   """)
-case class Log10(child: Expression) extends UnaryLogExpression(math.log10, "LOG10")
+case class Log10(child: Expression) extends UnaryLogExpression(StrictMath.log10, "LOG10")
 
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns log(1 + `expr`).",
@@ -526,7 +531,7 @@ case class Log10(child: Expression) extends UnaryLogExpression(math.log10, "LOG1
       > SELECT _FUNC_(0);
        0.0
   """)
-case class Log1p(child: Expression) extends UnaryLogExpression(math.log1p, "LOG1P") {
+case class Log1p(child: Expression) extends UnaryLogExpression(StrictMath.log1p, "LOG1P") {
   protected override val yAsymptote: Double = -1.0
 }
 
@@ -584,10 +589,6 @@ case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH"
   usage = """
     _FUNC_(expr) - Returns inverse hyperbolic sine of `expr`.
   """,
-  arguments = """
-    Arguments:
-      * expr - hyperbolic angle
-  """,
   examples = """
     Examples:
       > SELECT _FUNC_(0);
@@ -597,11 +598,11 @@ case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH"
 case class Asinh(child: Expression)
   extends UnaryMathExpression((x: Double) => x match {
     case Double.NegativeInfinity => Double.NegativeInfinity
-    case _ => math.log(x + math.sqrt(x * x + 1.0)) }, "ASINH") {
+    case _ => StrictMath.log(x + math.sqrt(x * x + 1.0)) }, "ASINH") {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, c =>
       s"$c == Double.NEGATIVE_INFINITY ? Double.NEGATIVE_INFINITY : " +
-      s"java.lang.Math.log($c + java.lang.Math.sqrt($c * $c + 1.0))")
+      s"java.lang.StrictMath.log($c + java.lang.Math.sqrt($c * $c + 1.0))")
   }
 }
 
@@ -669,10 +670,6 @@ case class Tanh(child: Expression) extends UnaryMathExpression(math.tanh, "TANH"
   usage = """
     _FUNC_(expr) - Returns inverse hyperbolic tangent of `expr`.
   """,
-  arguments = """
-    Arguments:
-      * expr - hyperbolic angle
-  """,
   examples = """
     Examples:
       > SELECT _FUNC_(0);
@@ -682,9 +679,12 @@ case class Tanh(child: Expression) extends UnaryMathExpression(math.tanh, "TANH"
   """,
   since = "3.0.0")
 case class Atanh(child: Expression)
-  extends UnaryMathExpression((x: Double) => 0.5 * math.log((1.0 + x) / (1.0 - x)), "ATANH") {
+  // SPARK-28519: more accurate express for 1/2 * ln((1 + x) / (1 - x))
+  extends UnaryMathExpression((x: Double) =>
+    0.5 * (StrictMath.log1p(x) - StrictMath.log1p(-x)), "ATANH") {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    defineCodeGen(ctx, ev, c => s"0.5 * java.lang.Math.log((1.0 + $c)/(1.0 - $c))")
+    defineCodeGen(ctx, ev,
+      c => s"0.5 * (java.lang.StrictMath.log1p($c) - java.lang.StrictMath.log1p(- $c))")
   }
 }
 
@@ -931,9 +931,9 @@ case class Atan2(left: Expression, right: Expression)
        8.0
   """)
 case class Pow(left: Expression, right: Expression)
-  extends BinaryMathExpression(math.pow, "POWER") {
+  extends BinaryMathExpression(StrictMath.pow, "POWER") {
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    defineCodeGen(ctx, ev, (c1, c2) => s"java.lang.Math.pow($c1, $c2)")
+    defineCodeGen(ctx, ev, (c1, c2) => s"java.lang.StrictMath.pow($c1, $c2)")
   }
 }
 
@@ -1064,7 +1064,7 @@ case class Hypot(left: Expression, right: Expression)
        2.0
   """)
 case class Logarithm(left: Expression, right: Expression)
-  extends BinaryMathExpression((c1, c2) => math.log(c2) / math.log(c1), "LOG") {
+  extends BinaryMathExpression((c1, c2) => StrictMath.log(c2) / StrictMath.log(c1), "LOG") {
 
   /**
    * Natural log, i.e. using e as the base.
@@ -1079,7 +1079,7 @@ case class Logarithm(left: Expression, right: Expression)
     val dLeft = input1.asInstanceOf[Double]
     val dRight = input2.asInstanceOf[Double]
     // Unlike Hive, we support Log base in (0.0, 1.0]
-    if (dLeft <= 0.0 || dRight <= 0.0) null else math.log(dRight) / math.log(dLeft)
+    if (dLeft <= 0.0 || dRight <= 0.0) null else StrictMath.log(dRight) / StrictMath.log(dLeft)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -1089,7 +1089,7 @@ case class Logarithm(left: Expression, right: Expression)
           if ($c2 <= 0.0) {
             ${ev.isNull} = true;
           } else {
-            ${ev.value} = java.lang.Math.log($c2);
+            ${ev.value} = java.lang.StrictMath.log($c2);
           }
         """)
     } else {
@@ -1098,7 +1098,7 @@ case class Logarithm(left: Expression, right: Expression)
           if ($c1 <= 0.0 || $c2 <= 0.0) {
             ${ev.isNull} = true;
           } else {
-            ${ev.value} = java.lang.Math.log($c2) / java.lang.Math.log($c1);
+            ${ev.value} = java.lang.StrictMath.log($c2) / java.lang.StrictMath.log($c1);
           }
         """)
     }

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathExpressionsSuite.scala

@@ -200,7 +200,7 @@ class MathExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("asinh") {
-    testUnary(Asinh, (x: Double) => math.log(x + math.sqrt(x * x + 1.0)))
+    testUnary(Asinh, (x: Double) => StrictMath.log(x + math.sqrt(x * x + 1.0)))
     checkConsistencyBetweenInterpretedAndCodegen(Asinh, DoubleType)
 
     checkEvaluation(Asinh(Double.NegativeInfinity), Double.NegativeInfinity)
@@ -228,7 +228,7 @@ class MathExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("acosh") {
-    testUnary(Acosh, (x: Double) => math.log(x + math.sqrt(x * x - 1.0)))
+    testUnary(Acosh, (x: Double) => StrictMath.log(x + math.sqrt(x * x - 1.0)))
     checkConsistencyBetweenInterpretedAndCodegen(Cosh, DoubleType)
 
     val nullLit = Literal.create(null, NullType)
@@ -267,7 +267,8 @@ class MathExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("atanh") {
-    testUnary(Atanh, (x: Double) => 0.5 * math.log((1.0 + x) / (1.0 - x)))
+    // SPARK-28519: more accurate express for 1/2 * ln((1 + x) / (1 - x))
+    testUnary(Atanh, (x: Double) => 0.5 * (StrictMath.log1p(x) - StrictMath.log1p(-x)))
     checkConsistencyBetweenInterpretedAndCodegen(Atanh, DoubleType)
 
     val nullLit = Literal.create(null, NullType)
@@ -372,12 +373,12 @@ class MathExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("exp") {
-    testUnary(Exp, math.exp)
+    testUnary(Exp, StrictMath.exp)
     checkConsistencyBetweenInterpretedAndCodegen(Exp, DoubleType)
   }
 
   test("expm1") {
-    testUnary(Expm1, math.expm1)
+    testUnary(Expm1, StrictMath.expm1)
     checkConsistencyBetweenInterpretedAndCodegen(Expm1, DoubleType)
   }
 
@@ -387,20 +388,20 @@ class MathExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("log") {
-    testUnary(Log, math.log, (1 to 20).map(_ * 0.1))
-    testUnary(Log, math.log, (-5 to 0).map(_ * 0.1), expectNull = true)
+    testUnary(Log, StrictMath.log, (1 to 20).map(_ * 0.1))
+    testUnary(Log, StrictMath.log, (-5 to 0).map(_ * 0.1), expectNull = true)
     checkConsistencyBetweenInterpretedAndCodegen(Log, DoubleType)
   }
 
   test("log10") {
-    testUnary(Log10, math.log10, (1 to 20).map(_ * 0.1))
-    testUnary(Log10, math.log10, (-5 to 0).map(_ * 0.1), expectNull = true)
+    testUnary(Log10, StrictMath.log10, (1 to 20).map(_ * 0.1))
+    testUnary(Log10, StrictMath.log10, (-5 to 0).map(_ * 0.1), expectNull = true)
     checkConsistencyBetweenInterpretedAndCodegen(Log10, DoubleType)
   }
 
   test("log1p") {
-    testUnary(Log1p, math.log1p, (0 to 20).map(_ * 0.1))
-    testUnary(Log1p, math.log1p, (-10 to -1).map(_ * 1.0), expectNull = true)
+    testUnary(Log1p, StrictMath.log1p, (0 to 20).map(_ * 0.1))
+    testUnary(Log1p, StrictMath.log1p, (-10 to -1).map(_ * 1.0), expectNull = true)
     checkConsistencyBetweenInterpretedAndCodegen(Log1p, DoubleType)
   }
 
@@ -427,7 +428,7 @@ class MathExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("log2") {
-    def f: (Double) => Double = (x: Double) => math.log(x) / math.log(2)
+    def f: (Double) => Double = (x: Double) => StrictMath.log(x) / StrictMath.log(2)
     testUnary(Log2, f, (1 to 20).map(_ * 0.1))
     testUnary(Log2, f, (-5 to 0).map(_ * 1.0), expectNull = true)
     checkConsistencyBetweenInterpretedAndCodegen(Log2, DoubleType)
@@ -444,8 +445,8 @@ class MathExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("pow") {
-    testBinary(Pow, math.pow, (-5 to 5).map(v => (v * 1.0, v * 1.0)))
-    testBinary(Pow, math.pow, Seq((-1.0, 0.9), (-2.2, 1.7), (-2.2, -1.7)), expectNaN = true)
+    testBinary(Pow, StrictMath.pow, (-5 to 5).map(v => (v * 1.0, v * 1.0)))
+    testBinary(Pow, StrictMath.pow, Seq((-1.0, 0.9), (-2.2, 1.7), (-2.2, -1.7)), expectNaN = true)
     checkConsistencyBetweenInterpretedAndCodegen(Pow, DoubleType, DoubleType)
   }
 
@@ -569,7 +570,7 @@ class MathExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("binary log") {
-    val f = (c1: Double, c2: Double) => math.log(c2) / math.log(c1)
+    val f = (c1: Double, c2: Double) => StrictMath.log(c2) / StrictMath.log(c1)
     val domain = (1 to 20).map(v => (v * 0.1, v * 0.2))
 
     domain.foreach { case (v1, v2) =>

sql/core/src/test/resources/sql-tests/results/pgSQL/float8.sql.out

@@ -528,7 +528,7 @@ SELECT atanh(double('0.5'))
 -- !query 56 schema
 struct<ATANH(CAST(0.5 AS DOUBLE)):double>
 -- !query 56 output
-0.5493061443340549
+0.5493061443340548
 
 
 -- !query 57

sql/core/src/test/resources/sql-tests/results/pgSQL/numeric.sql.out

@@ -4192,13 +4192,13 @@ SELECT t1.id1, t1.result, t2.expected
 -- !query 496 schema
 struct<id1:int,result:decimal(38,10),expected:decimal(38,10)>
 -- !query 496 output
-2	224790267919917472	224790267919917955.1326161858
+2	224790267919917440	224790267919917955.1326161858
 4	7405685069595001	7405685069594999.0773399947
 5	5068226527.321263	5068226527.3212726541
 6	281839893606.99365	281839893606.9937234336
-7	1716699575118595840	1716699575118597095.4233081991
+7	1716699575118595580	1716699575118597095.4233081991
 8	167361463828.0749	167361463828.0749132007
-9	107511333880051856	107511333880052007.0414112467
+9	107511333880051872	107511333880052007.0414112467
 
 
 -- !query 497
@@ -4541,7 +4541,7 @@ select exp(1.0)
 -- !query 537 schema
 struct<EXP(CAST(1.0 AS DOUBLE)):double>
 -- !query 537 output
-2.718281828459045
+2.7182818284590455
 
 
 -- !query 538