Fix tests

It seems that the tests assume we can keep opening UI ports. Our
existing default for spark.ports.maxRetries does not tolerate this,
however, so the test process throws an exception and returns exit
code 1.

Author: andrewor14

core/src/main/scala/org/apache/spark/util/Utils.scala

@@ -1335,11 +1335,15 @@ private[spark] object Utils extends Logging {
    * Default number of retries in binding to a port.
    */
   val portMaxRetries: Int = {
-    // SparkEnv may be null during tests
-    Option(SparkEnv.get)
-      .flatMap(_.conf.getOption("spark.ports.maxRetries"))
-      .map(_.toInt)
-      .getOrElse(16)
+    if (sys.props.contains("spark.testing")) {
+      // Set a higher number of retries for tests...
+      sys.props.get("spark.ports.maxRetries").map(_.toInt).getOrElse(100)
+    } else {
+      Option(SparkEnv.get)
+        .flatMap(_.conf.getOption("spark.ports.maxRetries"))
+        .map(_.toInt)
+        .getOrElse(16)
+    }
   }
 
   /**

project/SparkBuild.scala

@@ -330,6 +330,7 @@ object TestSettings {
     fork := true,
     javaOptions in Test += "-Dspark.test.home=" + sparkHome,
     javaOptions in Test += "-Dspark.testing=1",
+    javaOptions in Test += "-Dspark.ports.maxRetries=100",
     javaOptions in Test += "-Dsun.io.serialization.extendedDebugInfo=true",
     javaOptions in Test ++= System.getProperties.filter(_._1 startsWith "spark")
       .map { case (k,v) => s"-D$k=$v" }.toSeq,