[SPARK-16987] [None] Add spark-default.conf property to define https port for spark history server

core/src/main/scala/org/apache/spark/SSLOptions.scala

@@ -56,6 +56,7 @@ private[spark] case class SSLOptions(
     trustStorePassword: Option[String] = None,
     trustStoreType: Option[String] = None,
     protocol: Option[String] = None,
+    port: Int = 0,
     enabledAlgorithms: Set[String] = Set.empty)
     extends Logging {
 
@@ -147,6 +148,7 @@ private[spark] object SSLOptions extends Logging {
    * $ - `[ns].trustStorePassword` - a password to the trust-store file
    * $ - `[ns].trustStoreType` - the type of trust-store
    * $ - `[ns].protocol` - a protocol name supported by a particular Java version
+   * $ - `[ns].port` - a port number
    * $ - `[ns].enabledAlgorithms` - a comma separated list of ciphers
    *
    * For a list of protocols and ciphers supported by particular Java versions, you may go to
@@ -191,6 +193,8 @@ private[spark] object SSLOptions extends Logging {
     val protocol = conf.getOption(s"$ns.protocol")
         .orElse(defaults.flatMap(_.protocol))
 
+    val port = conf.getInt(s"$ns.port", defaultValue = defaults.map(_.port).getOrElse(0))
+
     val enabledAlgorithms = conf.getOption(s"$ns.enabledAlgorithms")
         .map(_.split(",").map(_.trim).filter(_.nonEmpty).toSet)
         .orElse(defaults.map(_.enabledAlgorithms))
@@ -207,6 +211,7 @@ private[spark] object SSLOptions extends Logging {
       trustStorePassword,
       trustStoreType,
       protocol,
+      port,
       enabledAlgorithms)
   }
 

core/src/main/scala/org/apache/spark/ui/JettyUtils.scala

@@ -307,15 +307,26 @@ private[spark] object JettyUtils extends Logging {
       connectors += httpConnector
 
       sslOptions.createJettySslContextFactory().foreach { factory =>
-        // If the new port wraps around, do not try a privileged port.
+
+        require(sslOptions.port == 0 || (1024 <= sslOptions.port && sslOptions.port < 65536),
+          "securePort should be between 1024 and 65535 (inclusive)," +
+            " or 0 for determined automatically.")
+
         val securePort =
           if (currentPort != 0) {
-            (currentPort + 400 - 1024) % (65536 - 1024) + 1024
+            // If the new port wraps around, do not try a privileged port.
+            if (1024 <= sslOptions.port && sslOptions.port < 65536) {
+              sslOptions.port
+            } else {
+              // If the new port wraps around, do not try a privilege port
+              (currentPort + 400 - 1024) % (65536 - 1024) + 1024
+            }
           } else {
             0
           }
         val scheme = "https"
-        // Create a connector on port securePort to listen for HTTPS requests
+        // Create a connector on port securePort to listen for HTTPS requests.
+
         val connector = new ServerConnector(server, factory)
         connector.setPort(securePort)
 

core/src/test/scala/org/apache/spark/SSLOptionsSuite.scala

@@ -113,6 +113,7 @@ class SSLOptionsSuite extends SparkFunSuite with BeforeAndAfterAll {
       "TLS_RSA_WITH_AES_128_CBC_SHA, TLS_RSA_WITH_AES_256_CBC_SHA")
     conf.set("spark.ui.ssl.enabledAlgorithms", "ABC, DEF")
     conf.set("spark.ssl.protocol", "SSLv3")
+    conf.set("spark.ssl.port", "18999")
 
     val defaultOpts = SSLOptions.parse(conf, "spark.ssl", defaults = None)
     val opts = SSLOptions.parse(conf, "spark.ui.ssl", defaults = Some(defaultOpts))
@@ -128,6 +129,7 @@ class SSLOptionsSuite extends SparkFunSuite with BeforeAndAfterAll {
     assert(opts.keyStorePassword === Some("12345"))
     assert(opts.keyPassword === Some("password"))
     assert(opts.protocol === Some("SSLv3"))
+    assert(opts.port === 18999)
     assert(opts.enabledAlgorithms === Set("ABC", "DEF"))
   }
 

docs/configuration.md

@@ -1663,6 +1663,16 @@ Apart from these, the following properties are also available, and may be useful
             page.
         </td>
     </tr>
+    <tr>
+        <td><code>spark.ssl.port</code></td>
+        <td>0</td>
+        <td>
+            Port number to listen on for SSL connections.
+            The SSL port should be between 1024 and 65535 (inclusive). 
+            Default value of 0 means the port will be determined automatically.
+            Attention that the port should be separated for each particular protocols.
+        </td>
+    </tr>
     <tr>
         <td><code>spark.ssl.needClientAuth</code></td>
         <td>false</td>

mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala

@@ -66,7 +66,7 @@ private[ml] trait VectorIndexerParams extends Params with HasInputCol with HasOu
  *     - This helps process a dataset of unknown vectors into a dataset with some continuous
  *       features and some categorical features. The choice between continuous and categorical
  *       is based upon a maxCategories parameter.
- *     - Set maxCategories to the maximum number of categorical any categorical feature should have.
+ *     - Set maxCategories to the maximum number of categories which categorical feature should have.
  *     - E.g.: Feature 0 has unique values {-1.0, 0.0}, and feature 1 values {1.0, 3.0, 5.0}.
  *       If maxCategories = 2, then feature 0 will be declared categorical and use indices {0, 1},
  *       and feature 1 will be declared continuous.

mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala

@@ -85,8 +85,10 @@ private[ml] trait DecisionTreeParams extends PredictorParams
    * (default = 256 MB)
    * @group expertParam
    */
-  final val maxMemoryInMB: IntParam = new IntParam(this, "maxMemoryInMB",
-    "Maximum memory in MB allocated to histogram aggregation.",
+  final val maxMemoryInMB: IntParam = new IntParam(this, "maxMemoryInMB", "Maximum memory in MB" +
+    " allocated to histogram aggregation." +
+    " If too small, then 1 node will be split per iteration," +
+    " and its aggregates may exceed this size.",
     ParamValidators.gtEq(0))
 
   /**