Cleanup.

Author: harishreedharan

core/src/main/scala/org/apache/spark/SparkContext.scala

@@ -34,7 +34,6 @@ import scala.collection.mutable.HashMap
 import scala.reflect.{ClassTag, classTag}
 import scala.util.control.NonFatal
 
-import com.google.common.annotations.VisibleForTesting
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.io.{ArrayWritable, BooleanWritable, BytesWritable, DoubleWritable,
@@ -226,7 +225,6 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
   private var _jars: Seq[String] = _
   private var _files: Seq[String] = _
   private var _shutdownHookRef: AnyRef = _
-  private var _logUrls: Option[Predef.Map[String, String]] = None
 
   /* ------------------------------------------------------------------------------------- *
    | Accessors and public fields. These provide access to the internal state of the        |
@@ -317,15 +315,6 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
     _dagScheduler = ds
   }
 
-  @VisibleForTesting
-  private[spark] def logUrls: Option[Predef.Map[String, String]] = _logUrls
-
-  @VisibleForTesting
-  private[spark] def logUrls_=(logUrlsMap: Option[Predef.Map[String, String]]): Unit = {
-    _logUrls = logUrlsMap
-    logInfo(s"Setting log urls to ${_logUrls.get.mkString(" | ")}")
-  }
-
   def applicationId: String = _applicationId
   def applicationAttemptId: Option[String] = _applicationAttemptId
 

core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala

@@ -52,7 +52,7 @@ private[spark] trait SchedulerBackend {
   /**
    * Get the URLs for the driver logs. These URLs are used to display the links in the UI
    * Executors tab for the driver.
-   * @return
+   * @return The urls to the logs of the driver
    */
   def getDriverLogUrls: Option[Map[String, String]] = None
 

yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala

@@ -21,17 +21,13 @@ import scala.util.control.NonFatal
 
 import java.io.{File, IOException}
 import java.lang.reflect.InvocationTargetException
-import java.net.{NetworkInterface, Socket, URL}
+import java.net.{Socket, URL}
 import java.util.concurrent.atomic.AtomicReference
 
-import scala.collection.JavaConverters._
-
 import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.hadoop.yarn.api._
 import org.apache.hadoop.yarn.api.records._
-import org.apache.hadoop.yarn.client.api.YarnClient
 import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.hadoop.yarn.util.ConverterUtils
 
 import org.apache.spark.rpc._
 import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkContext, SparkEnv}
@@ -266,26 +262,25 @@ private[spark] class ApplicationMaster(
 
   private def runDriver(securityMgr: SecurityManager): Unit = {
     addAmIpFilter()
-    if (isClusterMode) {
-      userClassThread = startUserApplication()
-      // This a bit hacky, but we need to wait until the spark.driver.port property has
-      // been set by the Thread executing the user class.
-      val sc = waitForSparkContextInitialized()
+    userClassThread = startUserApplication()
 
-      // If there is no SparkContext at this point, just fail the app.
-      if (sc == null) {
-        finish(FinalApplicationStatus.FAILED,
-          ApplicationMaster.EXIT_SC_NOT_INITED,
-          "Timed out waiting for SparkContext.")
-      } else {
-        rpcEnv = sc.env.rpcEnv
-        runAMEndpoint(
-          sc.getConf.get("spark.driver.host"),
-          sc.getConf.get("spark.driver.port"),
-          isClusterMode = true)
-        registerAM(sc.ui.map(_.appUIAddress).getOrElse(""), securityMgr)
-        userClassThread.join()
-      }
+    // This a bit hacky, but we need to wait until the spark.driver.port property has
+    // been set by the Thread executing the user class.
+    val sc = waitForSparkContextInitialized()
+
+    // If there is no SparkContext at this point, just fail the app.
+    if (sc == null) {
+      finish(FinalApplicationStatus.FAILED,
+        ApplicationMaster.EXIT_SC_NOT_INITED,
+        "Timed out waiting for SparkContext.")
+    } else {
+      rpcEnv = sc.env.rpcEnv
+      runAMEndpoint(
+        sc.getConf.get("spark.driver.host"),
+        sc.getConf.get("spark.driver.port"),
+        isClusterMode = true)
+      registerAM(sc.ui.map(_.appUIAddress).getOrElse(""), securityMgr)
+      userClassThread.join()
     }
   }
 

yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterSchedulerBackend.scala

@@ -21,11 +21,9 @@ import java.net.NetworkInterface
 
 import scala.collection.JavaConverters._
 
-import org.apache.hadoop.yarn.api.ApplicationConstants
 import org.apache.hadoop.yarn.api.records.NodeState
 import org.apache.hadoop.yarn.client.api.YarnClient
 import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.hadoop.yarn.util.ConverterUtils
 
 import org.apache.spark.SparkContext
 import org.apache.spark.deploy.yarn.YarnSparkHadoopUtil