Merge remote-tracking branch 'upstream/master' into SPARK-2177

Author: yhuai

core/src/main/scala/org/apache/spark/storage/BlockManager.scala

@@ -363,6 +363,13 @@ private[spark] class BlockManager(
     val info = blockInfo.get(blockId).orNull
     if (info != null) {
       info.synchronized {
+        // Double check to make sure the block is still there, since removeBlock
+        // method also synchronizes on BlockInfo object, so the block might have
+        // been removed when we actually come here.
+        if (blockInfo.get(blockId).isEmpty) {
+          logDebug(s"Block $blockId had been removed")
+          return None
+        }
 
         // If another thread is writing the block, wait for it to become ready.
         if (!info.waitForReady()) {

sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala

@@ -307,6 +307,8 @@ class SQLContext(@transient val sparkContext: SparkContext)
     lazy val optimizedPlan = optimizer(analyzed)
     // TODO: Don't just pick the first one...
     lazy val sparkPlan = planner(optimizedPlan).next()
+    // executedPlan should not be used to initialize any SparkPlan. It should be
+    // only used for execution.
     lazy val executedPlan: SparkPlan = prepareForExecution(sparkPlan)
 
     /** Internal version of the RDD. Avoids copies and has no schema */

sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala

@@ -251,8 +251,8 @@ private[sql] abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
       case logical.SetCommand(key, value) =>
         Seq(execution.SetCommand(key, value, plan.output)(context))
       case logical.ExplainCommand(child) =>
-        val executedPlan = context.executePlan(child).executedPlan
-        Seq(execution.ExplainCommand(executedPlan, plan.output)(context))
+        val sparkPlan = context.executePlan(child).sparkPlan
+        Seq(execution.ExplainCommand(sparkPlan, plan.output)(context))
       case logical.CacheCommand(tableName, cache) =>
         Seq(execution.CacheCommand(tableName, cache)(context))
       case _ => Nil