From 9d5c1baa4acfe66e04dc58a4ece5a97792db8749 Mon Sep 17 00:00:00 2001 From: Xutingjun Date: Mon, 1 Jun 2015 10:01:29 +0800 Subject: [PATCH 1/6] To make sure every time click the "App ID" on HistoryPage, the SparkUI will be refreshed. --- .../org/apache/spark/UIHandlerCollection.java | 217 ++++++++++++++++++ .../spark/deploy/history/HistoryServer.scala | 2 +- .../org/apache/spark/ui/JettyUtils.scala | 4 +- 3 files changed, 220 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/org/apache/spark/UIHandlerCollection.java diff --git a/core/src/main/java/org/apache/spark/UIHandlerCollection.java b/core/src/main/java/org/apache/spark/UIHandlerCollection.java new file mode 100644 index 000000000000..ea1e73c4074d --- /dev/null +++ b/core/src/main/java/org/apache/spark/UIHandlerCollection.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark; + +import org.eclipse.jetty.http.PathMap; +import org.eclipse.jetty.server.AsyncContinuation; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.HandlerContainer; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.server.handler.ContextHandlerCollection; +import org.eclipse.jetty.util.LazyList; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.log.Logger; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class UIHandlerCollection extends ContextHandlerCollection { + private static final Logger LOG = Log.getLogger(ContextHandlerCollection.class); + private volatile PathMap _contextMap; + private int flag = 0; + + public UIHandlerCollection() { + super(); + } + + public void mapContexts() { + PathMap contextMap = new PathMap(); + Handler[] branches = this.getHandlers(); + + for (int b = 0; branches != null && b < branches.length; ++b) { + Handler[] handlers = null; + if (branches[b] instanceof ContextHandler) { + handlers = new Handler[]{branches[b]}; + } else { + if (!(branches[b] instanceof HandlerContainer)) { + continue; + } + + handlers = ((HandlerContainer) branches[b]).getChildHandlersByClass(ContextHandler.class); + } + + for (int i = 0; i < handlers.length; ++i) { + ContextHandler handler = (ContextHandler) handlers[i]; + String contextPath = handler.getContextPath(); + if (contextPath == null || contextPath.indexOf(44) >= 0 || contextPath.startsWith("*")) { + throw new IllegalArgumentException("Illegal context spec:" + contextPath); + } + + if (!contextPath.startsWith("/")) { + contextPath = '/' + contextPath; + } + + if (contextPath.length() > 1) { + if (contextPath.endsWith("/")) { + contextPath = contextPath + "*"; + } else if (!contextPath.endsWith("/*")) { + contextPath = contextPath + "/*"; + } + } + + Object contexts = contextMap.get(contextPath); + String[] vhosts = handler.getVirtualHosts(); + if (vhosts != null && vhosts.length > 0) { + Object var13; + if (contexts instanceof Map) { + var13 = (Map) contexts; + } else { + var13 = new HashMap(); + ((Map) var13).put("*", contexts); + contextMap.put(contextPath, var13); + } + + for (int j = 0; j < vhosts.length; ++j) { + String vhost = vhosts[j]; + contexts = ((Map) var13).get(vhost); + contexts = LazyList.add(contexts, branches[b]); + ((Map) var13).put(vhost, contexts); + } + } else if (contexts instanceof Map) { + Map hosts = (Map) contexts; + contexts = hosts.get("*"); + contexts = LazyList.add(contexts, branches[b]); + hosts.put("*", contexts); + } else { + contexts = LazyList.add(contexts, branches[b]); + contextMap.put(contextPath, contexts); + } + } + } + + this._contextMap = contextMap; + } + + public void setHandlers(Handler[] handlers) { + this._contextMap = null; + super.setHandlers(handlers); + if (this.isStarted()) { + this.mapContexts(); + } + + } + + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + Handler[] handlers = super.getHandlers(); + if (handlers != null && handlers.length != 0) { + AsyncContinuation async = baseRequest.getAsyncContinuation(); + if (async.isAsync()) { + ContextHandler map = async.getContextHandler(); + if (map != null) { + map.handle(target, baseRequest, request, response); + return; + } + } + + PathMap var16 = this._contextMap; + if (var16 != null && target != null && target.startsWith("/")) { + // To make sure every time click the "App ID" on HistoryPage, the SparkUI will be refreshed. + if(target.matches("^/history/application_[^/]+")) { + if(flag == 0) { + Handler handler = (Handler) var16.get("/history/*"); + handler.handle(target, baseRequest, request, response); + flag = 1; + return; + } + if(flag == 1) { + Handler handler = (Handler) var16.get(target + "/*"); + handler.handle(target, baseRequest, request, response); + flag = 0; + return; + } + } + else { + Object var17 = var16.getLazyMatches(target); + + for (int i1 = 0; i1 < LazyList.size(var17); ++i1) { + Map.Entry entry = (Map.Entry) LazyList.get(var17, i1); + Object list = entry.getValue(); + if (!(list instanceof Map)) { + for (int var18 = 0; var18 < LazyList.size(list); ++var18) { + Handler var19 = (Handler) LazyList.get(list, var18); + var19.handle(target, baseRequest, request, response); + if (baseRequest.isHandled()) { + return; + } + } + } else { + Map j = (Map) list; + String handler = this.normalizeHostname(request.getServerName()); + list = j.get(handler); + + int j1; + Handler handler1; + for (j1 = 0; j1 < LazyList.size(list); ++j1) { + handler1 = (Handler) LazyList.get(list, j1); + handler1.handle(target, baseRequest, request, response); + if (baseRequest.isHandled()) { + return; + } + } + + list = j.get("*." + handler.substring(handler.indexOf(".") + 1)); + + for (j1 = 0; j1 < LazyList.size(list); ++j1) { + handler1 = (Handler) LazyList.get(list, j1); + handler1.handle(target, baseRequest, request, response); + if (baseRequest.isHandled()) { + return; + } + } + + list = j.get("*"); + + for (j1 = 0; j1 < LazyList.size(list); ++j1) { + handler1 = (Handler) LazyList.get(list, j1); + handler1.handle(target, baseRequest, request, response); + if (baseRequest.isHandled()) { + return; + } + } + } + } + } + } else { + for (int i = 0; i < handlers.length; ++i) { + handlers[i].handle(target, baseRequest, request, response); + if (baseRequest.isHandled()) { + return; + } + } + } + } + } + + private String normalizeHostname(String host) { + return host == null ? null : (host.endsWith(".") ? host.substring(0, host.length() - 1) : host); + } +} diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index 5a0eb585a904..deb1ba1b25fc 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -182,7 +182,7 @@ class HistoryServer( private def loadAppUi(appId: String, attemptId: Option[String]): Boolean = { try { - appCache.get(appId + attemptId.map { id => s"/$id" }.getOrElse("")) + appCache.refresh(appId + attemptId.map { id => s"/$id" }.getOrElse("")) true } catch { case e: Exception => e.getCause() match { diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala index 06e616220c70..3ee55ade80b8 100644 --- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala @@ -31,7 +31,7 @@ import org.eclipse.jetty.util.thread.QueuedThreadPool import org.json4s.JValue import org.json4s.jackson.JsonMethods.{pretty, render} -import org.apache.spark.{Logging, SecurityManager, SparkConf} +import org.apache.spark.{UIHandlerCollection, Logging, SecurityManager, SparkConf} import org.apache.spark.util.Utils /** @@ -210,7 +210,7 @@ private[spark] object JettyUtils extends Logging { conf: SparkConf, serverName: String = ""): ServerInfo = { - val collection = new ContextHandlerCollection + val collection = new UIHandlerCollection collection.setHandlers(handlers.toArray) addFilters(handlers, conf) From a097bd81807c6079a5f7cd5f6f7fb3e09fea8bd7 Mon Sep 17 00:00:00 2001 From: Xutingjun Date: Mon, 1 Jun 2015 16:48:24 +0800 Subject: [PATCH 2/6] When app is imcompleted, will refresh; If completed, won't refresh --- .../history/ApplicationHistoryProvider.scala | 2 ++ .../spark/deploy/history/FsHistoryProvider.scala | 15 +++++++++++++++ .../spark/deploy/history/HistoryServer.scala | 12 +++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala index 298a8201960d..0bbbf2e0366a 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala @@ -62,4 +62,6 @@ private[history] abstract class ApplicationHistoryProvider { */ def getConfig(): Map[String, String] = Map() + def getAppStatus(appid: String): Boolean + } diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala index 45c2be34c868..cb5fea5806cb 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala @@ -80,6 +80,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) // List of application logs to be deleted by event log cleaner. private var attemptsToClean = new mutable.ListBuffer[FsApplicationAttemptInfo] + private val appStatus = new mutable.HashMap[String, Boolean]() + // Constants used to parse Spark 1.0.0 log directories. private[history] val LOG_PREFIX = "EVENT_LOG_" private[history] val SPARK_VERSION_PREFIX = EventLoggingListener.SPARK_VERSION_KEY + "_" @@ -390,6 +392,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) val appCompleted = isApplicationCompleted(eventLog) bus.addListener(appListener) bus.replay(logInput, logPath.toString, !appCompleted) + appStatus.put(logPath.getName(), appCompleted) new FsApplicationAttemptInfo( logPath.getName(), appListener.appName.getOrElse(NOT_STARTED), @@ -474,6 +477,18 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) } } + def getAppStatus(appid: String): Boolean = { + if (appStatus.keySet.contains(appid)) { + return true + } + else if (appStatus.contains(appid + EventLoggingListener.IN_PROGRESS)) { + return false + } + else { + throw new Exception(s"no app with key $appid.") + } + } + } private object FsHistoryProvider { diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index deb1ba1b25fc..38714b535d9f 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -31,6 +31,8 @@ import org.apache.spark.ui.{SparkUI, UIUtils, WebUI} import org.apache.spark.ui.JettyUtils._ import org.apache.spark.util.{SignalLogger, Utils} +import scala.collection.mutable + /** * A web server that renders SparkUIs of completed applications. * @@ -49,6 +51,8 @@ class HistoryServer( port: Int) extends WebUI(securityManager, port, conf) with Logging with UIRoot { + private val loadedAppStatus = new mutable.HashMap[String, Boolean]() + // How many applications to retain private val retainedApplications = conf.getInt("spark.history.retainedApplications", 50) @@ -182,7 +186,13 @@ class HistoryServer( private def loadAppUi(appId: String, attemptId: Option[String]): Boolean = { try { - appCache.refresh(appId + attemptId.map { id => s"/$id" }.getOrElse("")) + if (!loadedAppStatus.get(appId).isDefined) { + loadedAppStatus.put(appId, false) + } + if (!loadedAppStatus.get(appId).get) { + appCache.refresh(appId + attemptId.map { id => s"/$id" }.getOrElse("")) + loadedAppStatus.update(appId, provider.getAppStatus(appId)) + } true } catch { case e: Exception => e.getCause() match { From fdcb6a1c12710b051a60e5d1eea36e094538b373 Mon Sep 17 00:00:00 2001 From: Xutingjun Date: Tue, 2 Jun 2015 16:25:05 +0800 Subject: [PATCH 3/6] change the requset url from historypage to sparkui --- .../org/apache/spark/UIHandlerCollection.java | 217 ------------------ .../deploy/history/FsHistoryProvider.scala | 4 +- .../spark/deploy/history/HistoryServer.scala | 18 +- .../org/apache/spark/ui/JettyUtils.scala | 4 +- .../scala/org/apache/spark/ui/SparkUI.scala | 1 + 5 files changed, 19 insertions(+), 225 deletions(-) delete mode 100644 core/src/main/java/org/apache/spark/UIHandlerCollection.java diff --git a/core/src/main/java/org/apache/spark/UIHandlerCollection.java b/core/src/main/java/org/apache/spark/UIHandlerCollection.java deleted file mode 100644 index ea1e73c4074d..000000000000 --- a/core/src/main/java/org/apache/spark/UIHandlerCollection.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark; - -import org.eclipse.jetty.http.PathMap; -import org.eclipse.jetty.server.AsyncContinuation; -import org.eclipse.jetty.server.Handler; -import org.eclipse.jetty.server.HandlerContainer; -import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.handler.ContextHandler; -import org.eclipse.jetty.server.handler.ContextHandlerCollection; -import org.eclipse.jetty.util.LazyList; -import org.eclipse.jetty.util.log.Log; -import org.eclipse.jetty.util.log.Logger; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class UIHandlerCollection extends ContextHandlerCollection { - private static final Logger LOG = Log.getLogger(ContextHandlerCollection.class); - private volatile PathMap _contextMap; - private int flag = 0; - - public UIHandlerCollection() { - super(); - } - - public void mapContexts() { - PathMap contextMap = new PathMap(); - Handler[] branches = this.getHandlers(); - - for (int b = 0; branches != null && b < branches.length; ++b) { - Handler[] handlers = null; - if (branches[b] instanceof ContextHandler) { - handlers = new Handler[]{branches[b]}; - } else { - if (!(branches[b] instanceof HandlerContainer)) { - continue; - } - - handlers = ((HandlerContainer) branches[b]).getChildHandlersByClass(ContextHandler.class); - } - - for (int i = 0; i < handlers.length; ++i) { - ContextHandler handler = (ContextHandler) handlers[i]; - String contextPath = handler.getContextPath(); - if (contextPath == null || contextPath.indexOf(44) >= 0 || contextPath.startsWith("*")) { - throw new IllegalArgumentException("Illegal context spec:" + contextPath); - } - - if (!contextPath.startsWith("/")) { - contextPath = '/' + contextPath; - } - - if (contextPath.length() > 1) { - if (contextPath.endsWith("/")) { - contextPath = contextPath + "*"; - } else if (!contextPath.endsWith("/*")) { - contextPath = contextPath + "/*"; - } - } - - Object contexts = contextMap.get(contextPath); - String[] vhosts = handler.getVirtualHosts(); - if (vhosts != null && vhosts.length > 0) { - Object var13; - if (contexts instanceof Map) { - var13 = (Map) contexts; - } else { - var13 = new HashMap(); - ((Map) var13).put("*", contexts); - contextMap.put(contextPath, var13); - } - - for (int j = 0; j < vhosts.length; ++j) { - String vhost = vhosts[j]; - contexts = ((Map) var13).get(vhost); - contexts = LazyList.add(contexts, branches[b]); - ((Map) var13).put(vhost, contexts); - } - } else if (contexts instanceof Map) { - Map hosts = (Map) contexts; - contexts = hosts.get("*"); - contexts = LazyList.add(contexts, branches[b]); - hosts.put("*", contexts); - } else { - contexts = LazyList.add(contexts, branches[b]); - contextMap.put(contextPath, contexts); - } - } - } - - this._contextMap = contextMap; - } - - public void setHandlers(Handler[] handlers) { - this._contextMap = null; - super.setHandlers(handlers); - if (this.isStarted()) { - this.mapContexts(); - } - - } - - public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - Handler[] handlers = super.getHandlers(); - if (handlers != null && handlers.length != 0) { - AsyncContinuation async = baseRequest.getAsyncContinuation(); - if (async.isAsync()) { - ContextHandler map = async.getContextHandler(); - if (map != null) { - map.handle(target, baseRequest, request, response); - return; - } - } - - PathMap var16 = this._contextMap; - if (var16 != null && target != null && target.startsWith("/")) { - // To make sure every time click the "App ID" on HistoryPage, the SparkUI will be refreshed. - if(target.matches("^/history/application_[^/]+")) { - if(flag == 0) { - Handler handler = (Handler) var16.get("/history/*"); - handler.handle(target, baseRequest, request, response); - flag = 1; - return; - } - if(flag == 1) { - Handler handler = (Handler) var16.get(target + "/*"); - handler.handle(target, baseRequest, request, response); - flag = 0; - return; - } - } - else { - Object var17 = var16.getLazyMatches(target); - - for (int i1 = 0; i1 < LazyList.size(var17); ++i1) { - Map.Entry entry = (Map.Entry) LazyList.get(var17, i1); - Object list = entry.getValue(); - if (!(list instanceof Map)) { - for (int var18 = 0; var18 < LazyList.size(list); ++var18) { - Handler var19 = (Handler) LazyList.get(list, var18); - var19.handle(target, baseRequest, request, response); - if (baseRequest.isHandled()) { - return; - } - } - } else { - Map j = (Map) list; - String handler = this.normalizeHostname(request.getServerName()); - list = j.get(handler); - - int j1; - Handler handler1; - for (j1 = 0; j1 < LazyList.size(list); ++j1) { - handler1 = (Handler) LazyList.get(list, j1); - handler1.handle(target, baseRequest, request, response); - if (baseRequest.isHandled()) { - return; - } - } - - list = j.get("*." + handler.substring(handler.indexOf(".") + 1)); - - for (j1 = 0; j1 < LazyList.size(list); ++j1) { - handler1 = (Handler) LazyList.get(list, j1); - handler1.handle(target, baseRequest, request, response); - if (baseRequest.isHandled()) { - return; - } - } - - list = j.get("*"); - - for (j1 = 0; j1 < LazyList.size(list); ++j1) { - handler1 = (Handler) LazyList.get(list, j1); - handler1.handle(target, baseRequest, request, response); - if (baseRequest.isHandled()) { - return; - } - } - } - } - } - } else { - for (int i = 0; i < handlers.length; ++i) { - handlers[i].handle(target, baseRequest, request, response); - if (baseRequest.isHandled()) { - return; - } - } - } - } - } - - private String normalizeHostname(String host) { - return host == null ? null : (host.endsWith(".") ? host.substring(0, host.length() - 1) : host); - } -} diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala index cb5fea5806cb..36b19a04cc69 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala @@ -485,7 +485,9 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) return false } else { - throw new Exception(s"no app with key $appid.") + val e = new NoSuchElementException(s"no app with key $appid.") + e.initCause(new NoSuchElementException) + throw e } } diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index 38714b535d9f..f93deaa80bb1 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -64,6 +64,12 @@ class HistoryServer( .getAppUI(parts(0), if (parts.length > 1) Some(parts(1)) else None) .getOrElse(throw new NoSuchElementException(s"no app with key $key")) attachSparkUI(ui) + val newUrl = if(parts.length > 1) "/history/" + parts(0) + "/" + parts(1) else "/history/" + parts(0) + for(hanler <- getHandlers) { + if (hanler != null && hanler.getContextPath.equals(newUrl)) { + detachHandler(hanler) + } + } ui } } @@ -106,7 +112,7 @@ class HistoryServer( // Note we don't use the UI retrieved from the cache; the cache loader above will register // the app's UI, and all we need to do is redirect the user to the same URI that was // requested, and the proper data should be served at that point. - res.sendRedirect(res.encodeRedirectURL(req.getRequestURI())) + res.sendRedirect(res.encodeRedirectURL(req.getRequestURI() + "/ui")) } // SPARK-5983 ensure TRACE is not supported @@ -186,12 +192,14 @@ class HistoryServer( private def loadAppUi(appId: String, attemptId: Option[String]): Boolean = { try { - if (!loadedAppStatus.get(appId).isDefined) { - loadedAppStatus.put(appId, false) + val app_attemp_id = appId + attemptId.map { id => s"_$id" }.getOrElse("") + if (!loadedAppStatus.get(app_attemp_id).isDefined) { + loadedAppStatus.put(app_attemp_id, provider.getAppStatus(app_attemp_id)) + appCache.refresh(appId + attemptId.map { id => s"/$id" }.getOrElse("")) } - if (!loadedAppStatus.get(appId).get) { + if (!loadedAppStatus.get(app_attemp_id).get) { + loadedAppStatus.update(app_attemp_id, provider.getAppStatus(app_attemp_id)) appCache.refresh(appId + attemptId.map { id => s"/$id" }.getOrElse("")) - loadedAppStatus.update(appId, provider.getAppStatus(appId)) } true } catch { diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala index 3ee55ade80b8..06e616220c70 100644 --- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala @@ -31,7 +31,7 @@ import org.eclipse.jetty.util.thread.QueuedThreadPool import org.json4s.JValue import org.json4s.jackson.JsonMethods.{pretty, render} -import org.apache.spark.{UIHandlerCollection, Logging, SecurityManager, SparkConf} +import org.apache.spark.{Logging, SecurityManager, SparkConf} import org.apache.spark.util.Utils /** @@ -210,7 +210,7 @@ private[spark] object JettyUtils extends Logging { conf: SparkConf, serverName: String = ""): ServerInfo = { - val collection = new UIHandlerCollection + val collection = new ContextHandlerCollection collection.setHandlers(handlers.toArray) addFilters(handlers, conf) diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala index 3788916cf39b..f7e049980ec4 100644 --- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala @@ -65,6 +65,7 @@ private[spark] class SparkUI private ( attachTab(new ExecutorsTab(this)) attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static")) attachHandler(createRedirectHandler("/", "/jobs", basePath = basePath)) + attachHandler(createRedirectHandler("/ui", "/jobs", basePath = basePath)) attachHandler(ApiRootResource.getServletHandler(this)) // This should be POST only, but, the YARN AM proxy won't proxy POSTs attachHandler(createRedirectHandler( From 45145cb72dc129610655bcc407e8af4dc2c42d86 Mon Sep 17 00:00:00 2001 From: xutingjun Date: Tue, 2 Jun 2015 17:27:28 +0800 Subject: [PATCH 4/6] diff handler attach between sparkui and historyserver --- .../org/apache/spark/deploy/history/HistoryServer.scala | 6 ------ core/src/main/scala/org/apache/spark/ui/SparkUI.scala | 7 +++++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index f93deaa80bb1..eeeb05a9660d 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -64,12 +64,6 @@ class HistoryServer( .getAppUI(parts(0), if (parts.length > 1) Some(parts(1)) else None) .getOrElse(throw new NoSuchElementException(s"no app with key $key")) attachSparkUI(ui) - val newUrl = if(parts.length > 1) "/history/" + parts(0) + "/" + parts(1) else "/history/" + parts(0) - for(hanler <- getHandlers) { - if (hanler != null && hanler.getContextPath.equals(newUrl)) { - detachHandler(hanler) - } - } ui } } diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala index f7e049980ec4..ed4648bf579d 100644 --- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala @@ -64,8 +64,11 @@ private[spark] class SparkUI private ( attachTab(new EnvironmentTab(this)) attachTab(new ExecutorsTab(this)) attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static")) - attachHandler(createRedirectHandler("/", "/jobs", basePath = basePath)) - attachHandler(createRedirectHandler("/ui", "/jobs", basePath = basePath)) + if (sc.isDefined) { + attachHandler(createRedirectHandler("/", "/jobs", basePath = basePath)) + } else { + attachHandler(createRedirectHandler("/ui", "/jobs", basePath = basePath)) + } attachHandler(ApiRootResource.getServletHandler(this)) // This should be POST only, but, the YARN AM proxy won't proxy POSTs attachHandler(createRedirectHandler( From f9de42e0c005a6ac81af1f848a2c3bda2d3850e7 Mon Sep 17 00:00:00 2001 From: Xutingjun Date: Fri, 5 Jun 2015 10:11:07 +0800 Subject: [PATCH 5/6] set executor cores into system --- core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 8cf4d58847d8..7521f517e94b 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -446,6 +446,7 @@ object SparkSubmit { OptionAssigner(args.keytab, YARN, CLUSTER, clOption = "--keytab"), // Other options + OptionAssigner(args.executorCores, YARN, CLUSTER, sysProp = "spark.executor.cores"), OptionAssigner(args.executorCores, STANDALONE, ALL_DEPLOY_MODES, sysProp = "spark.executor.cores"), OptionAssigner(args.executorMemory, STANDALONE | MESOS | YARN, ALL_DEPLOY_MODES, From b290a7ce83b84f1cab00d6986745b43e7316f696 Mon Sep 17 00:00:00 2001 From: Xutingjun Date: Fri, 5 Jun 2015 10:29:15 +0800 Subject: [PATCH 6/6] fix bug --- core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 1 - .../scala/org/apache/spark/deploy/history/HistoryServer.scala | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 7521f517e94b..8cf4d58847d8 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -446,7 +446,6 @@ object SparkSubmit { OptionAssigner(args.keytab, YARN, CLUSTER, clOption = "--keytab"), // Other options - OptionAssigner(args.executorCores, YARN, CLUSTER, sysProp = "spark.executor.cores"), OptionAssigner(args.executorCores, STANDALONE, ALL_DEPLOY_MODES, sysProp = "spark.executor.cores"), OptionAssigner(args.executorMemory, STANDALONE | MESOS | YARN, ALL_DEPLOY_MODES, diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index eeeb05a9660d..695f44f9c74e 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -190,8 +190,7 @@ class HistoryServer( if (!loadedAppStatus.get(app_attemp_id).isDefined) { loadedAppStatus.put(app_attemp_id, provider.getAppStatus(app_attemp_id)) appCache.refresh(appId + attemptId.map { id => s"/$id" }.getOrElse("")) - } - if (!loadedAppStatus.get(app_attemp_id).get) { + } else if (!loadedAppStatus.get(app_attemp_id).get) { loadedAppStatus.update(app_attemp_id, provider.getAppStatus(app_attemp_id)) appCache.refresh(appId + attemptId.map { id => s"/$id" }.getOrElse("")) }