diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala index 26dfa5af101e3..b23d0770a3601 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala @@ -143,7 +143,8 @@ private[v1] class StagesResource extends BaseAppResource { @Context uriInfo: UriInfo): HashMap[String, Object] = { withUI { ui => - val uriQueryParameters = uriInfo.getQueryParameters(true) + // Decode URI params twice here to avoid percent-encoding twice + val uriQueryParameters = UIUtils.decodeURLParameter(uriInfo.getQueryParameters(true)) val totalRecords = uriQueryParameters.getFirst("numTasks") var isSearch = false var searchValue: String = null @@ -204,7 +205,7 @@ private[v1] class StagesResource extends BaseAppResource { pageLength = queryParameters.getFirst("length").toInt } withUI(_.store.taskList(stageId, stageAttemptId, pageStartIndex, pageLength, - indexName(columnNameToSort), isAscendingStr.equalsIgnoreCase("asc"))) + indexName(columnNameToSort), "asc".equalsIgnoreCase(isAscendingStr))) } // Filters task list based on search parameter diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 111e8f8b3ad4b..aca03fde4c96d 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -24,13 +24,15 @@ import java.nio.charset.StandardCharsets.UTF_8 import java.text.SimpleDateFormat import java.util.{Date, Locale, TimeZone} import javax.servlet.http.HttpServletRequest -import javax.ws.rs.core.{MediaType, Response} +import javax.ws.rs.core.{MediaType, MultivaluedMap, Response} import scala.collection.JavaConverters._ import scala.util.control.NonFatal import scala.xml._ import scala.xml.transform.{RewriteRule, RuleTransformer} +import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap + import org.apache.spark.internal.Logging import org.apache.spark.ui.scope.RDDOperationGraph @@ -636,6 +638,22 @@ private[spark] object UIUtils extends Logging { param } + /** + * Decode URLParameter if URL is encoded by YARN-WebAppProxyServlet. + */ + def decodeURLParameter(params: MultivaluedMap[String, String]): MultivaluedStringMap = { + val decodedParameters = new MultivaluedStringMap + params.forEach((encodeKey, encodeValues) => { + val decodeKey = decodeURLParameter(encodeKey) + val decodeValues = new java.util.LinkedList[String] + encodeValues.forEach(v => { + decodeValues.add(decodeURLParameter(v)) + }) + decodedParameters.addAll(decodeKey, decodeValues) + }) + decodedParameters + } + def getTimeZoneOffset() : Int = TimeZone.getDefault().getOffset(System.currentTimeMillis()) / 1000 / 60 diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index 015f299fc6bdf..0a751e975ff8f 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -26,6 +26,7 @@ import scala.xml.Node import com.gargoylesoftware.css.parser.CSSParseException import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler +import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap import org.json4s._ import org.json4s.jackson.JsonMethods import org.openqa.selenium.{By, WebDriver} @@ -800,6 +801,39 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B } } + test("SPARK-41365: Stage page can be accessed if URI was encoded twice") { + withSpark(newSparkContext()) { sc => + val rdd = sc.parallelize(0 to 10, 10).repartition(10) + rdd.count() + eventually(timeout(5.seconds), interval(50.milliseconds)) { + val encodeParams = new MultivaluedStringMap + encodeParams.add("order%255B0%255D%255Bcolumn%255D", "Locality%2520Level") + encodeParams.add("order%255B0%255D%255Bcolumn%255D", "Executor%2520ID") + encodeParams.add("search%255Bvalue%255D", null) + val decodeParams = UIUtils.decodeURLParameter(encodeParams) + // assert no change in order + assert(decodeParams.getFirst("order[0][column]").equals("Locality Level")) + assert(decodeParams.get("order[0][column]").size() == 2) + assert(decodeParams.getFirst("search[value]").equals("")) + + val decodeQuery = "draw=2&order[0][column]=4&order[0][dir]=asc&start=0&length=20" + + "&search[value]=&search[regex]=false&numTasks=10&columnIndexToSort=4" + + "&columnNameToSort=Locality Level" + val encodeOnceQuery = "draw=2&order%5B0%5D%5Bcolumn%5D=4&start=0&length=20" + + "&search%5Bvalue%5D=&search%5Bregex%5D=false&numTasks=10&columnIndexToSort=4" + + "&columnNameToSort=Locality%20Level" + val encodeTwiceQuery = "draw=2&order%255B0%255D%255Bcolumn%255D=4&start=0&length=20" + + "&search%255Bvalue%255D=&search%255Bregex%255D=false&numTasks=10&columnIndexToSort=4" + + "&columnNameToSort=Locality%2520Level" + val encodeOnceRes = Utils.tryWithResource(Source.fromURL( + apiUrl(sc.ui.get, "stages/0/0/taskTable?" + encodeOnceQuery)))(_.mkString) + val encodeTwiceRes = Utils.tryWithResource(Source.fromURL( + apiUrl(sc.ui.get, "stages/0/0/taskTable?" + encodeTwiceQuery)))(_.mkString) + assert(encodeOnceRes.equals(encodeTwiceRes)) + } + } + } + def goToUi(sc: SparkContext, path: String): Unit = { goToUi(sc.ui.get, path) }