[SPARK-41365][UI] Stages UI page fails to load for proxy in specific yarn environment

core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala

@@ -143,7 +143,8 @@ private[v1] class StagesResource extends BaseAppResource {
     @Context uriInfo: UriInfo):
   HashMap[String, Object] = {
     withUI { ui =>
-      val uriQueryParameters = uriInfo.getQueryParameters(true)
+      // Decode URI params twice here to avoid percent-encoding twice
+      val uriQueryParameters = UIUtils.decodeURLParameter(uriInfo.getQueryParameters(true))
       val totalRecords = uriQueryParameters.getFirst("numTasks")
       var isSearch = false
       var searchValue: String = null
@@ -204,7 +205,7 @@ private[v1] class StagesResource extends BaseAppResource {
       pageLength = queryParameters.getFirst("length").toInt
     }
     withUI(_.store.taskList(stageId, stageAttemptId, pageStartIndex, pageLength,
-      indexName(columnNameToSort), isAscendingStr.equalsIgnoreCase("asc")))
+      indexName(columnNameToSort), "asc".equalsIgnoreCase(isAscendingStr)))
   }
 
   // Filters task list based on search parameter

core/src/main/scala/org/apache/spark/ui/UIUtils.scala

@@ -24,13 +24,15 @@ import java.nio.charset.StandardCharsets.UTF_8
 import java.text.SimpleDateFormat
 import java.util.{Date, Locale, TimeZone}
 import javax.servlet.http.HttpServletRequest
-import javax.ws.rs.core.{MediaType, Response}
+import javax.ws.rs.core.{MediaType, MultivaluedMap, Response}
 
 import scala.collection.JavaConverters._
 import scala.util.control.NonFatal
 import scala.xml._
 import scala.xml.transform.{RewriteRule, RuleTransformer}
 
+import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap
+
 import org.apache.spark.internal.Logging
 import org.apache.spark.ui.scope.RDDOperationGraph
 
@@ -636,6 +638,22 @@ private[spark] object UIUtils extends Logging {
     param
   }
 
+  /**
+   * Decode URLParameter if URL is encoded by YARN-WebAppProxyServlet.
+   */
+  def decodeURLParameter(params: MultivaluedMap[String, String]): MultivaluedStringMap = {
+    val decodedParameters = new MultivaluedStringMap
+    params.forEach((encodeKey, encodeValues) => {
+      val decodeKey = decodeURLParameter(encodeKey)
+      val decodeValues = new java.util.LinkedList[String]
+      encodeValues.forEach(v => {
+        decodeValues.add(decodeURLParameter(v))
+      })
+      decodedParameters.addAll(decodeKey, decodeValues)
+    })
+    decodedParameters
+  }
+
   def getTimeZoneOffset() : Int =
     TimeZone.getDefault().getOffset(System.currentTimeMillis()) / 1000 / 60
 

core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala

@@ -26,6 +26,7 @@ import scala.xml.Node
 
 import com.gargoylesoftware.css.parser.CSSParseException
 import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler
+import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap
 import org.json4s._
 import org.json4s.jackson.JsonMethods
 import org.openqa.selenium.{By, WebDriver}
@@ -819,6 +820,39 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers {
     }
   }
 
+  test("SPARK-41365: Stage page can be accessed if URI was encoded twice") {
+    withSpark(newSparkContext()) { sc =>
+      val rdd = sc.parallelize(0 to 10, 10).repartition(10)
+      rdd.count()
+      eventually(timeout(5.seconds), interval(50.milliseconds)) {
+        val encodeParams = new MultivaluedStringMap
+        encodeParams.add("order%255B0%255D%255Bcolumn%255D", "Locality%2520Level")
+        encodeParams.add("order%255B0%255D%255Bcolumn%255D", "Executor%2520ID")
+        encodeParams.add("search%255Bvalue%255D", null)
+        val decodeParams = UIUtils.decodeURLParameter(encodeParams)
+        // assert no change in order
+        assert(decodeParams.getFirst("order[0][column]").equals("Locality Level"))
+        assert(decodeParams.get("order[0][column]").size() == 2)
+        assert(decodeParams.getFirst("search[value]").equals(""))
+
+        val decodeQuery = "draw=2&order[0][column]=4&order[0][dir]=asc&start=0&length=20" +
+          "&search[value]=&search[regex]=false&numTasks=10&columnIndexToSort=4" +
+          "&columnNameToSort=Locality Level"
+        val encodeOnceQuery = "draw=2&order%5B0%5D%5Bcolumn%5D=4&start=0&length=20" +
+          "&search%5Bvalue%5D=&search%5Bregex%5D=false&numTasks=10&columnIndexToSort=4" +
+          "&columnNameToSort=Locality%20Level"
+        val encodeTwiceQuery = "draw=2&order%255B0%255D%255Bcolumn%255D=4&start=0&length=20" +
+          "&search%255Bvalue%255D=&search%255Bregex%255D=false&numTasks=10&columnIndexToSort=4" +
+          "&columnNameToSort=Locality%2520Level"
+        val encodeOnceRes = Utils.tryWithResource(Source.fromURL(
+          apiUrl(sc.ui.get, "stages/0/0/taskTable?" + encodeOnceQuery)))(_.mkString)
+        val encodeTwiceRes = Utils.tryWithResource(Source.fromURL(
+          apiUrl(sc.ui.get, "stages/0/0/taskTable?" + encodeTwiceQuery)))(_.mkString)
+        assert(encodeOnceRes.equals(encodeTwiceRes))
+      }
+    }
+  }
+
   def goToUi(sc: SparkContext, path: String): Unit = {
     goToUi(sc.ui.get, path)
   }