[WIP][SPARK-53757] Upgrade Jetty to 12.0.12

connector/kafka-0-10-sql/pom.xml

@@ -132,8 +132,8 @@
       <scope>test</scope>
     </dependency>
      <dependency>
-        <groupId>org.eclipse.jetty</groupId>
-        <artifactId>jetty-servlet</artifactId>
+        <groupId>org.eclipse.jetty.ee10</groupId>
+        <artifactId>jetty-ee10-servlet</artifactId>
         <version>${jetty.version}</version>
         <scope>test</scope>
       </dependency>

core/pom.xml

@@ -163,13 +163,13 @@
       <scope>compile</scope>
     </dependency>
     <dependency>
-      <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-servlet</artifactId>
+      <groupId>org.eclipse.jetty.ee10</groupId>
+      <artifactId>jetty-ee10-servlet</artifactId>
       <scope>compile</scope>
     </dependency>
     <dependency>
-      <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-proxy</artifactId>
+      <groupId>org.eclipse.jetty.ee10</groupId>
+      <artifactId>jetty-ee10-proxy</artifactId>
       <scope>compile</scope>
     </dependency>
     <dependency>
@@ -178,8 +178,8 @@
       <scope>compile</scope>
     </dependency>
     <dependency>
-      <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-servlets</artifactId>
+      <groupId>org.eclipse.jetty.ee10</groupId>
+      <artifactId>jetty-ee10-servlets</artifactId>
       <scope>compile</scope>
     </dependency>
     <dependency>
@@ -545,7 +545,7 @@
               <overWriteIfNewer>true</overWriteIfNewer>
               <useSubDirectoryPerType>true</useSubDirectoryPerType>
               <includeArtifactIds>
-                guava,protobuf-java,jetty-io,jetty-servlet,jetty-servlets,jetty-http,jetty-plus,jetty-util,jetty-server,jetty-security,jetty-proxy,jetty-client
+                guava,protobuf-java,jetty-io,jetty-ee10-servlet,jetty-ee10-servlets,jetty-http,jetty-plus,jetty-util,jetty-server,jetty-security,jetty-ee10-proxy,jetty-client
               </includeArtifactIds>
               <silent>true</silent>
             </configuration>
@@ -563,10 +563,10 @@
               <include>org.spark-project.spark:unused</include>
               <include>org.eclipse.jetty:jetty-io</include>
               <include>org.eclipse.jetty:jetty-http</include>
-              <include>org.eclipse.jetty:jetty-proxy</include>
+              <include>org.eclipse.jetty.ee10:jetty-ee10-proxy</include>
               <include>org.eclipse.jetty:jetty-client</include>
-              <include>org.eclipse.jetty:jetty-servlet</include>
-              <include>org.eclipse.jetty:jetty-servlets</include>
+              <include>org.eclipse.jetty.ee10:jetty-ee10-servlet</include>
+              <include>org.eclipse.jetty.ee10:jetty-ee10-servlets</include>
               <include>org.eclipse.jetty:jetty-plus</include>
               <include>org.eclipse.jetty:jetty-security</include>
               <include>org.eclipse.jetty:jetty-util</include>

core/src/main/scala/org/apache/spark/TestUtils.scala

@@ -41,10 +41,9 @@ import org.apache.logging.log4j.LogManager
 import org.apache.logging.log4j.core.LoggerContext
 import org.apache.logging.log4j.core.appender.ConsoleAppender
 import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory
-import org.eclipse.jetty.server.Handler
+import org.eclipse.jetty.server.Handler.Sequence
 import org.eclipse.jetty.server.Server
 import org.eclipse.jetty.server.handler.DefaultHandler
-import org.eclipse.jetty.server.handler.HandlerList
 import org.eclipse.jetty.server.handler.ResourceHandler
 import org.json4s.JsonAST.JValue
 import org.json4s.jackson.JsonMethods.{compact, render}
@@ -333,9 +332,8 @@ private[spark] object TestUtils extends SparkTestUtils {
     // 0 as port means choosing randomly from the available ports
     val server = new Server(new InetSocketAddress(Utils.localCanonicalHostName(), 0))
     val resHandler = new ResourceHandler()
-    resHandler.setResourceBase(resBaseDir)
-    val handlers = new HandlerList()
-    handlers.setHandlers(Array[Handler](resHandler, new DefaultHandler()))
+    resHandler.setBaseResourceAsString(resBaseDir)
+    val handlers = new Sequence(resHandler, new DefaultHandler())
     server.setHandler(handlers)
     server.start()
     try {

core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala

@@ -26,7 +26,7 @@ import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache, Removal
 import com.google.common.util.concurrent.UncheckedExecutionException
 import jakarta.servlet.{DispatcherType, Filter, FilterChain, ServletException, ServletRequest, ServletResponse}
 import jakarta.servlet.http.{HttpServletRequest, HttpServletResponse}
-import org.eclipse.jetty.servlet.FilterHolder
+import org.eclipse.jetty.ee10.servlet.FilterHolder
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.LogKeys._

core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala

@@ -23,7 +23,7 @@ import scala.util.control.NonFatal
 import scala.xml.Node
 
 import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse}
-import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
+import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder}
 
 import org.apache.spark.{SecurityManager, SparkConf}
 import org.apache.spark.deploy.SparkHadoopUtil

core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala

@@ -25,8 +25,8 @@ import scala.io.Source
 import com.fasterxml.jackson.core.JsonProcessingException
 import jakarta.servlet.DispatcherType
 import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse}
+import org.eclipse.jetty.ee10.servlet.{FilterHolder, ServletContextHandler, ServletHolder}
 import org.eclipse.jetty.server.{HttpConfiguration, HttpConnectionFactory, Server, ServerConnector}
-import org.eclipse.jetty.servlet.{FilterHolder, ServletContextHandler, ServletHolder}
 import org.eclipse.jetty.util.thread.{QueuedThreadPool, ScheduledExecutorScheduler}
 import org.json4s._
 import org.json4s.jackson.JsonMethods._

core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala

@@ -23,7 +23,7 @@ import java.util.concurrent.TimeUnit
 import scala.collection.mutable
 
 import com.codahale.metrics.{Metric, MetricRegistry}
-import org.eclipse.jetty.servlet.ServletContextHandler
+import org.eclipse.jetty.ee10.servlet.ServletContextHandler
 
 import org.apache.spark.{SecurityManager, SparkConf}
 import org.apache.spark.internal.Logging

core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala

@@ -24,7 +24,7 @@ import com.codahale.metrics.MetricRegistry
 import com.codahale.metrics.json.MetricsModule
 import com.fasterxml.jackson.databind.ObjectMapper
 import jakarta.servlet.http.HttpServletRequest
-import org.eclipse.jetty.servlet.ServletContextHandler
+import org.eclipse.jetty.ee10.servlet.ServletContextHandler
 
 import org.apache.spark.SparkConf
 import org.apache.spark.ui.JettyUtils._

core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala

@@ -21,7 +21,7 @@ import java.util.Properties
 
 import com.codahale.metrics.MetricRegistry
 import jakarta.servlet.http.HttpServletRequest
-import org.eclipse.jetty.servlet.ServletContextHandler
+import org.eclipse.jetty.ee10.servlet.ServletContextHandler
 
 import org.apache.spark.SparkConf
 import org.apache.spark.annotation.{DeveloperApi, Since, Unstable}

core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala

@@ -22,8 +22,8 @@ import jakarta.servlet.ServletContext
 import jakarta.servlet.http.HttpServletRequest
 import jakarta.ws.rs._
 import jakarta.ws.rs.core.{Context, Response}
+import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder}
 import org.eclipse.jetty.server.handler.ContextHandler
-import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
 import org.glassfish.jersey.server.ServerProperties
 import org.glassfish.jersey.servlet.ServletContainer
 

core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala

@@ -18,7 +18,7 @@ package org.apache.spark.status.api.v1
 
 import jakarta.ws.rs._
 import jakarta.ws.rs.core.MediaType
-import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
+import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder}
 import org.glassfish.jersey.server.ServerProperties
 import org.glassfish.jersey.servlet.ServletContainer
 

core/src/main/scala/org/apache/spark/ui/JettyUtils.scala

@@ -20,20 +20,26 @@ package org.apache.spark.ui
 import java.net.{URI, URL, URLDecoder}
 import java.util.EnumSet
 
+import scala.jdk.CollectionConverters.IterableHasAsScala
+import scala.jdk.CollectionConverters.SeqHasAsJava
 import scala.language.implicitConversions
 import scala.util.Try
 import scala.xml.Node
 
 import jakarta.servlet.DispatcherType
 import jakarta.servlet.http._
 import org.eclipse.jetty.client.HttpClient
-import org.eclipse.jetty.client.api.Response
-import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP
-import org.eclipse.jetty.proxy.ProxyServlet
+import org.eclipse.jetty.client.Response
+import org.eclipse.jetty.client.transport.HttpClientTransportOverHTTP
+import org.eclipse.jetty.ee10.proxy.ProxyServlet
+import org.eclipse.jetty.ee10.servlet._
+import org.eclipse.jetty.http.HttpHeader
+import org.eclipse.jetty.server
 import org.eclipse.jetty.server._
 import org.eclipse.jetty.server.handler._
+import org.eclipse.jetty.server.handler.ErrorHandler
 import org.eclipse.jetty.server.handler.gzip.GzipHandler
-import org.eclipse.jetty.servlet._
+import org.eclipse.jetty.util.Callback
 import org.eclipse.jetty.util.component.LifeCycle
 import org.eclipse.jetty.util.thread.{QueuedThreadPool, ScheduledExecutorScheduler}
 import org.json4s.JValue
@@ -266,7 +272,7 @@ private[spark] object JettyUtils extends Logging {
 
     val errorHandler = new ErrorHandler()
     errorHandler.setShowStacks(true)
-    errorHandler.setServer(server)
+    server.setErrorHandler(errorHandler)
     server.addBean(errorHandler)
 
     val collection = new ContextHandlerCollection
@@ -395,19 +401,18 @@ private[spark] object JettyUtils extends Logging {
     val redirectHandler: ContextHandler = new ContextHandler
     redirectHandler.setContextPath("/")
     redirectHandler.setVirtualHosts(toVirtualHosts(REDIRECT_CONNECTOR_NAME))
-    redirectHandler.setHandler(new AbstractHandler {
+    redirectHandler.setHandler(new Handler.Abstract {
       override def handle(
-          target: String,
-          baseRequest: Request,
-          request: HttpServletRequest,
-          response: HttpServletResponse): Unit = {
-        if (baseRequest.isSecure) {
-          return
+                           request: Request,
+                           response: server.Response,
+                           callback: Callback): Boolean = {
+        if (request.isSecure) {
+          return true
         }
-        val httpsURI = createRedirectURI(scheme, securePort, baseRequest)
-        response.setContentLength(0)
-        response.sendRedirect(response.encodeRedirectURL(httpsURI))
-        baseRequest.setHandled(true)
+        val httpsURI = createRedirectURI(scheme, securePort, request)
+        response.getHeaders.put(HttpHeader.CONTENT_LENGTH, 0)
+        server.Response.sendRedirect(request, response, callback, httpsURI)
+        return true
       }
     })
     redirectHandler
@@ -472,27 +477,28 @@ private[spark] object JettyUtils extends Logging {
 
   // Create a new URI from the arguments, handling IPv6 host encoding and default ports.
   private def createRedirectURI(scheme: String, port: Int, request: Request): String = {
-    val server = request.getServerName
+    val server = Request.getServerName(request)
     val redirectServer = if (server.contains(":") && !server.startsWith("[")) {
       s"[${server}]"
     } else {
       server
     }
     val authority = s"$redirectServer:$port"
-    val queryEncoding = if (request.getQueryEncoding != null) {
-      request.getQueryEncoding
+    val queryEncoding = if (Request.getCharset(request) != null) {
+      Request.getCharset(request).toString
     } else {
-      // By default decoding the URI as "UTF-8" should be enough for SparkUI
+      // By default, decoding the URI as "UTF-8" should be enough for SparkUI
       "UTF-8"
     }
     // The request URL can be raw or encoded here. To avoid the request URL being
     // encoded twice, let's decode it here.
-    val requestURI = decodeURL(request.getRequestURI, queryEncoding)
-    val queryString = decodeURL(request.getQueryString, queryEncoding)
+    val requestURI = decodeURL(request.getHttpURI.getPath, queryEncoding)
+    val queryString = decodeURL(request.getHttpURI.getQuery, queryEncoding)
     new URI(scheme, authority, requestURI, queryString, null).toString
   }
 
-  def toVirtualHosts(connectors: String*): Array[String] = connectors.map("@" + _).toArray
+  def toVirtualHosts(connectors: String*): java.util.List[String] =
+    connectors.map("@" + _).toList.asJava
 
 }
 
@@ -522,7 +528,7 @@ private[spark] case class ServerInfo(
   def removeHandler(handler: ServletContextHandler): Unit = synchronized {
     // Since addHandler() always adds a wrapping gzip handler, find the container handler
     // and remove it.
-    rootHandler.getHandlers()
+    rootHandler.getHandlers.asScala
       .find { h =>
         h.isInstanceOf[GzipHandler] && h.asInstanceOf[GzipHandler].getHandler() == handler
       }
@@ -550,7 +556,7 @@ private[spark] case class ServerInfo(
     }
     server.stop()
     // Stop the ThreadPool if it supports stop() method (through LifeCycle).
-    // It is needed because stopping the Server won't stop the ThreadPool it uses.
+    // It is necessary because stopping the Server won't stop the ThreadPool it uses.
     if (threadPool != null && threadPool.isInstanceOf[LifeCycle]) {
       threadPool.asInstanceOf[LifeCycle].stop
     }
@@ -589,24 +595,20 @@ private[spark] case class ServerInfo(
 }
 
 /**
- * A Jetty handler to handle redirects to a proxy server. It intercepts redirects and rewrites the
+ * A Jetty handler to handle redirects to a proxy server. It intercepts, redirects, and rewrites the
  * location to point to the proxy server.
  *
  * The handler needs to be set as the server's handler, because Jetty sometimes generates redirects
  * before invoking any servlet handlers or filters. One of such cases is when asking for the root of
  * a servlet context without the trailing slash (e.g. "/jobs") - Jetty will send a redirect to the
  * same URL, but with a trailing slash.
  */
-private class ProxyRedirectHandler(_proxyUri: String) extends HandlerWrapper {
+private class ProxyRedirectHandler(_proxyUri: String) extends Handler.Wrapper {
 
   private val proxyUri = _proxyUri.stripSuffix("/")
 
-  override def handle(
-      target: String,
-      baseRequest: Request,
-      request: HttpServletRequest,
-      response: HttpServletResponse): Unit = {
-    super.handle(target, baseRequest, request, new ResponseWrapper(request, response))
+  override def handle(request: Request, response: server.Response, callback: Callback): Boolean = {
+    super.handle(request, response, callback)
   }
 
   private class ResponseWrapper(

core/src/main/scala/org/apache/spark/ui/SparkUI.scala

@@ -20,7 +20,7 @@ package org.apache.spark.ui
 import java.util.Date
 
 import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse}
-import org.eclipse.jetty.servlet.ServletContextHandler
+import org.eclipse.jetty.ee10.servlet.ServletContextHandler
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkContext}
 import org.apache.spark.internal.Logging

core/src/main/scala/org/apache/spark/ui/UIUtils.scala

@@ -32,6 +32,7 @@ import scala.xml.transform.{RewriteRule, RuleTransformer}
 
 import jakarta.servlet.http.HttpServletRequest
 import jakarta.ws.rs.core.{MediaType, MultivaluedMap, Response}
+import org.eclipse.jetty.server.Request
 import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap
 
 import org.apache.spark.internal.Logging
@@ -196,6 +197,16 @@ private[spark] object UIUtils extends Logging {
       .getOrElse("")
   }
 
+  def uiRoot(request: Request): String = {
+    // Knox uses X-Forwarded-Context to notify the application of the base path
+    val knoxBasePath = Option(request.getHeaders.get("X-Forwarded-Context"))
+    // SPARK-11484 - Use the proxyBase set by the AM, if not found, then use env.
+    sys.props.get("spark.ui.proxyBase")
+      .orElse(sys.env.get("APPLICATION_WEB_PROXY_BASE"))
+      .orElse(knoxBasePath)
+      .getOrElse("")
+  }
+
   def prependBaseUri(
       request: HttpServletRequest,
       basePath: String = "",

core/src/main/scala/org/apache/spark/ui/WebUI.scala

@@ -25,7 +25,7 @@ import scala.xml.Node
 
 import jakarta.servlet.DispatcherType
 import jakarta.servlet.http.{HttpServlet, HttpServletRequest}
-import org.eclipse.jetty.servlet.{FilterHolder, FilterMapping, ServletContextHandler, ServletHolder}
+import org.eclipse.jetty.ee10.servlet.{FilterHolder, FilterMapping, ServletContextHandler, ServletHolder}
 import org.json4s.JsonAST.{JNothing, JValue}
 
 import org.apache.spark.{SecurityManager, SparkConf, SSLOptions}

core/src/main/scala/org/apache/spark/util/Utils.scala

@@ -60,7 +60,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.logging.log4j.{Level, LogManager}
 import org.apache.logging.log4j.core.LoggerContext
 import org.apache.logging.log4j.core.config.LoggerConfig
-import org.eclipse.jetty.util.MultiException
 import org.slf4j.Logger
 
 import org.apache.spark.{SPARK_VERSION, _}
@@ -2299,8 +2298,6 @@ private[spark] object Utils
           return true
         }
         isBindCollision(e.getCause)
-      case e: MultiException =>
-        e.getThrowables.asScala.exists(isBindCollision)
       case e: NativeIoException =>
         (e.getMessage != null && e.getMessage.startsWith("bind() failed: ")) ||
           isBindCollision(e.getCause)

core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala

@@ -25,7 +25,7 @@ import scala.collection.mutable
 import com.codahale.metrics.Counter
 import jakarta.servlet.http.{HttpServletRequest, HttpServletResponse}
 import org.apache.hadoop.conf.Configuration
-import org.eclipse.jetty.servlet.ServletContextHandler
+import org.eclipse.jetty.ee10.servlet.ServletContextHandler
 import org.mockito.ArgumentMatchers.any
 import org.mockito.Mockito._
 import org.mockito.invocation.InvocationOnMock

core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala

@@ -18,8 +18,8 @@
 package org.apache.spark.deploy.history
 
 import jakarta.servlet.http.HttpServletRequest
-import org.eclipse.jetty.proxy.ProxyServlet
-import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
+import org.eclipse.jetty.ee10.proxy.ProxyServlet
+import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder}
 import org.openqa.selenium.WebDriver
 import org.scalatest.concurrent.Eventually._
 import org.scalatest.matchers.must.Matchers