diff --git a/connector/kafka-0-10-sql/pom.xml b/connector/kafka-0-10-sql/pom.xml index e22a57354b89a..d518313f36b88 100644 --- a/connector/kafka-0-10-sql/pom.xml +++ b/connector/kafka-0-10-sql/pom.xml @@ -134,8 +134,8 @@ test - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet ${jetty.version} test diff --git a/core/pom.xml b/core/pom.xml index 6468f500db046..6b3dd71d416a6 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -122,8 +122,8 @@ - org.eclipse.jetty - jetty-plus + org.eclipse.jetty.ee10 + jetty-ee10-plus compile @@ -147,13 +147,13 @@ compile - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet compile - org.eclipse.jetty - jetty-proxy + org.eclipse.jetty.ee10 + jetty-ee10-proxy compile @@ -162,8 +162,8 @@ compile - org.eclipse.jetty - jetty-servlets + org.eclipse.jetty.ee10 + jetty-ee10-servlets compile diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala index 5e3078d7292ba..6c71db535c578 100644 --- a/core/src/main/scala/org/apache/spark/TestUtils.scala +++ b/core/src/main/scala/org/apache/spark/TestUtils.scala @@ -46,8 +46,8 @@ import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFact import org.eclipse.jetty.server.Handler import org.eclipse.jetty.server.Server import org.eclipse.jetty.server.handler.DefaultHandler -import org.eclipse.jetty.server.handler.HandlerList import org.eclipse.jetty.server.handler.ResourceHandler +import org.eclipse.jetty.util.resource.ResourceFactory import org.json4s.JsonAST.JValue import org.json4s.jackson.JsonMethods.{compact, render} @@ -55,6 +55,7 @@ import org.apache.spark.executor.TaskMetrics import org.apache.spark.scheduler._ import org.apache.spark.util.{SparkTestUtils, Utils} + /** * Utilities for tests. Included in main codebase since it's used by multiple * projects. @@ -335,9 +336,9 @@ private[spark] object TestUtils extends SparkTestUtils { // 0 as port means choosing randomly from the available ports val server = new Server(new InetSocketAddress(Utils.localCanonicalHostName(), 0)) val resHandler = new ResourceHandler() - resHandler.setResourceBase(resBaseDir) - val handlers = new HandlerList() - handlers.setHandlers(Array[Handler](resHandler, new DefaultHandler())) + resHandler.setBaseResource(ResourceFactory.of(resHandler).newResource(resBaseDir)) + val handlers = new Handler.Sequence; + handlers.setHandlers(resHandler, new DefaultHandler()) server.setHandler(handlers) server.start() try { diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala index 662746cf0c782..13d505a84010a 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala @@ -26,7 +26,7 @@ import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache, Removal import com.google.common.util.concurrent.UncheckedExecutionException import jakarta.servlet.{DispatcherType, Filter, FilterChain, FilterConfig, ServletException, ServletRequest, ServletResponse} import jakarta.servlet.http.{HttpServletRequest, HttpServletResponse} -import org.eclipse.jetty.servlet.FilterHolder +import org.eclipse.jetty.ee10.servlet.FilterHolder import org.apache.spark.internal.Logging import org.apache.spark.metrics.source.Source diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index 7362634d5b09e..ab50bb20feec2 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -23,7 +23,7 @@ import scala.util.control.NonFatal import scala.xml.Node import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} +import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} import org.apache.spark.{SecurityManager, SparkConf} import org.apache.spark.deploy.SparkHadoopUtil diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala index bb91c7e7f4a22..c76607202894e 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala @@ -21,8 +21,8 @@ import scala.io.Source import com.fasterxml.jackson.core.JsonProcessingException import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} +import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} import org.eclipse.jetty.server.{HttpConnectionFactory, Server, ServerConnector} -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} import org.eclipse.jetty.util.thread.{QueuedThreadPool, ScheduledExecutorScheduler} import org.json4s._ import org.json4s.jackson.JsonMethods._ diff --git a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala index 777bc0a60e016..1488106907add 100644 --- a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala +++ b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala @@ -23,7 +23,7 @@ import java.util.concurrent.TimeUnit import scala.collection.mutable import com.codahale.metrics.{Metric, MetricRegistry} -import org.eclipse.jetty.servlet.ServletContextHandler +import org.eclipse.jetty.ee10.servlet.ServletContextHandler import org.apache.spark.{SecurityManager, SparkConf} import org.apache.spark.internal.Logging diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala b/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala index 7d676cd2c27b9..305b391cc5fcf 100644 --- a/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala +++ b/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala @@ -24,7 +24,7 @@ import com.codahale.metrics.MetricRegistry import com.codahale.metrics.json.MetricsModule import com.fasterxml.jackson.databind.ObjectMapper import jakarta.servlet.http.HttpServletRequest -import org.eclipse.jetty.servlet.ServletContextHandler +import org.eclipse.jetty.ee10.servlet.ServletContextHandler import org.apache.spark.SparkConf import org.apache.spark.ui.JettyUtils._ diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala b/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala index 7571395289967..e41c7c588c2ba 100644 --- a/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala +++ b/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala @@ -21,7 +21,7 @@ import java.util.Properties import com.codahale.metrics.MetricRegistry import jakarta.servlet.http.HttpServletRequest -import org.eclipse.jetty.servlet.ServletContextHandler +import org.eclipse.jetty.ee10.servlet.ServletContextHandler import org.apache.spark.SparkConf import org.apache.spark.ui.JettyUtils._ diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala index 66fac8a9d105a..09e9bdf738114 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala @@ -22,8 +22,8 @@ import jakarta.servlet.ServletContext import jakarta.servlet.http.HttpServletRequest import jakarta.ws.rs._ import jakarta.ws.rs.core.{Context, Response} +import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} import org.eclipse.jetty.server.handler.ContextHandler -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} import org.glassfish.jersey.server.ServerProperties import org.glassfish.jersey.servlet.ServletContainer diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala index 6efe3106ba56d..ea30d59832b65 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala @@ -18,7 +18,7 @@ package org.apache.spark.status.api.v1 import jakarta.ws.rs._ import jakarta.ws.rs.core.MediaType -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} +import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} import org.glassfish.jersey.server.ServerProperties import org.glassfish.jersey.servlet.ServletContainer diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala index 5e567a891d587..440229e72d57f 100644 --- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala @@ -18,8 +18,10 @@ package org.apache.spark.ui import java.net.{URI, URL, URLDecoder} +import java.nio.charset.Charset import java.util.EnumSet +import scala.jdk.CollectionConverters._ import scala.language.implicitConversions import scala.util.Try import scala.xml.Node @@ -27,13 +29,14 @@ import scala.xml.Node import jakarta.servlet.DispatcherType import jakarta.servlet.http._ import org.eclipse.jetty.client.HttpClient -import org.eclipse.jetty.client.api.Response -import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP -import org.eclipse.jetty.proxy.ProxyServlet +import org.eclipse.jetty.client.transport.HttpClientTransportOverHTTP +import org.eclipse.jetty.ee10.proxy.ProxyServlet +import org.eclipse.jetty.ee10.servlet.{DefaultServlet, FilterHolder, ServletContextHandler, ServletHolder} +import org.eclipse.jetty.http.HttpHeader import org.eclipse.jetty.server._ import org.eclipse.jetty.server.handler._ import org.eclipse.jetty.server.handler.gzip.GzipHandler -import org.eclipse.jetty.servlet._ +import org.eclipse.jetty.util.{Callback, UrlEncoded} import org.eclipse.jetty.util.component.LifeCycle import org.eclipse.jetty.util.thread.{QueuedThreadPool, ScheduledExecutorScheduler} import org.json4s.JValue @@ -149,6 +152,7 @@ private[spark] object JettyUtils extends Logging { // Make sure we don't end up with "//" in the middle val newUrl = new URL(new URL(request.getRequestURL.toString), prefixedDestPath).toString response.sendRedirect(newUrl) +// Response.sendRedirect(request, response, callback, location) } // SPARK-5983 ensure TRACE is not supported protected override def doTrace(req: HttpServletRequest, res: HttpServletResponse): Unit = { @@ -209,12 +213,12 @@ private[spark] object JettyUtils extends Logging { override def filterServerResponseHeader( clientRequest: HttpServletRequest, - serverResponse: Response, + serverResponse: org.eclipse.jetty.client.Response, headerName: String, headerValue: String): String = { if (headerName.equalsIgnoreCase("location")) { val newHeader = createProxyLocationHeader(headerValue, clientRequest, - serverResponse.getRequest().getURI()) + serverResponse.getRequest.getURI) if (newHeader != null) { return newHeader } @@ -259,7 +263,7 @@ private[spark] object JettyUtils extends Logging { val errorHandler = new ErrorHandler() errorHandler.setShowStacks(true) - errorHandler.setServer(server) + server.setErrorHandler(errorHandler); server.addBean(errorHandler) val collection = new ContextHandlerCollection @@ -387,20 +391,19 @@ private[spark] object JettyUtils extends Logging { private def createRedirectHttpsHandler(securePort: Int, scheme: String): ContextHandler = { val redirectHandler: ContextHandler = new ContextHandler redirectHandler.setContextPath("/") - redirectHandler.setVirtualHosts(toVirtualHosts(REDIRECT_CONNECTOR_NAME)) - redirectHandler.setHandler(new AbstractHandler { + redirectHandler.setVirtualHosts(toVirtualHosts(REDIRECT_CONNECTOR_NAME).asJava) + redirectHandler.setHandler(new Handler.Abstract { override def handle( - target: String, - baseRequest: Request, - request: HttpServletRequest, - response: HttpServletResponse): Unit = { - if (baseRequest.isSecure) { - return - } - val httpsURI = createRedirectURI(scheme, securePort, baseRequest) - response.setContentLength(0) - response.sendRedirect(response.encodeRedirectURL(httpsURI)) - baseRequest.setHandled(true) + request: Request, + response: Response, + callback: Callback): Boolean = { + if (request.isSecure) return false + val httpsURI = createRedirectURI(scheme, securePort, request) + val responseHeaders = response.getHeaders + responseHeaders.put(HttpHeader.CONTENT_LENGTH, 0L) + val location = Response.toRedirectURI(request, httpsURI); + Response.sendRedirect(request, response, callback, location) + true } }) redirectHandler @@ -455,7 +458,7 @@ private[spark] object JettyUtils extends Logging { handler.addFilter(holder, "/*", EnumSet.allOf(classOf[DispatcherType])) } - private def decodeURL(url: String, encoding: String): String = { + private def decodeURL(url: String, encoding: Charset): String = { if (url == null) { null } else { @@ -465,27 +468,20 @@ private[spark] object JettyUtils extends Logging { // Create a new URI from the arguments, handling IPv6 host encoding and default ports. private def createRedirectURI(scheme: String, port: Int, request: Request): String = { - val server = request.getServerName + val server = Request.getServerName(request) val redirectServer = if (server.contains(":") && !server.startsWith("[")) { s"[${server}]" } else { server } + request.getHttpURI.getDecodedPath val authority = s"$redirectServer:$port" - val queryEncoding = if (request.getQueryEncoding != null) { - request.getQueryEncoding - } else { - // By default decoding the URI as "UTF-8" should be enough for SparkUI - "UTF-8" - } - // The request URL can be raw or encoded here. To avoid the request URL being - // encoded twice, let's decode it here. - val requestURI = decodeURL(request.getRequestURI, queryEncoding) - val queryString = decodeURL(request.getQueryString, queryEncoding) + val requestURI = request.getHttpURI.getDecodedPath + val queryString = decodeURL(request.getHttpURI.getQuery, UrlEncoded.ENCODING) new URI(scheme, authority, requestURI, queryString, null).toString } - def toVirtualHosts(connectors: String*): Array[String] = connectors.map("@" + _).toArray + def toVirtualHosts(connectors: String*): List[String] = connectors.map("@" + _).toList } @@ -499,7 +495,7 @@ private[spark] case class ServerInfo( def addHandler( handler: ServletContextHandler, securityMgr: SecurityManager): Unit = synchronized { - handler.setVirtualHosts(JettyUtils.toVirtualHosts(JettyUtils.SPARK_CONNECTOR_NAME)) + handler.setVirtualHosts(JettyUtils.toVirtualHosts(JettyUtils.SPARK_CONNECTOR_NAME).asJava) addFilters(handler, securityMgr) val gzipHandler = new GzipHandler() @@ -515,7 +511,7 @@ private[spark] case class ServerInfo( def removeHandler(handler: ServletContextHandler): Unit = synchronized { // Since addHandler() always adds a wrapping gzip handler, find the container handler // and remove it. - rootHandler.getHandlers() + rootHandler.getHandlers.asScala .find { h => h.isInstanceOf[GzipHandler] && h.asInstanceOf[GzipHandler].getHandler() == handler } @@ -579,6 +575,12 @@ private[spark] case class ServerInfo( } +// private def getRedirectUrl(location: String): Unit = { +// +// val proxyUri = _proxyUri.stripSuffix("/") +// +// } + /** * A Jetty handler to handle redirects to a proxy server. It intercepts redirects and rewrites the * location to point to the proxy server. @@ -588,36 +590,37 @@ private[spark] case class ServerInfo( * a servlet context without the trailing slash (e.g. "/jobs") - Jetty will send a redirect to the * same URL, but with a trailing slash. */ -private class ProxyRedirectHandler(_proxyUri: String) extends HandlerWrapper { +private class ProxyRedirectHandler(_proxyUri: String) extends Handler.Wrapper { private val proxyUri = _proxyUri.stripSuffix("/") override def handle( - target: String, - baseRequest: Request, - request: HttpServletRequest, - response: HttpServletResponse): Unit = { - super.handle(target, baseRequest, request, new ResponseWrapper(request, response)) - } - - private class ResponseWrapper( - req: HttpServletRequest, - res: HttpServletResponse) - extends HttpServletResponseWrapper(res) { - - override def sendRedirect(location: String): Unit = { - val newTarget = if (location != null) { - val target = new URI(location) - // The target path should already be encoded, so don't re-encode it, just the - // proxy address part. - val proxyBase = UIUtils.uiRoot(req) - val proxyPrefix = if (proxyBase.nonEmpty) s"$proxyUri$proxyBase" else proxyUri - s"${res.encodeURL(proxyPrefix)}${target.getPath()}" - } else { - null - } - super.sendRedirect(newTarget) - } + request: Request, + response: org.eclipse.jetty.server.Response, + callback: Callback): Boolean = { + // Todo: Fix the proxy redirect behaviour. +// super.handle(request, new ResponseWrapper(request, response), callback) + super.handle(request, response, callback) } +// +// private class ResponseWrapper( +// req: Request, +// res: Response) +// extends Response.Wrapper(req, res) { +// +// override def sendRedirect(location: String): Unit = { +// val newTarget = if (location != null) { +// val target = new URI(location) +// // The target path should already be encoded, so don't re-encode it, just the +// // proxy address part. +// val proxyBase = UIUtils.uiRoot(req) +// val proxyPrefix = if (proxyBase.nonEmpty) s"$proxyUri$proxyBase" else proxyUri +// s"${res.encodeURL(proxyPrefix)}${target.getPath()}" +// } else { +// null +// } +// super.sendRedirect(newTarget) +// } +// } } diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala index 099e47abf408a..c7b85b2daab79 100644 --- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala @@ -20,7 +20,7 @@ package org.apache.spark.ui import java.util.Date import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} -import org.eclipse.jetty.servlet.ServletContextHandler +import org.eclipse.jetty.ee10.servlet.ServletContextHandler import org.apache.spark.{SecurityManager, SparkConf, SparkContext} import org.apache.spark.internal.Logging diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala index 2c937e71f64b9..2955860e6acfd 100644 --- a/core/src/main/scala/org/apache/spark/ui/WebUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/WebUI.scala @@ -25,7 +25,7 @@ import scala.xml.Node import jakarta.servlet.DispatcherType import jakarta.servlet.http.{HttpServlet, HttpServletRequest} -import org.eclipse.jetty.servlet.{FilterHolder, FilterMapping, ServletContextHandler, ServletHolder} +import org.eclipse.jetty.ee10.servlet.{FilterHolder, FilterMapping, ServletContextHandler, ServletHolder} import org.json4s.JsonAST.{JNothing, JValue} import org.apache.spark.{SecurityManager, SparkConf, SSLOptions} diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index e1ced9f8b41d5..1d29bf3bddeb3 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -63,7 +63,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.logging.log4j.{Level, LogManager} import org.apache.logging.log4j.core.LoggerContext import org.apache.logging.log4j.core.config.LoggerConfig -import org.eclipse.jetty.util.MultiException import org.slf4j.Logger import org.apache.spark._ @@ -2198,8 +2197,6 @@ private[spark] object Utils return true } isBindCollision(e.getCause) - case e: MultiException => - e.getThrowables.asScala.exists(isBindCollision) case e: NativeIoException => (e.getMessage != null && e.getMessage.startsWith("bind() failed: ")) || isBindCollision(e.getCause) diff --git a/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala index d27c07c36c0de..f5968e383b05c 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala @@ -25,7 +25,7 @@ import scala.collection.mutable import com.codahale.metrics.Counter import jakarta.servlet.http.{HttpServletRequest, HttpServletResponse} import org.apache.hadoop.conf.Configuration -import org.eclipse.jetty.servlet.ServletContextHandler +import org.eclipse.jetty.ee10.servlet.ServletContextHandler import org.mockito.ArgumentMatchers.any import org.mockito.Mockito._ import org.mockito.invocation.InvocationOnMock diff --git a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala index 7effeee3424b8..a1ac2a0f09d90 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala @@ -18,8 +18,8 @@ package org.apache.spark.deploy.history import jakarta.servlet.http.HttpServletRequest -import org.eclipse.jetty.proxy.ProxyServlet -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} +import org.eclipse.jetty.ee10.proxy.ProxyServlet +import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} import org.openqa.selenium.WebDriver import org.scalatest.concurrent.Eventually._ import org.scalatest.matchers.must.Matchers diff --git a/core/src/test/scala/org/apache/spark/ui/UISuite.scala b/core/src/test/scala/org/apache/spark/ui/UISuite.scala index 10681f22fa624..85ab49a4c2593 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISuite.scala @@ -25,7 +25,7 @@ import scala.io.Source import jakarta.servlet._ import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} +import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} import org.eclipse.jetty.util.thread.QueuedThreadPool import org.mockito.Mockito.{mock, when} import org.scalatest.concurrent.Eventually._ @@ -358,21 +358,9 @@ class UISuite extends SparkFunSuite { val serverInfo = JettyUtils.startJettyServer("0.0.0.0", 0, sslOptions, conf) try { val serverAddr = s"http://$localhost:${serverInfo.boundPort}" - - val (_, ctx) = newContext("/ctx1") - serverInfo.addHandler(ctx, securityMgr) - val redirect = JettyUtils.createRedirectHandler("/src", "/dst") serverInfo.addHandler(redirect, securityMgr) - // Test Jetty's built-in redirect to add the trailing slash to the context path. - TestUtils.withHttpConnection(new URL(s"$serverAddr/ctx1")) { conn => - assert(conn.getResponseCode() === HttpServletResponse.SC_FOUND) - val location = Option(conn.getHeaderFields().get("Location")) - .map(_.get(0)).orNull - assert(location === s"$proxyRoot/ctx1/") - } - // Test with a URL handled by the added redirect handler, and also including a path prefix. val headers = Seq("X-Forwarded-Context" -> "/prefix") TestUtils.withHttpConnection( @@ -398,8 +386,8 @@ class UISuite extends SparkFunSuite { } } - test("SPARK-45522: Jetty 10 and above shouuld return status code 302 with correct redirect url" + - " when request URL ends with a context path without trailing '/'") { + test("SPARK-34449: Jetty 9.4.35.v20201120 and later no longer return status code 302 " + + " and handle internally when request URL ends with a context path without trailing '/'") { val proxyRoot = "https://proxy.example.com:443/prefix" val (conf, securityMgr, sslOptions) = sslDisabledConf() conf.set(UI.PROXY_REDIRECT_URI, proxyRoot) @@ -412,10 +400,9 @@ class UISuite extends SparkFunSuite { assert(TestUtils.httpResponseCode(new URL(urlStr + "/")) === HttpServletResponse.SC_OK) - // In the case of trailing slash, - // 302 should be return and the redirect URL shouuld be part of the header. - assert(TestUtils.redirectUrl(new URL(urlStr)) === proxyRoot + "/ctx/"); - assert(TestUtils.httpResponseCode(new URL(urlStr)) === HttpServletResponse.SC_FOUND) + // If the following assertion fails when we upgrade Jetty, it seems to change the behavior of + // handling context path which doesn't have the trailing slash. + assert(TestUtils.httpResponseCode(new URL(urlStr)) === HttpServletResponse.SC_OK) } finally { stopServer(serverInfo) } diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 6b357b3e4b70f..315cc78a9898f 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -13,7 +13,7 @@ aliyun-sdk-oss/3.13.0//aliyun-sdk-oss-3.13.0.jar annotations/17.0.0//annotations-17.0.0.jar antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar antlr4-runtime/4.13.1//antlr4-runtime-4.13.1.jar -aopalliance-repackaged/3.0.3//aopalliance-repackaged-3.0.3.jar +aopalliance-repackaged/3.0.5//aopalliance-repackaged-3.0.5.jar arpack/3.0.3//arpack-3.0.3.jar arpack_combined_all/0.1//arpack_combined_all-0.1.jar arrow-format/15.0.0//arrow-format-15.0.0.jar @@ -94,9 +94,9 @@ hive-shims-common/2.3.9//hive-shims-common-2.3.9.jar hive-shims-scheduler/2.3.9//hive-shims-scheduler-2.3.9.jar hive-shims/2.3.9//hive-shims-2.3.9.jar hive-storage-api/2.8.1//hive-storage-api-2.8.1.jar -hk2-api/3.0.3//hk2-api-3.0.3.jar -hk2-locator/3.0.3//hk2-locator-3.0.3.jar -hk2-utils/3.0.3//hk2-utils-3.0.3.jar +hk2-api/3.0.5//hk2-api-3.0.5.jar +hk2-locator/3.0.5//hk2-locator-3.0.5.jar +hk2-utils/3.0.5//hk2-utils-3.0.5.jar httpclient/4.5.14//httpclient-4.5.14.jar httpcore/4.4.16//httpcore-4.4.16.jar icu4j/72.1//icu4j-72.1.jar @@ -112,11 +112,11 @@ jackson-dataformat-yaml/2.16.1//jackson-dataformat-yaml-2.16.1.jar jackson-datatype-jsr310/2.16.1//jackson-datatype-jsr310-2.16.1.jar jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar jackson-module-scala_2.13/2.16.1//jackson-module-scala_2.13-2.16.1.jar -jakarta.annotation-api/2.0.0//jakarta.annotation-api-2.0.0.jar +jakarta.annotation-api/2.1.1//jakarta.annotation-api-2.1.1.jar jakarta.inject-api/2.0.1//jakarta.inject-api-2.0.1.jar -jakarta.servlet-api/5.0.0//jakarta.servlet-api-5.0.0.jar +jakarta.servlet-api/6.0.0//jakarta.servlet-api-6.0.0.jar jakarta.validation-api/3.0.2//jakarta.validation-api-3.0.2.jar -jakarta.ws.rs-api/3.0.0//jakarta.ws.rs-api-3.0.0.jar +jakarta.ws.rs-api/3.1.0//jakarta.ws.rs-api-3.1.0.jar jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar janino/3.1.9//janino-3.1.9.jar java-diff-utils/4.12//java-diff-utils-4.12.jar @@ -129,15 +129,15 @@ jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar jcl-over-slf4j/2.0.12//jcl-over-slf4j-2.0.12.jar jdo-api/3.0.1//jdo-api-3.0.1.jar jdom2/2.0.6//jdom2-2.0.6.jar -jersey-client/3.0.12//jersey-client-3.0.12.jar -jersey-common/3.0.12//jersey-common-3.0.12.jar -jersey-container-servlet-core/3.0.12//jersey-container-servlet-core-3.0.12.jar -jersey-container-servlet/3.0.12//jersey-container-servlet-3.0.12.jar -jersey-hk2/3.0.12//jersey-hk2-3.0.12.jar -jersey-server/3.0.12//jersey-server-3.0.12.jar +jersey-client/3.1.5//jersey-client-3.1.5.jar +jersey-common/3.1.5//jersey-common-3.1.5.jar +jersey-container-servlet-core/3.1.5//jersey-container-servlet-core-3.1.5.jar +jersey-container-servlet/3.1.5//jersey-container-servlet-3.1.5.jar +jersey-hk2/3.1.5//jersey-hk2-3.1.5.jar +jersey-server/3.1.5//jersey-server-3.1.5.jar jettison/1.5.4//jettison-1.5.4.jar -jetty-util-ajax/11.0.20//jetty-util-ajax-11.0.20.jar -jetty-util/11.0.20//jetty-util-11.0.20.jar +jetty-util-ajax/12.0.6//jetty-util-ajax-12.0.6.jar +jetty-util/12.0.6//jetty-util-12.0.6.jar jline/2.14.6//jline-2.14.6.jar jline/3.22.0//jline-3.22.0.jar jna/5.13.0//jna-5.13.0.jar diff --git a/pom.xml b/pom.xml index b74569650bf86..cd8d1309a6c3a 100644 --- a/pom.xml +++ b/pom.xml @@ -143,8 +143,8 @@ 1.13.1 2.0.0 shaded-protobuf - 11.0.20 - 5.0.0 + 12.0.6 + 6.0.0 4.0.1 0.10.0 @@ -201,7 +201,7 @@ 4.1.17 14.0.1 3.1.9 - 3.0.12 + 3.1.5 2.12.7 3.5.2 3.0.0 @@ -486,20 +486,20 @@ provided - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet ${jetty.version} provided - org.eclipse.jetty - jetty-servlets + org.eclipse.jetty.ee10 + jetty-ee10-servlets ${jetty.version} provided - org.eclipse.jetty - jetty-proxy + org.eclipse.jetty.ee10 + jetty-ee10-proxy ${jetty.version} provided @@ -522,8 +522,8 @@ provided - org.eclipse.jetty - jetty-plus + org.eclipse.jetty.ee10 + jetty-ee10-plus ${jetty.version} provided @@ -534,8 +534,8 @@ provided - org.eclipse.jetty - jetty-webapp + org.eclipse.jetty.ee10 + jetty-ee10-webapp ${jetty.version} provided @@ -1428,7 +1428,7 @@ org.eclipse.jetty - jetty-webapp + jetty-ee10-webapp log4j diff --git a/repl/pom.xml b/repl/pom.xml index 831379467a29e..dba1613088beb 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -109,8 +109,8 @@ jetty-server - org.eclipse.jetty - jetty-plus + org.eclipse.jetty.ee10 + jetty-ee10-plus org.eclipse.jetty diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index 694d81b3c25e3..f7ca581a3056a 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -114,8 +114,8 @@ jetty-server - org.eclipse.jetty - jetty-plus + org.eclipse.jetty.ee10 + jetty-ee10-plus org.eclipse.jetty @@ -126,12 +126,12 @@ jetty-http - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet - org.eclipse.jetty - jetty-servlets + org.eclipse.jetty.ee10 + jetty-ee10-servlets diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 05f906206e5e2..46ded6a7e0086 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -139,8 +139,8 @@ parquet-hadoop - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet com.fasterxml.jackson.core diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index d9b16f64970e7..fb4fa06806c94 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -91,8 +91,8 @@ provided - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet provided diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java index 4d99496876fdc..755cc2c9b60da 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java @@ -92,9 +92,9 @@ protected void initializeServer() { // Server args int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE); int requestTimeout = (int) hiveConf.getTimeVar( - HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_TIMEOUT, TimeUnit.SECONDS); + HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_TIMEOUT, TimeUnit.SECONDS); int beBackoffSlotLength = (int) hiveConf.getTimeVar( - HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_BEBACKOFF_SLOT_LENGTH, TimeUnit.MILLISECONDS); + HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_BEBACKOFF_SLOT_LENGTH, TimeUnit.MILLISECONDS); TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(serverSocket) .processorFactory(processorFactory).transportFactory(transportFactory) .protocolFactory(new TBinaryProtocol.Factory()) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java index 9592bffcf1bfd..1ada2bdb0bca2 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java @@ -41,8 +41,8 @@ import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.ee10.servlet.ServletHolder; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.ExecutorThreadPool; import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler; diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java index b0bede741cb19..f525cf550964f 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java @@ -154,7 +154,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) SessionManager.setUserName(clientUserName); // find proxy user if any from query param - String doAsQueryParam = getDoAsQueryParam(request.getQueryString()); + String doAsQueryParam = getDoAsQueryParam(request); if (doAsQueryParam != null) { SessionManager.setProxyUserName(doAsQueryParam); } @@ -542,14 +542,8 @@ private boolean isKerberosAuthMode(String authType) { return authType.equalsIgnoreCase(HiveAuthFactory.AuthTypes.KERBEROS.toString()); } - private static String getDoAsQueryParam(String queryString) { - if (LOG.isDebugEnabled()) { - LOG.debug("URL query string:" + queryString); - } - if (queryString == null) { - return null; - } - Map params = jakarta.servlet.http.HttpUtils.parseQueryString( queryString ); + private static String getDoAsQueryParam(HttpServletRequest request) { + Map params = request.getParameterMap(); Set keySet = params.keySet(); for (String key: keySet) { if (key.equalsIgnoreCase("doAs")) { diff --git a/streaming/pom.xml b/streaming/pom.xml index 85a4d268d2a25..2ee245f410c3c 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -76,8 +76,8 @@ jetty-server - org.eclipse.jetty - jetty-plus + org.eclipse.jetty.ee10 + jetty-ee10-plus org.eclipse.jetty @@ -88,12 +88,12 @@ jetty-http - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet - org.eclipse.jetty - jetty-servlets + org.eclipse.jetty.ee10 + jetty-ee10-servlets