From b821daf9fdfda9fe013eba6615b440785129d2d6 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 9 Nov 2020 23:46:36 -0800 Subject: [PATCH 1/9] HADOOP-17371. Bump Jetty to the latest version 9.4.34. Contributed by Wei-Chiu Chuang. Change-Id: I972c3120ec37ea024de9ea43afcb164457be1ea8 --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 831275e163b2c..c8028d2b5d78e 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -37,7 +37,7 @@ true true - 9.4.20.v20190813 + 9.4.34.v20201102 _ _ From 14cebc399240586fa4a41766b9f9dcfcabbbb8a2 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 10 Nov 2020 11:34:29 -0800 Subject: [PATCH 2/9] Exclude jetty-dir.css from jetty-server in the shaded client. Change-Id: I0db7b8ac45df0c6afb951c72c799a511f30b3230 --- hadoop-client-modules/hadoop-client-minicluster/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 70a627cdc06d2..8c4779790ec1c 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -840,6 +840,12 @@ */** + + org.eclipse.jetty:jetty-server + + jetty-dir.css + + From 8cf5649ab2a4942f642c196314cebbc0fb07c33c Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Wed, 11 Nov 2020 00:07:30 -0800 Subject: [PATCH 3/9] use o.e.j.s.Response.setStatusWithReason() to set a custom error message. Change-Id: Ifa9dcc9fcf6ba1dc4b0d96c66c64a6cddab3d120 --- hadoop-common-project/hadoop-auth/pom.xml | 4 ++++ .../authentication/server/AuthenticationFilter.java | 9 +++++++-- hadoop-project/pom.xml | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index 4761945c6080d..10e0b9c825677 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -193,6 +193,10 @@ guava test + + org.eclipse.jetty + jetty-server + diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index 94d11f48cf2a9..4e70d02616716 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -19,6 +19,7 @@ import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; import org.apache.hadoop.security.authentication.util.*; +import org.eclipse.jetty.server.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -619,10 +620,14 @@ && getMaxInactiveInterval() > 0) { KerberosAuthenticator.WWW_AUTHENTICATE))) { errCode = HttpServletResponse.SC_FORBIDDEN; } + // After Jetty 9.4.21, sendError() no longer allows a custom message. + // use setStatusWithReason() to set a custom message. if (authenticationEx == null) { - httpResponse.sendError(errCode, "Authentication required"); + ((Response)httpResponse).setStatusWithReason(errCode, "Authentication required"); + httpResponse.sendError(errCode); } else { - httpResponse.sendError(errCode, authenticationEx.getMessage()); + ((Response)httpResponse).setStatusWithReason(errCode, authenticationEx.getMessage()); + httpResponse.sendError(errCode); } } } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c8028d2b5d78e..30fcbbb314ca9 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -37,7 +37,7 @@ true true - 9.4.34.v20201102 + 9.4.21.v20190926 _ _ From 90354afa041b154788d1ab19149239b22f1de0fc Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Wed, 11 Nov 2020 00:09:32 -0800 Subject: [PATCH 4/9] Reset to Jetty 9.4.34 Change-Id: I2595590289bc203c13c82024ba611d69fcca249b --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 30fcbbb314ca9..c8028d2b5d78e 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -37,7 +37,7 @@ true true - 9.4.21.v20190926 + 9.4.34.v20201102 _ _ From b5b208aa7e6bc93571bb085a627a4a69dbb186c0 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 21 Dec 2020 10:48:03 -0800 Subject: [PATCH 5/9] Fix test failures Change-Id: Idab5e694d69b0960981fea7a39476ec767aa57bf --- .../authentication/server/AuthenticationFilter.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index 4e70d02616716..9f40c42d24135 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -622,13 +622,18 @@ && getMaxInactiveInterval() > 0) { } // After Jetty 9.4.21, sendError() no longer allows a custom message. // use setStatusWithReason() to set a custom message. + String reason; if (authenticationEx == null) { - ((Response)httpResponse).setStatusWithReason(errCode, "Authentication required"); - httpResponse.sendError(errCode); + reason = "Authentication required"; } else { - ((Response)httpResponse).setStatusWithReason(errCode, authenticationEx.getMessage()); - httpResponse.sendError(errCode); + reason = authenticationEx.getMessage(); } + + if (httpResponse instanceof Response) { + ((Response)httpResponse).setStatusWithReason(errCode, reason); + } + + httpResponse.sendError(errCode, reason); } } } From 5405e28c370d2272c522ca350afa9761dda202e0 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 21 Dec 2020 12:04:11 -0800 Subject: [PATCH 6/9] Call Response.setStatusWithReason() in KMSResponse.sendError() Change-Id: Id82219e354066e68c783b46f293c56c8874b4498 --- .../key/kms/server/KMSAuthenticationFilter.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index c020af27b96aa..ead22e4686645 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java @@ -28,6 +28,7 @@ import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler; +import org.eclipse.jetty.server.Response; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; @@ -113,6 +114,18 @@ public void setStatus(int sc) { public void sendError(int sc, String msg) throws IOException { statusCode = sc; this.msg = msg; + + ServletResponse response = getResponse(); + + // After Jetty 9.4.21, sendError() no longer allows a custom message. + // use setStatusWithReason() to set a custom message. + if (response instanceof Response) { + ((Response) response).setStatusWithReason(sc, msg); + } else { + KMS.LOG.warn("The wrapped response object is instance of {}" + + ", not org.eclipse.jetty.server.Response. Can't set custom error " + + "message", response.getClass()); + } super.sendError(sc, HtmlQuoting.quoteHtmlChars(msg)); } From 48f044cae368df07e4f5a3a2c26951a757c42245 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 21 Dec 2020 12:10:42 -0800 Subject: [PATCH 7/9] Update to Jetty 9.4.35 Change-Id: I4405e030399504e715dc7bb9675300f079215a70 --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c8028d2b5d78e..f3fa47505aaff 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -37,7 +37,7 @@ true true - 9.4.34.v20201102 + 9.4.35.v20201120 _ _ From 48c0a5d4c6bdcc2acb2b28bd60836c1d89d7be43 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 22 Dec 2020 09:45:21 -0800 Subject: [PATCH 8/9] Remove jetty-util-ajax from shaded hadoop-cient-minicluster jar. jetty-security 9.4.35 adds a new dependency jetty-util-ajax. Exclude it to make shading pass. Change-Id: I9f62d76cd74bc89ede2c28d440dbe316545393aa --- hadoop-client-modules/hadoop-client-minicluster/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 8c4779790ec1c..0bedf1d235b95 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -840,6 +840,12 @@ */** + + org.eclipse.jetty:jetty-util-ajax + + */** + + org.eclipse.jetty:jetty-server From f7a601e7b045629d7fe0b954e7756b6eaffcc89e Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Wed, 30 Dec 2020 12:26:37 -0800 Subject: [PATCH 9/9] Use Response.setStatusWithReason() for RestCsrfPreventionFilter and image servlet. Change-Id: Id4aaf799a481902a2ca32f25399d454d376066f1 --- .../http/RestCsrfPreventionFilter.java | 5 ++++ .../hdfs/server/namenode/ImageServlet.java | 30 ++++++++++++------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java index 59cb0d6599595..b81ed8e90155e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java @@ -37,6 +37,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.eclipse.jetty.server.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -271,6 +272,10 @@ public void proceed() throws IOException, ServletException { @Override public void sendError(int code, String message) throws IOException { + if (httpResponse instanceof Response) { + ((Response)httpResponse).setStatusWithReason(code, message); + } + httpResponse.sendError(code, message); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java index 54c87383c7d36..54825d8a8a261 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.security.SecurityUtil; +import org.eclipse.jetty.server.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -119,7 +120,7 @@ private FSImage getAndValidateFSImage(ServletContext context, if (nnImage == null) { String errorMsg = "NameNode initialization not yet complete. " + "FSImage has not been set in the NameNode."; - response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg); + sendError(response, HttpServletResponse.SC_FORBIDDEN, errorMsg); throw new IOException(errorMsg); } return nnImage; @@ -218,7 +219,7 @@ private void serveFile(File file) throws IOException { } catch (Throwable t) { String errMsg = "GetImage failed. " + StringUtils.stringifyException(t); - response.sendError(HttpServletResponse.SC_GONE, errMsg); + sendError(response, HttpServletResponse.SC_GONE, errMsg); throw new IOException(errMsg); } finally { response.getOutputStream().close(); @@ -234,7 +235,7 @@ private void validateRequest(ServletContext context, Configuration conf, conf)) { String errorMsg = "Only Namenode, Secondary Namenode, and administrators may access " + "this servlet"; - response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg); + sendError(response, HttpServletResponse.SC_FORBIDDEN, errorMsg); LOG.warn("Received non-NN/SNN/administrator request for image or edits from " + request.getUserPrincipal().getName() + " at " @@ -247,7 +248,7 @@ private void validateRequest(ServletContext context, Configuration conf, && !myStorageInfoString.equals(theirStorageInfoString)) { String errorMsg = "This namenode has storage info " + myStorageInfoString + " but the secondary expected " + theirStorageInfoString; - response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg); + sendError(response, HttpServletResponse.SC_FORBIDDEN, errorMsg); LOG.warn("Received an invalid request file transfer request " + "from a secondary with storage info " + theirStorageInfoString); throw new IOException(errorMsg); @@ -578,7 +579,7 @@ public Void run() throws Exception { // we need a different response type here so the client can differentiate this // from the failure to upload due to (1) security, or (2) other checkpoints already // present - response.sendError(HttpServletResponse.SC_EXPECTATION_FAILED, + sendError(response, HttpServletResponse.SC_EXPECTATION_FAILED, "Nameode "+request.getLocalAddr()+" is currently not in a state which can " + "accept uploads of new fsimages. State: "+state); return null; @@ -593,7 +594,7 @@ public Void run() throws Exception { // if the node is attempting to upload an older transaction, we ignore it SortedSet larger = currentlyDownloadingCheckpoints.tailSet(imageRequest); if (larger.size() > 0) { - response.sendError(HttpServletResponse.SC_CONFLICT, + sendError(response, HttpServletResponse.SC_CONFLICT, "Another checkpointer is already in the process of uploading a" + " checkpoint made up to transaction ID " + larger.last()); return null; @@ -601,7 +602,7 @@ public Void run() throws Exception { //make sure no one else has started uploading one if (!currentlyDownloadingCheckpoints.add(imageRequest)) { - response.sendError(HttpServletResponse.SC_CONFLICT, + sendError(response, HttpServletResponse.SC_CONFLICT, "Either current namenode is checkpointing or another" + " checkpointer is already in the process of " + "uploading a checkpoint made at transaction ID " @@ -648,7 +649,7 @@ public Void run() throws Exception { (txid - lastCheckpointTxid) + " expecting at least " + checkpointTxnCount; LOG.info(message); - response.sendError(HttpServletResponse.SC_CONFLICT, message); + sendError(response, HttpServletResponse.SC_CONFLICT, message); return null; } @@ -658,7 +659,7 @@ public Void run() throws Exception { + "another checkpointer already uploaded an " + "checkpoint for txid " + txid; LOG.info(message); - response.sendError(HttpServletResponse.SC_CONFLICT, message); + sendError(response, HttpServletResponse.SC_CONFLICT, message); return null; } @@ -695,11 +696,20 @@ public Void run() throws Exception { }); } catch (Throwable t) { String errMsg = "PutImage failed. " + StringUtils.stringifyException(t); - response.sendError(HttpServletResponse.SC_GONE, errMsg); + sendError(response, HttpServletResponse.SC_GONE, errMsg); throw new IOException(errMsg); } } + private void sendError(HttpServletResponse response, int code, String message) + throws IOException { + if (response instanceof Response) { + ((Response)response).setStatusWithReason(code, message); + } + + response.sendError(code, message); + } + /* * Params required to handle put image request */