diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index f3a838e41b160..bf3d20b805d8c 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -133,35 +133,34 @@ runtime - com.sun.jersey - jersey-core + commons-logging + commons-logging compile - com.sun.jersey - jersey-servlet + log4j + log4j compile - - com.sun.jersey - jersey-json + org.glassfish.jersey.core + jersey-common compile - com.sun.jersey + org.glassfish.jersey.core jersey-server compile - commons-logging - commons-logging + org.glassfish.jersey.containers + jersey-container-servlet-core compile - log4j - log4j - compile + org.glassfish.jersey.inject + jersey-hk2 + runtime junit diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index dfc9436a6f03f..5f6771248485d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -53,7 +53,6 @@ import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; -import com.sun.jersey.spi.container.servlet.ServletContainer; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -105,6 +104,8 @@ import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.eclipse.jetty.webapp.WebAppContext; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.servlet.ServletContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -885,10 +886,24 @@ public void addJerseyResourcePackage(final String packageName, final String pathSpec, Map params) { LOG.info("addJerseyResourcePackage: packageName=" + packageName + ", pathSpec=" + pathSpec); - final ServletHolder sh = new ServletHolder(ServletContainer.class); - sh.setInitParameter("com.sun.jersey.config.property.resourceConfigClass", - "com.sun.jersey.api.core.PackagesResourceConfig"); - sh.setInitParameter("com.sun.jersey.config.property.packages", packageName); + final ResourceConfig config = new ResourceConfig().packages(packageName); + final ServletHolder sh = new ServletHolder(new ServletContainer(config)); + for (Map.Entry entry : params.entrySet()) { + sh.setInitParameter(entry.getKey(), entry.getValue()); + } + webAppContext.addServlet(sh, pathSpec); + } + + /** + * Add a Jersey resource config. + * @param config The Jersey ResourceConfig to be registered. + * @param pathSpec The path spec for the servlet + * @param params properties and features for ResourceConfig + */ + public void addJerseyResourceConfig(final ResourceConfig config, + final String pathSpec, Map params) { + LOG.info("addJerseryResourceConfig: pathSpec={}", pathSpec); + final ServletHolder sh = new ServletHolder(new ServletContainer(config)); for (Map.Entry entry : params.entrySet()) { sh.setInitParameter(entry.getKey(), entry.getValue()); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java index ffe7ee5676aee..bef8e74c3d93a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java @@ -72,7 +72,7 @@ public class DNS { * @return The host name associated with the provided IP * @throws NamingException If a NamingException is encountered */ - public static String reverseDns(InetAddress hostIp, @Nullable String ns) + public static String reverseDns(InetAddress hostIp, String ns) throws NamingException { // // Builds the reverse IP lookup form @@ -241,7 +241,7 @@ public static String getDefaultIP(String strInterface) * @throws UnknownHostException if the given interface is invalid */ public static String[] getHosts(String strInterface, - @Nullable String nameserver, + String nameserver, boolean tryfallbackResolution) throws UnknownHostException { final List hosts = new Vector(); @@ -350,8 +350,8 @@ public static String[] getHosts(String strInterface) * @throws UnknownHostException * If one is encountered while querying the default interface */ - public static String getDefaultHost(@Nullable String strInterface, - @Nullable String nameserver, + public static String getDefaultHost(String strInterface, + String nameserver, boolean tryfallbackResolution) throws UnknownHostException { if (strInterface == null || "default".equals(strInterface)) { @@ -378,7 +378,7 @@ public static String getDefaultHost(@Nullable String strInterface, * @throws UnknownHostException * If one is encountered while querying the default interface */ - public static String getDefaultHost(@Nullable String strInterface) + public static String getDefaultHost(String strInterface) throws UnknownHostException { return getDefaultHost(strInterface, null, false); } @@ -394,8 +394,8 @@ public static String getDefaultHost(@Nullable String strInterface) * @throws UnknownHostException * If one is encountered while querying the default interface */ - public static String getDefaultHost(@Nullable String strInterface, - @Nullable String nameserver) + public static String getDefaultHost(String strInterface, + String nameserver) throws UnknownHostException { return getDefaultHost(strInterface, nameserver, false); } diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 338af127f8bd7..84f066da78990 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -59,15 +59,30 @@ compile - com.sun.jersey - jersey-core + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider compile - com.sun.jersey + org.glassfish.jersey.core + jersey-common + compile + + + org.glassfish.jersey.core jersey-server compile + + org.glassfish.jersey.containers + jersey-container-servlet-core + compile + + + org.glassfish.jersey.inject + jersey-hk2 + runtime + javax.servlet javax.servlet-api diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties b/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties index e2afd41be0898..10a26ff5b7481 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-log4j.properties @@ -34,7 +34,6 @@ log4j.additivity.kms-audit=false log4j.rootLogger=INFO, kms log4j.logger.org.apache.hadoop=INFO -log4j.logger.com.sun.jersey.server.wadl.generators.WadlGeneratorJAXBGrammarGenerator=OFF # make zookeeper log level an explicit config, and not changing with rootLogger. log4j.logger.org.apache.zookeeper=INFO log4j.logger.org.apache.curator=INFO diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java index 4788dfd8755f7..4057a89b60f51 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java @@ -39,6 +39,7 @@ import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; +import javax.ws.rs.OPTIONS; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -114,6 +115,14 @@ private static URI getKeyURI(String domain, String keyName) { .build(domain, KMSRESTConstants.KEY_RESOURCE, keyName); } + @OPTIONS + public Response handleOptions() { + return Response.ok() + .header("Allow", "GET") + .header("Allow", "OPTIONS") + .build(); + } + @POST @Path(KMSRESTConstants.KEYS_RESOURCE) @Consumes(MediaType.APPLICATION_JSON) @@ -175,13 +184,9 @@ public KeyVersion run() throws Exception { keyVersion = removeKeyMaterial(keyVersion); } Map json = KMSUtil.toJSON(keyVersion); - String requestURL = KMSMDCFilter.getURL(); - int idx = requestURL.lastIndexOf(KMSRESTConstants.KEYS_RESOURCE); - requestURL = requestURL.substring(0, idx); LOG.trace("Exiting createKey Method."); return Response.created(getKeyURI(KMSRESTConstants.SERVICE_VERSION, name)) - .type(MediaType.APPLICATION_JSON) - .header("Location", getKeyURI(requestURL, name)).entity(json).build(); + .type(MediaType.APPLICATION_JSON).entity(json).build(); } catch (Exception e) { LOG.debug("Exception in createKey.", e); throw e; diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java index ceaa8bc815e5f..1bc142773666d 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java @@ -19,13 +19,12 @@ import org.apache.hadoop.classification.InterfaceAudience; -import com.sun.jersey.api.container.ContainerException; - import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.util.HttpExceptionUtils; +import org.glassfish.jersey.server.ContainerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-kms/src/main/resources/webapps/kms/WEB-INF/web.xml b/hadoop-common-project/hadoop-kms/src/main/resources/webapps/kms/WEB-INF/web.xml index 737236c24a1b5..9565d378d444e 100644 --- a/hadoop-common-project/hadoop-kms/src/main/resources/webapps/kms/WEB-INF/web.xml +++ b/hadoop-common-project/hadoop-kms/src/main/resources/webapps/kms/WEB-INF/web.xml @@ -20,23 +20,11 @@ webservices-driver - com.sun.jersey.spi.container.servlet.ServletContainer + org.glassfish.jersey.servlet.ServletContainer - com.sun.jersey.config.property.packages + jersey.config.server.provider.packages org.apache.hadoop.crypto.key.kms.server - - - 1 diff --git a/hadoop-common-project/hadoop-kms/src/test/resources/log4j.properties b/hadoop-common-project/hadoop-kms/src/test/resources/log4j.properties index 73c48534a0a01..50f5d4edf8c8b 100644 --- a/hadoop-common-project/hadoop-kms/src/test/resources/log4j.properties +++ b/hadoop-common-project/hadoop-kms/src/test/resources/log4j.properties @@ -25,8 +25,6 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n log4j.rootLogger=INFO, stdout log4j.logger.org.apache.hadoop.conf=ERROR log4j.logger.org.apache.hadoop.crytpo.key.kms.server=ALL -log4j.logger.com.sun.jersey.server.wadl.generators.WadlGeneratorJAXBGrammarGenerator=OFF -log4j.logger.com.sun.jersey.server.wadl.generators.AbstractWadlGeneratorGrammarGenerator=OFF log4j.logger.org.apache.hadoop.security=OFF log4j.logger.org.apache.directory.server.core=OFF log4j.logger.org.apache.hadoop.util.NativeCodeLoader=OFF diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index de7112270883a..296ab6148c13d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -58,12 +58,12 @@ compile - com.sun.jersey - jersey-core + org.glassfish.jersey.core + jersey-common compile - com.sun.jersey + org.glassfish.jersey.core jersey-server compile diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java index 4739e42137ccb..e1eaf424087b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java @@ -18,10 +18,10 @@ package org.apache.hadoop.fs.http.server; -import com.sun.jersey.api.container.ContainerException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.lib.service.FileSystemAccessException; import org.apache.hadoop.lib.wsrs.ExceptionProvider; +import org.glassfish.jersey.server.ContainerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java index f6c84dcae4e07..f59f13726e095 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java @@ -33,7 +33,6 @@ import org.apache.hadoop.lib.wsrs.ShortParam; import org.apache.hadoop.lib.wsrs.StringParam; import org.apache.hadoop.util.StringUtils; -import javax.ws.rs.ext.Provider; import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; @@ -41,7 +40,6 @@ /** * HttpFS ParametersProvider. */ -@Provider @InterfaceAudience.Private @SuppressWarnings("unchecked") public class HttpFSParametersProvider extends ParametersProvider { @@ -125,7 +123,7 @@ public class HttpFSParametersProvider extends ParametersProvider { PARAMS_DEF.put(Operation.SATISFYSTORAGEPOLICY, new Class[] {}); } - public HttpFSParametersProvider() { + HttpFSParametersProvider() { super(HttpFSFileSystem.OP_PARAM, HttpFSFileSystem.Operation.class, PARAMS_DEF); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java index 0e7038b635d35..f9fa1e652b9e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java @@ -140,6 +140,15 @@ private UserGroupInformation getHttpUGI(HttpServletRequest request) { } + private static final HttpFSParametersProvider PARAMETERS_PROVIDER = + new HttpFSParametersProvider(); + + private Parameters getParams(HttpServletRequest request) { + return PARAMETERS_PROVIDER.get(request); + } + + private static final Object[] NULL = new Object[0]; + /** * Executes a {@link FileSystemAccess.FileSystemExecutor} using a filesystem for the effective * user. @@ -199,7 +208,6 @@ private void enforceRootPath(HttpFSFileSystem.Operation op, String path) { * * @param uriInfo uri info of the request. * @param op the HttpFS operation of the request. - * @param params the HttpFS parameters of the request. * * @return the request response. * @@ -213,10 +221,9 @@ private void enforceRootPath(HttpFSFileSystem.Operation op, String path) { @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) public Response getRoot(@Context UriInfo uriInfo, @QueryParam(OperationParam.NAME) OperationParam op, - @Context Parameters params, @Context HttpServletRequest request) throws IOException, FileSystemAccessException { - return get("", uriInfo, op, params, request); + return get("", uriInfo, op, request); } private String makeAbsolute(String path) { @@ -229,7 +236,6 @@ private String makeAbsolute(String path) { * @param path the path for operation. * @param uriInfo uri info of the request. * @param op the HttpFS operation of the request. - * @param params the HttpFS parameters of the request. * * @return the request response. * @@ -246,7 +252,6 @@ private String makeAbsolute(String path) { public Response get(@PathParam("path") String path, @Context UriInfo uriInfo, @QueryParam(OperationParam.NAME) OperationParam op, - @Context Parameters params, @Context HttpServletRequest request) throws IOException, FileSystemAccessException { // Restrict access to only GETFILESTATUS and LISTSTATUS in write-only mode @@ -258,6 +263,7 @@ public Response get(@PathParam("path") String path, UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); + final Parameters params = getParams(request); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); MDC.put("hostname", request.getRemoteAddr()); switch (op.value()) { @@ -517,7 +523,6 @@ private URI createOpenRedirectionURL(UriInfo uriInfo) { * * @param path the path for operation. * @param op the HttpFS operation of the request. - * @param params the HttpFS parameters of the request. * * @return the request response. * @@ -532,7 +537,6 @@ private URI createOpenRedirectionURL(UriInfo uriInfo) { @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) public Response delete(@PathParam("path") String path, @QueryParam(OperationParam.NAME) OperationParam op, - @Context Parameters params, @Context HttpServletRequest request) throws IOException, FileSystemAccessException { // Do not allow DELETE commands in read-only mode @@ -542,6 +546,7 @@ public Response delete(@PathParam("path") String path, UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); + final Parameters params = getParams(request); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); MDC.put("hostname", request.getRemoteAddr()); switch (op.value()) { @@ -579,7 +584,7 @@ public Response delete(@PathParam("path") String path, * @param is the inputstream for the request payload. * @param uriInfo the of the request. * @param op the HttpFS operation of the request. - * @param params the HttpFS parameters of the request. + * @param request the HttpFS request. * * @return the request response. * @@ -593,9 +598,9 @@ public Response delete(@PathParam("path") String path, @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8 }) public Response postRoot(InputStream is, @Context UriInfo uriInfo, @QueryParam(OperationParam.NAME) OperationParam op, - @Context Parameters params, @Context HttpServletRequest request) + @Context HttpServletRequest request) throws IOException, FileSystemAccessException { - return post(is, uriInfo, "/", op, params, request); + return post(is, uriInfo, "/", op, request); } /** @@ -605,7 +610,6 @@ public Response postRoot(InputStream is, @Context UriInfo uriInfo, * @param uriInfo the of the request. * @param path the path for operation. * @param op the HttpFS operation of the request. - * @param params the HttpFS parameters of the request. * * @return the request response. * @@ -623,7 +627,6 @@ public Response post(InputStream is, @Context UriInfo uriInfo, @PathParam("path") String path, @QueryParam(OperationParam.NAME) OperationParam op, - @Context Parameters params, @Context HttpServletRequest request) throws IOException, FileSystemAccessException { // Do not allow POST commands in read-only mode @@ -633,6 +636,7 @@ public Response post(InputStream is, UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); + final Parameters params = getParams(request); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); MDC.put("hostname", request.getRemoteAddr()); switch (op.value()) { @@ -712,10 +716,11 @@ public Response post(InputStream is, */ protected URI createUploadRedirectionURL(UriInfo uriInfo, Enum uploadOperation) { UriBuilder uriBuilder = uriInfo.getRequestUriBuilder(); - uriBuilder = uriBuilder.replaceQueryParam(OperationParam.NAME, uploadOperation). - queryParam(DataParam.NAME, Boolean.TRUE) - .replaceQueryParam(NoRedirectParam.NAME, (Object[]) null); - return uriBuilder.build(null); + uriBuilder = uriBuilder.replaceQueryParam(OperationParam.NAME, uploadOperation) + .queryParam(DataParam.NAME, Boolean.TRUE) + .replaceQueryParam(NoRedirectParam.NAME, (Object[]) null); + // Workaround: NPE occurs when using null in Jersey 2.29 + return uriBuilder.build(NULL); } /** @@ -723,7 +728,7 @@ protected URI createUploadRedirectionURL(UriInfo uriInfo, Enum uploadOperatio * @param is the inputstream for the request payload. * @param uriInfo the of the request. * @param op the HttpFS operation of the request. - * @param params the HttpFS parameters of the request. + * @param request the HttpFS request. * * @return the request response. * @@ -737,9 +742,9 @@ protected URI createUploadRedirectionURL(UriInfo uriInfo, Enum uploadOperatio @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8 }) public Response putRoot(InputStream is, @Context UriInfo uriInfo, @QueryParam(OperationParam.NAME) OperationParam op, - @Context Parameters params, @Context HttpServletRequest request) + @Context HttpServletRequest request) throws IOException, FileSystemAccessException { - return put(is, uriInfo, "/", op, params, request); + return put(is, uriInfo, "/", op, request); } /** @@ -749,7 +754,6 @@ public Response putRoot(InputStream is, @Context UriInfo uriInfo, * @param uriInfo the of the request. * @param path the path for operation. * @param op the HttpFS operation of the request. - * @param params the HttpFS parameters of the request. * * @return the request response. * @@ -767,7 +771,6 @@ public Response put(InputStream is, @Context UriInfo uriInfo, @PathParam("path") String path, @QueryParam(OperationParam.NAME) OperationParam op, - @Context Parameters params, @Context HttpServletRequest request) throws IOException, FileSystemAccessException { // Do not allow PUT commands in read-only mode @@ -777,6 +780,7 @@ public Response put(InputStream is, UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); + final Parameters params = getParams(request); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); MDC.put("hostname", request.getRemoteAddr()); switch (op.value()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ParametersProvider.java index 56a999f519cdf..7b97e762a1a97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ParametersProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/ParametersProvider.java @@ -19,31 +19,21 @@ package org.apache.hadoop.lib.wsrs; import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; -import com.sun.jersey.api.core.HttpContext; -import com.sun.jersey.core.spi.component.ComponentContext; -import com.sun.jersey.core.spi.component.ComponentScope; -import com.sun.jersey.server.impl.inject.AbstractHttpContextInjectable; -import com.sun.jersey.spi.inject.Injectable; -import com.sun.jersey.spi.inject.InjectableProvider; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.util.StringUtils; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MultivaluedMap; -import java.lang.reflect.Type; +import javax.servlet.http.HttpServletRequest; import java.text.MessageFormat; import java.util.HashMap; import java.util.List; import java.util.Map; /** - * Jersey provider that parses the request parameters based on the - * given parameter definition. + * Provider that parses the request parameters based on the + * given parameter definition. */ @InterfaceAudience.Private -public class ParametersProvider - extends AbstractHttpContextInjectable - implements InjectableProvider { +public class ParametersProvider { private String driverParam; private Class enumClass; @@ -56,34 +46,42 @@ public ParametersProvider(String driverParam, Class enumClass, this.paramsDef = paramsDef; } - @Override - @SuppressWarnings("unchecked") - public Parameters getValue(HttpContext httpContext) { - Map>> map = new HashMap>>(); - Map> queryString = - httpContext.getRequest().getQueryParameters(); - String str = ((MultivaluedMap) queryString). - getFirst(driverParam); + private Param newParam(Class> paramClass) { + try { + return paramClass.newInstance(); + } catch (Exception ex) { + throw new UnsupportedOperationException( + MessageFormat.format( + "Param class [{0}] does not have default constructor", + paramClass.getName())); + } + } + + public Parameters get(HttpServletRequest request) { + Map>> map = new HashMap<>(); + + Map queryString = request.getParameterMap(); + String str = queryString.get(driverParam)[0]; if (str == null) { throw new IllegalArgumentException( - MessageFormat.format("Missing Operation parameter [{0}]", - driverParam)); + MessageFormat.format("Missing Operation parameter [{0}]", + driverParam)); } Enum op; try { op = Enum.valueOf(enumClass, StringUtils.toUpperCase(str)); } catch (IllegalArgumentException ex) { throw new IllegalArgumentException( - MessageFormat.format("Invalid Operation [{0}]", str)); + MessageFormat.format("Invalid Operation [{0}]", str)); } if (!paramsDef.containsKey(op)) { throw new IllegalArgumentException( - MessageFormat.format("Unsupported Operation [{0}]", op)); + MessageFormat.format("Unsupported Operation [{0}]", op)); } for (Class> paramClass : paramsDef.get(op)) { Param param = newParam(paramClass); List> paramList = Lists.newArrayList(); - List ps = queryString.get(param.getName()); + String[] ps = queryString.get(param.getName()); if (ps != null) { for (String p : ps) { try { @@ -103,25 +101,4 @@ public Parameters getValue(HttpContext httpContext) { } return new Parameters(map); } - - private Param newParam(Class> paramClass) { - try { - return paramClass.newInstance(); - } catch (Exception ex) { - throw new UnsupportedOperationException( - MessageFormat.format( - "Param class [{0}] does not have default constructor", - paramClass.getName())); - } - } - - @Override - public ComponentScope getScope() { - return ComponentScope.PerRequest; - } - - @Override - public Injectable getInjectable(ComponentContext componentContext, Context context, Type type) { - return (type.equals(Parameters.class)) ? this : null; - } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/webapps/webhdfs/WEB-INF/web.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/webapps/webhdfs/WEB-INF/web.xml index 3da9a5cf9761f..c0567ba96edcf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/webapps/webhdfs/WEB-INF/web.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/webapps/webhdfs/WEB-INF/web.xml @@ -20,23 +20,11 @@ webservices-driver - com.sun.jersey.spi.container.servlet.ServletContainer + org.glassfish.jersey.servlet.ServletContainer - com.sun.jersey.config.property.packages + jersey.config.server.provider.packages org.apache.hadoop.fs.http.server,org.apache.hadoop.lib.wsrs - - - 1 diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml index 4c0b3aedb0ef0..a4ba0abd892ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml @@ -20,23 +20,11 @@ webservices-driver - com.sun.jersey.spi.container.servlet.ServletContainer + org.glassfish.jersey.servlet.ServletContainer - com.sun.jersey.config.property.packages + jersey.config.server.provider.packages org.apache.hadoop.fs.http.server,org.apache.hadoop.lib.wsrs - - - 1 diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/default-log4j.properties b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/default-log4j.properties index 45a8412f5a76c..9ef5ec63d3464 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/default-log4j.properties +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/default-log4j.properties @@ -23,4 +23,3 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n -log4j.logger.com.sun.jersey.server.wadl.generators.AbstractWadlGeneratorGrammarGenerator=OFF diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml index 18a91cbcdc6f8..7967e1f24f0b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml @@ -98,16 +98,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> jetty-util compile - - com.sun.jersey - jersey-core - compile - - - com.sun.jersey - jersey-server - compile - commons-cli commons-cli diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml index 23e0b8feb0df9..ec70a778d3400 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml @@ -93,6 +93,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> com.fasterxml.jackson.core jackson-databind + + org.codehaus.jettison + jettison + junit junit diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java index 39f06a3b66f4d..34297a3e0362e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java @@ -35,9 +35,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import com.sun.jersey.spi.container.ResourceFilters; import org.apache.hadoop.hdfs.web.JsonUtil; -import org.apache.hadoop.hdfs.web.ParamFilter; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.AccessTimeParam; import org.apache.hadoop.hdfs.web.resources.AclPermissionParam; @@ -110,7 +108,6 @@ * {@link NamenodeWebHdfsMethods}, and tries to reuse as much as possible. */ @Path("") -@ResourceFilters(ParamFilter.class) public class RouterWebHdfsMethods extends NamenodeWebHdfsMethods { private static final Logger LOG = LoggerFactory.getLogger(RouterWebHdfsMethods.class); diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 90ce0020d7caf..39fc2c6e37168 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -88,15 +88,25 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> compile - com.sun.jersey - jersey-core + org.glassfish.jersey.core + jersey-common compile - com.sun.jersey + org.glassfish.jersey.core jersey-server compile + + org.glassfish.jersey.inject + jersey-hk2 + runtime + + + org.glassfish.jersey.media + jersey-media-json-jettison + test + commons-cli commons-cli diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java index 02ec25c13c874..b74558aaf631c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java @@ -18,11 +18,11 @@ package org.apache.hadoop.hdfs.server.datanode.web.webhdfs; import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; -import com.sun.jersey.api.ParamException; -import com.sun.jersey.api.container.ContainerException; import io.netty.buffer.Unpooled; import io.netty.handler.codec.http.DefaultFullHttpResponse; import io.netty.handler.codec.http.HttpResponseStatus; +import org.glassfish.jersey.server.ContainerException; +import org.glassfish.jersey.server.ParamException; import org.slf4j.Logger; import org.apache.hadoop.hdfs.web.JsonUtil; import org.apache.hadoop.ipc.RemoteException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index 33913227af2ce..5693cb33f4b84 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -39,16 +39,19 @@ import org.apache.hadoop.hdfs.server.common.TokenVerifier; import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; +import org.apache.hadoop.hdfs.web.ParamFilter; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.AclPermissionParam; import org.apache.hadoop.hdfs.web.resources.Param; import org.apache.hadoop.hdfs.web.resources.UserParam; +import org.apache.hadoop.hdfs.web.resources.UserProvider; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.http.RestCsrfPreventionFilter; - -import com.sun.jersey.api.core.ResourceConfig; +import org.glassfish.jersey.internal.inject.AbstractBinder; +import org.glassfish.jersey.server.ResourceConfig; /** * Encapsulates the HTTP server started by the NameNode. @@ -101,12 +104,24 @@ public static void initWebHdfs(Configuration conf, String hostname, new String[] {pathSpec}); } + // add a filter to change parameter names to lower cases + HttpServer2.defineFilter(httpServer2.getWebAppContext(), + ParamFilter.class.getName(), ParamFilter.class.getName(), null, + new String[] {pathSpec}); + // add webhdfs packages final Map params = new HashMap<>(); - params.put(ResourceConfig.FEATURE_MATCH_MATRIX_PARAMS, "true"); - httpServer2.addJerseyResourcePackage( - jerseyResourcePackage + ";" + Param.class.getPackage().getName(), - pathSpec, params); + ResourceConfig config = new ResourceConfig(); + config.packages(jerseyResourcePackage, Param.class.getPackage().getName()); + config.register(new AbstractBinder() { + // add a factory to generate UserGroupInformation + @Override + protected void configure() { + bindFactory(UserProvider.class) + .to(UserGroupInformation.class); + } + }); + httpServer2.addJerseyResourceConfig(config, pathSpec, params); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index e7bf32fcc05f0..6728e0817042f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -54,6 +54,7 @@ import javax.ws.rs.core.StreamingOutput; import javax.ws.rs.core.Response.ResponseBuilder; import javax.ws.rs.core.Response.Status; +import javax.ws.rs.core.UriInfo; import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.fs.StorageType; @@ -98,7 +99,6 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.web.JsonUtil; -import org.apache.hadoop.hdfs.web.ParamFilter; import org.apache.hadoop.hdfs.web.WebHdfsConstants; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.*; @@ -117,11 +117,9 @@ import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; -import com.sun.jersey.spi.container.ResourceFilters; /** Web-hdfs NameNode implementation. */ @Path("") -@ResourceFilters(ParamFilter.class) public class NamenodeWebHdfsMethods { public static final Logger LOG = LoggerFactory.getLogger(NamenodeWebHdfsMethods.class); @@ -448,12 +446,12 @@ private URI redirectURI(ResponseBuilder rb, final NameNode namenode, /** Handle HTTP PUT request for the root. */ @PUT - @Path("/") @Consumes({"*/*"}) @Produces({MediaType.APPLICATION_OCTET_STREAM + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8}) public Response putRoot( @Context final UserGroupInformation ugi, + @Context final UriInfo uriInfo, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) @@ -523,7 +521,7 @@ public Response putRoot( @DefaultValue(StorageTypeParam.DEFAULT) final StorageTypeParam storageType ) throws IOException, InterruptedException { - return put(ugi, delegation, username, doAsUser, ROOT, op, destination, + return put(ugi, uriInfo, delegation, username, doAsUser, op, destination, owner, group, permission, unmaskedPermission, overwrite, bufferSize, replication, blockSize, modificationTime, accessTime, renameOptions, createParent, delegationTokenArgument, aclPermission, xattrName, @@ -552,13 +550,13 @@ protected void validateOpParams(HttpOpParam op, Param... params) { MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8}) public Response put( @Context final UserGroupInformation ugi, + @Context final UriInfo uriInfo, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) final UserParam username, @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) final DoAsParam doAsUser, - @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT) final PutOpParam op, @QueryParam(DestinationParam.NAME) @DefaultValue(DestinationParam.DEFAULT) @@ -621,6 +619,7 @@ public Response put( @QueryParam(StorageTypeParam.NAME) @DefaultValue(StorageTypeParam.DEFAULT) final StorageTypeParam storageType ) throws IOException, InterruptedException { + final UriFsPathParam path = new UriFsPathParam(uriInfo.getPath()); init(ugi, delegation, username, doAsUser, path, op, destination, owner, group, permission, unmaskedPermission, overwrite, bufferSize, replication, blockSize, modificationTime, accessTime, renameOptions, @@ -874,7 +873,6 @@ protected Response put( /** Handle HTTP POST request for the root. */ @POST - @Path("/") @Consumes({"*/*"}) @Produces({MediaType.APPLICATION_OCTET_STREAM + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8}) @@ -1004,11 +1002,11 @@ protected Response post( /** Handle HTTP GET request for the root. */ @GET - @Path("/") @Produces({MediaType.APPLICATION_OCTET_STREAM + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8}) public Response getRoot( @Context final UserGroupInformation ugi, + @Context final UriInfo uriInfo, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) @@ -1046,7 +1044,7 @@ public Response getRoot( @QueryParam(StartAfterParam.NAME) @DefaultValue(StartAfterParam.DEFAULT) final StartAfterParam startAfter ) throws IOException, InterruptedException { - return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length, + return get(ugi, uriInfo, delegation, username, doAsUser, op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, fsAction, snapshotName, oldSnapshotName, tokenKind, tokenService, noredirect, startAfter); @@ -1059,13 +1057,13 @@ public Response getRoot( MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8}) public Response get( @Context final UserGroupInformation ugi, + @Context final UriInfo uriInfo, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) final UserParam username, @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) final DoAsParam doAsUser, - @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) final GetOpParam op, @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) @@ -1097,7 +1095,7 @@ public Response get( @QueryParam(StartAfterParam.NAME) @DefaultValue(StartAfterParam.DEFAULT) final StartAfterParam startAfter ) throws IOException, InterruptedException { - + final UriFsPathParam path = new UriFsPathParam(uriInfo.getPath()); init(ugi, delegation, username, doAsUser, path, op, offset, length, renewer, bufferSize, xattrEncoding, excludeDatanodes, fsAction, snapshotName, oldSnapshotName, tokenKind, tokenService, startAfter); @@ -1488,10 +1486,10 @@ public Void run() throws IOException { /** Handle HTTP DELETE request for the root. */ @DELETE - @Path("/") @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) public Response deleteRoot( @Context final UserGroupInformation ugi, + @Context final UriInfo uriInfo, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) @@ -1505,7 +1503,7 @@ public Response deleteRoot( @QueryParam(SnapshotNameParam.NAME) @DefaultValue(SnapshotNameParam.DEFAULT) final SnapshotNameParam snapshotName ) throws IOException, InterruptedException { - return delete(ugi, delegation, username, doAsUser, ROOT, op, recursive, + return delete(ugi, uriInfo, delegation, username, doAsUser, op, recursive, snapshotName); } @@ -1515,13 +1513,13 @@ public Response deleteRoot( @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) public Response delete( @Context final UserGroupInformation ugi, + @Context final UriInfo uriInfo, @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) final DelegationParam delegation, @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) final UserParam username, @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) final DoAsParam doAsUser, - @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, @QueryParam(DeleteOpParam.NAME) @DefaultValue(DeleteOpParam.DEFAULT) final DeleteOpParam op, @QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT) @@ -1529,7 +1527,7 @@ public Response delete( @QueryParam(SnapshotNameParam.NAME) @DefaultValue(SnapshotNameParam.DEFAULT) final SnapshotNameParam snapshotName ) throws IOException, InterruptedException { - + final UriFsPathParam path = new UriFsPathParam(uriInfo.getPath()); init(ugi, delegation, username, doAsUser, path, op, recursive, snapshotName); return doAs(ugi, new PrivilegedExceptionAction() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ParamFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ParamFilter.java index febe1253a829d..e8d0f7f515174 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ParamFilter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ParamFilter.java @@ -17,70 +17,82 @@ */ package org.apache.hadoop.hdfs.web; -import java.net.URI; -import java.util.List; +import java.io.IOException; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; import java.util.Map; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.UriBuilder; - -import com.sun.jersey.spi.container.ContainerRequest; -import com.sun.jersey.spi.container.ContainerRequestFilter; -import com.sun.jersey.spi.container.ContainerResponseFilter; -import com.sun.jersey.spi.container.ResourceFilter; -import org.apache.hadoop.util.StringUtils; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; /** * A filter to change parameter names to lower cases * so that parameter names are considered as case insensitive. */ -public class ParamFilter implements ResourceFilter { - private static final ContainerRequestFilter LOWER_CASE - = new ContainerRequestFilter() { - @Override - public ContainerRequest filter(final ContainerRequest request) { - final MultivaluedMap parameters = request.getQueryParameters(); - if (containsUpperCase(parameters.keySet())) { - //rebuild URI - final URI lower = rebuildQuery(request.getRequestUri(), parameters); - request.setUris(request.getBaseUri(), lower); - } - return request; - } - }; +public class ParamFilter implements Filter { @Override - public ContainerRequestFilter getRequestFilter() { - return LOWER_CASE; + public void init(FilterConfig filterConfig) throws ServletException { } @Override - public ContainerResponseFilter getResponseFilter() { - return null; + public void destroy() { } - /** Do the strings contain upper case letters? */ - static boolean containsUpperCase(final Iterable strings) { - for(String s : strings) { - for(int i = 0; i < s.length(); i++) { - if (Character.isUpperCase(s.charAt(i))) { - return true; - } - } + @Override + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain chain) + throws IOException, ServletException { + if (request instanceof HttpServletRequest) { + chain.doFilter(new CustomHttpServletRequestWrapper( + (HttpServletRequest) request), response); + } else { + chain.doFilter(request, response); } - return false; } - /** Rebuild the URI query with lower case parameter names. */ - private static URI rebuildQuery(final URI uri, - final MultivaluedMap parameters) { - UriBuilder b = UriBuilder.fromUri(uri).replaceQuery(""); - for(Map.Entry> e : parameters.entrySet()) { - final String key = StringUtils.toLowerCase(e.getKey()); - for(String v : e.getValue()) { - b = b.queryParam(key, v); + private static final class CustomHttpServletRequestWrapper + extends HttpServletRequestWrapper { + + private Map lowerCaseParams = new HashMap<>(); + + private CustomHttpServletRequestWrapper(HttpServletRequest request) { + super(request); + Map originalParams = request.getParameterMap(); + for (Map.Entry entry : originalParams.entrySet()) { + lowerCaseParams.put(entry.getKey().toLowerCase(), entry.getValue()); + } + } + + public String getParameter(String name) { + String[] values = getParameterValues(name); + if (values != null && values.length > 0) { + return values[0]; + } else { + return null; } } - return b.build(); + + @Override + public Map getParameterMap() { + return Collections.unmodifiableMap(lowerCaseParams); + } + + @Override + public Enumeration getParameterNames() { + return Collections.enumeration(lowerCaseParams.keySet()); + } + + @Override + public String[] getParameterValues(String name) { + return lowerCaseParams.get(name); + } } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ExceptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ExceptionHandler.java index 02b70c16be649..8f1d554e641c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ExceptionHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ExceptionHandler.java @@ -27,6 +27,8 @@ import javax.ws.rs.ext.ExceptionMapper; import javax.ws.rs.ext.Provider; +import org.glassfish.jersey.server.ContainerException; +import org.glassfish.jersey.server.ParamException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hdfs.web.JsonUtil; @@ -36,8 +38,6 @@ import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.api.ParamException; -import com.sun.jersey.api.container.ContainerException; /** Handle exceptions. */ @Provider diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserProvider.java index 32b3369f7a5a2..8e8977d4dd3b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserProvider.java @@ -17,57 +17,44 @@ */ package org.apache.hadoop.hdfs.web.resources; -import java.io.IOException; -import java.lang.reflect.Type; - -import javax.servlet.ServletContext; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.core.Context; -import javax.ws.rs.ext.Provider; - +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; -import com.sun.jersey.api.core.HttpContext; -import com.sun.jersey.core.spi.component.ComponentContext; -import com.sun.jersey.core.spi.component.ComponentScope; -import com.sun.jersey.server.impl.inject.AbstractHttpContextInjectable; -import com.sun.jersey.spi.inject.Injectable; -import com.sun.jersey.spi.inject.InjectableProvider; +import javax.inject.Inject; +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.util.function.Supplier; -/** Inject user information to http operations. */ -@Provider -public class UserProvider - extends AbstractHttpContextInjectable - implements InjectableProvider { - @Context HttpServletRequest request; - @Context ServletContext servletcontext; +/** + * Jersey provider to acquire {@link UserGroupInformation} + * from {@link HttpServletRequest}. + */ +@InterfaceAudience.Private +public class UserProvider implements Supplier { + + private final HttpServletRequest request; + private final ServletContext servletcontext; + + @Inject + public UserProvider(HttpServletRequest request) { + this.request = request; + this.servletcontext = request.getServletContext(); + } @Override - public UserGroupInformation getValue(final HttpContext context) { + public UserGroupInformation get() { final Configuration conf = (Configuration) servletcontext .getAttribute(JspHelper.CURRENT_CONF); try { return JspHelper.getUGI(servletcontext, request, conf, - AuthenticationMethod.KERBEROS, false); + UserGroupInformation.AuthenticationMethod.KERBEROS, false); } catch (IOException e) { throw new SecurityException( SecurityUtil.FAILED_TO_GET_UGI_MSG_HEADER + " " + e, e); } } - - @Override - public ComponentScope getScope() { - return ComponentScope.PerRequest; - } - - @Override - public Injectable getInjectable( - final ComponentContext componentContext, final Context context, - final Type type) { - return type.equals(UserGroupInformation.class)? this : null; - } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/log4j.properties b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/log4j.properties index 997854dcce7a3..73788467112e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/log4j.properties +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/log4j.properties @@ -47,6 +47,3 @@ log4j.appender.DNMETRICSRFA.layout=org.apache.log4j.PatternLayout log4j.appender.DNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n log4j.appender.DNMETRICSRFA.MaxBackupIndex=1 log4j.appender.DNMETRICSRFA.MaxFileSize=64MB - -# Supress KMS error log -log4j.logger.com.sun.jersey.server.wadl.generators.WadlGeneratorJAXBGrammarGenerator=OFF \ No newline at end of file diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 2fe07c357400b..3c2713537d510 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -68,6 +68,7 @@ 1.19 + 2.29.1 1.9.13 @@ -136,7 +137,7 @@ 2.7.5 1.1 5.2.0 - 2.2.21 + 2.4.4 2.2.4 3.1.0-incubating 4.1.0-incubating @@ -863,7 +864,71 @@ jersey-server ${jersey.version} - + + org.glassfish.jersey.core + jersey-common + ${jersey2.version} + + + org.glassfish.jersey.core + jersey-server + ${jersey2.version} + + + org.glassfish.jersey.core + jersey-client + ${jersey2.version} + + + org.glassfish.jersey.containers + jersey-container-servlet-core + ${jersey2.version} + + + org.glassfish.jersey.inject + jersey-hk2 + ${jersey2.version} + + + org.glassfish.jersey.media + jersey-media-json-jackson + ${jersey2.version} + + + org.glassfish.jersey.media + jersey-media-json-jettison + ${jersey2.version} + + + org.glassfish.jersey.test-framework + jersey-test-framework-core + ${jersey2.version} + + + org.glassfish.jersey.test-framework.providers + jersey-test-framework-provider-grizzly2 + ${jersey2.version} + + + org.glassfish.hk2 + guice-bridge + 2.6.1 + + + jakarta.annotation + jakarta.annotation-api + 1.3.5 + + + net.jodah + failsafe + 2.3.1 + + + org.javassist + javassist + 3.22.0-CR2 + com.google.inject guice diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index 2c5749cf2f567..0f468f818e2f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -88,6 +88,10 @@ jersey-test-framework-grizzly2 test + + org.codehaus.jettison + jettison + org.apache.hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index ca71f15c31dc4..5c41a51d7110a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -19,14 +19,6 @@ package org.apache.hadoop.yarn.client.cli; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientRequest; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.api.client.WebResource.Builder; -import com.sun.jersey.api.client.filter.ClientFilter; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -47,7 +39,12 @@ import java.util.Set; import java.util.TreeMap; import java.util.regex.Pattern; +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; @@ -145,7 +142,7 @@ public int run(String[] args) throws Exception { yarnClient.close(); } if (webServiceClient != null) { - webServiceClient.destroy(); + webServiceClient.close(); } } } @@ -450,44 +447,38 @@ protected List getAMContainerInfoForRMWebService( } private List getAMContainerInfoFromRM( - String webAppAddress, String appId) throws ClientHandlerException, - UniformInterfaceException, JSONException { + String webAppAddress, String appId) throws ProcessingException, + IllegalStateException, JSONException { List amContainersList = new ArrayList(); - ClientResponse response = null; - try { - Builder builder = webServiceClient.resource(webAppAddress) - .path("ws").path("v1").path("cluster") - .path("apps").path(appId).path("appattempts") - .accept(MediaType.APPLICATION_JSON); - response = builder.get(ClientResponse.class); - JSONObject json = response.getEntity(JSONObject.class) + final WebTarget target = webServiceClient.target(webAppAddress) + .path("ws").path("v1").path("cluster") + .path("apps").path(appId).path("appattempts"); + try (Response response = target.request(MediaType.APPLICATION_JSON) + .get(Response.class)) { + JSONObject json = response.readEntity(JSONObject.class) .getJSONObject("appAttempts"); JSONArray requests = json.getJSONArray("appAttempt"); for (int j = 0; j < requests.length(); j++) { amContainersList.add(requests.getJSONObject(j)); } return amContainersList; - } finally { - if (response != null) { - response.close(); - } } } private List getAMContainerInfoForAHSWebService( - Configuration conf, String appId) throws ClientHandlerException, - UniformInterfaceException, JSONException { + Configuration conf, String appId) throws ProcessingException, + IllegalStateException, JSONException { String webAppAddress = WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils.getAHSWebAppURLWithoutScheme(conf); - WebResource webResource = webServiceClient.resource(webAppAddress); + final WebTarget target = webServiceClient.target(webAppAddress); - ClientResponse response = - webResource.path("ws").path("v1").path("applicationhistory") + Response response = + target.path("ws").path("v1").path("applicationhistory") .path("apps").path(appId).path("appattempts") - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - JSONObject json = response.getEntity(JSONObject.class); + .request(MediaType.APPLICATION_JSON) + .get(Response.class); + JSONObject json = response.readEntity(JSONObject.class); JSONArray requests = json.getJSONArray("appAttempt"); List amContainersList = new ArrayList(); for (int i = 0; i < requests.length(); i++) { @@ -530,18 +521,18 @@ private List> getContainerLogFiles( List> logFileInfos = new ArrayList<>(); try { - WebResource webResource = webServiceClient - .resource(WebAppUtils.getHttpSchemePrefix(conf) + nodeHttpAddress); - ClientResponse response = - webResource.path("ws").path("v1").path("node").path("containers") + WebTarget target = webServiceClient + .target(WebAppUtils.getHttpSchemePrefix(conf) + nodeHttpAddress); + Response response = + target.path("ws").path("v1").path("node").path("containers") .path(containerIdStr).path("logs") - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + .request(MediaType.APPLICATION_JSON) + .get(Response.class); if (response.getStatusInfo().getStatusCode() == - ClientResponse.Status.OK.getStatusCode()) { + Response.Status.OK.getStatusCode()) { try { JSONArray array = new JSONArray(); - JSONObject json = response.getEntity(JSONObject.class); + JSONObject json = response.readEntity(JSONObject.class); if (!json.has("containerLogsInfo")) { return logFileInfos; } @@ -582,7 +573,7 @@ private List> getContainerLogFiles( } } - } catch (ClientHandlerException | UniformInterfaceException ex) { + } catch (ProcessingException | IllegalStateException ex) { System.err.println("Unable to fetch log files list"); throw new IOException(ex); } @@ -617,11 +608,12 @@ public int printContainerLogsFromRunningApplication(Configuration conf, for (String logFile : request.getLogTypes()) { InputStream is = null; try { - ClientResponse response = getResponseFromNMWebService(conf, + Response response = getResponseFromNMWebService(conf, webServiceClient, request, logFile); if (response != null && response.getStatusInfo().getStatusCode() == - ClientResponse.Status.OK.getStatusCode()) { - is = response.getEntityInputStream(); + Response.Status.OK.getStatusCode()) { + + is = response.readEntity(InputStream.class); int len = 0; while((len = is.read(buffer)) != -1) { out.write(buffer, 0, len); @@ -633,12 +625,12 @@ public int printContainerLogsFromRunningApplication(Configuration conf, " WebService is " + ((response == null) ? "null": "not successful," + " HTTP error code: " + response.getStatus() + ", Server response:\n" + - response.getEntity(String.class)); + response.readEntity(String.class)); out.println(msg); } out.flush(); foundAnyLogs = true; - } catch (ClientHandlerException | UniformInterfaceException ex) { + } catch (ProcessingException | IllegalStateException ex) { System.err.println("Can not find the log file:" + logFile + " for the container:" + containerIdStr + " in NodeManager:" + nodeId); @@ -811,11 +803,11 @@ private boolean createContainerLogsRequestForMasterContainer( private List getAMContainerInfoFromTimelineReader( Configuration conf, String appId) - throws IOException, ClientHandlerException, UniformInterfaceException, + throws IOException, ProcessingException, IllegalStateException, JSONException { - ClientResponse response = getClientResponseFromTimelineReader(conf, appId); + final Response response = getClientResponseFromTimelineReader(conf, appId); - JSONArray appAttemptEntities = response.getEntity(JSONArray.class); + JSONArray appAttemptEntities = response.readEntity(JSONArray.class); List amContainersList = new ArrayList(); for (int i = 0; i < appAttemptEntities.length(); i++) { JSONObject appAttemptEntity = appAttemptEntities.getJSONObject(i); @@ -826,28 +818,28 @@ private List getAMContainerInfoFromTimelineReader( return amContainersList; } - protected ClientResponse getClientResponseFromTimelineReader( + protected Response getClientResponseFromTimelineReader( Configuration conf, String appId) throws IOException { String webAppAddress = WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils .getTimelineReaderWebAppURLWithoutScheme(conf); - WebResource webResource = webServiceClient.resource(webAppAddress); + final WebTarget target = webServiceClient.target(webAppAddress); - ClientResponse response = - webResource.path("ws").path("v2").path("timeline").path("clusters") + final Response response = + target.path("ws").path("v2").path("timeline").path("clusters") .path(conf.get(YarnConfiguration.RM_CLUSTER_ID)).path("apps") .path(appId).path("entities") .path(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString()) - .queryParam("fields", "INFO").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + .queryParam("fields", "INFO").request(MediaType.APPLICATION_JSON) + .get(Response.class); if (response == null - || response.getStatusInfo().getStatusCode() != ClientResponse.Status.OK + || response.getStatusInfo().getStatusCode() != Response.Status.OK .getStatusCode()) { String msg = "Response from the timeline reader server is " + ((response == null) ? "null" : "not successful," + " HTTP error code: " + response.getStatus() - + ", Server response:\n" + response.getEntity(String.class)); + + ", Server response:\n" + response.readEntity(String.class)); System.out.println(msg); throw new IOException(msg); } @@ -884,7 +876,7 @@ private void outputAMContainerLogs(ContainerLogsRequest request, private int showContainerLogInfo(ContainerLogsRequest request, LogCLIHelpers logCliHelper) throws IOException, YarnException, - ClientHandlerException, UniformInterfaceException, JSONException { + ProcessingException, IllegalStateException, JSONException { if (!request.isAppFinished()) { return printContainerInfoFromRunningApplication(request, logCliHelper); } else { @@ -1105,7 +1097,7 @@ private int fetchAMContainerLogs(ContainerLogsRequest request, private int fetchContainerLogs(ContainerLogsRequest request, LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit) - throws IOException, ClientHandlerException, UniformInterfaceException, + throws IOException, ProcessingException, IllegalStateException, JSONException { String appIdStr = request.getAppId().toString(); String containerIdStr = request.getContainerId(); @@ -1400,8 +1392,8 @@ private List filterContainersInfo( private int printContainerInfoFromRunningApplication( ContainerLogsRequest options, LogCLIHelpers logCliHelper) - throws YarnException, IOException, ClientHandlerException, - UniformInterfaceException, JSONException { + throws YarnException, IOException, ProcessingException, + IllegalStateException, JSONException { String containerIdStr = options.getContainerId(); String nodeIdStr = options.getNodeId(); List reports = @@ -1485,7 +1477,7 @@ public Set getMatchedContainerLogFiles(ContainerLogsRequest request, } @VisibleForTesting - public ClientResponse getResponseFromNMWebService(Configuration conf, + public Response getResponseFromNMWebService(Configuration conf, Client webServiceClient, ContainerLogsRequest request, String logFile) { return LogToolUtils.getResponseFromNMWebService( conf, webServiceClient, request, logFile); @@ -1493,7 +1485,7 @@ public ClientResponse getResponseFromNMWebService(Configuration conf, @VisibleForTesting public String getNodeHttpAddressFromRMWebString(ContainerLogsRequest request) - throws ClientHandlerException, UniformInterfaceException, JSONException { + throws ProcessingException, IllegalStateException, JSONException { if (request.getNodeId() == null || request.getNodeId().isEmpty()) { return null; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java index 87ae27b304074..edd5582d55a36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java @@ -19,12 +19,6 @@ package org.apache.hadoop.yarn.client.cli; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.api.client.WebResource.Builder; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.MissingArgumentException; @@ -52,6 +46,7 @@ import javax.xml.transform.TransformerFactory; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; +import javax.ws.rs.client.Client; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response.Status; import java.io.StringReader; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index 5ab84a97677e7..b3db78d4512f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -77,17 +77,29 @@ jetty-util - com.sun.jersey - jersey-core + org.glassfish.jersey.core + jersey-common - com.sun.jersey + org.glassfish.jersey.core jersey-client org.apache.hadoop.thirdparty hadoop-shaded-guava + + org.glassfish.jersey.media + jersey-media-json-jettison + + + org.glassfish.hk2 + guice-bridge + + + net.jodah + failsafe + commons-cli commons-cli @@ -138,11 +150,6 @@ bcprov-jdk15on test - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - test - commons-io commons-io @@ -151,18 +158,6 @@ com.google.inject guice - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-json - - - com.sun.jersey.contribs - jersey-guice - log4j log4j diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/DirectTimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/DirectTimelineWriter.java index 7fea1dbcdbea5..57587c12668c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/DirectTimelineWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/DirectTimelineWriter.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.net.URI; +import net.jodah.failsafe.RetryPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -33,7 +34,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.exceptions.YarnException; -import com.sun.jersey.api.client.Client; +import javax.ws.rs.client.Client; /** * A simple writer class for storing Timeline data into Leveldb store. @@ -46,8 +47,8 @@ public class DirectTimelineWriter extends TimelineWriter{ .getLogger(DirectTimelineWriter.class); public DirectTimelineWriter(UserGroupInformation authUgi, - Client client, URI resURI) { - super(authUgi, client, resURI); + Client client, URI resURI, RetryPolicy retryPolicy) { + super(authUgi, client, resURI, retryPolicy); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java index b92f4e412347c..a94d57e4466d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java @@ -38,6 +38,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import net.jodah.failsafe.RetryPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -67,7 +68,8 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.type.TypeFactory; import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; -import com.sun.jersey.api.client.Client; + +import javax.ws.rs.client.Client; /** * A simple writer class for storing Timeline data in any storage that @@ -102,9 +104,10 @@ public class FileSystemTimelineWriter extends TimelineWriter{ private final AttemptDirCache attemptDirCache; public FileSystemTimelineWriter(Configuration conf, - UserGroupInformation authUgi, Client client, URI resURI) + UserGroupInformation authUgi, Client client, URI resURI, + RetryPolicy retryPolicy) throws IOException { - super(authUgi, client, resURI); + super(authUgi, client, resURI, retryPolicy); Configuration fsConf = new Configuration(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 07b41c3f6bf60..73fdbb6ffb89b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -24,6 +24,7 @@ import java.net.URI; import java.security.PrivilegedExceptionAction; +import net.jodah.failsafe.RetryPolicy; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; @@ -53,7 +54,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.api.client.Client; +import javax.ws.rs.client.Client; @Private @Evolving @@ -144,17 +145,18 @@ protected TimelineConnector createTimelineConnector() { protected void serviceStart() throws Exception { timelineWriter = createTimelineWriter(getConfig(), authUgi, connector.getClient(), TimelineConnector.constructResURI(getConfig(), - timelineServiceAddress, RESOURCE_URI_STR_V1)); + timelineServiceAddress, RESOURCE_URI_STR_V1), connector.getRetryPolicy()); } protected TimelineWriter createTimelineWriter(Configuration conf, - UserGroupInformation ugi, Client webClient, URI uri) + UserGroupInformation ugi, Client webClient, URI uri, + RetryPolicy retryPolicy) throws IOException { if (timelineServiceV15Enabled) { return new FileSystemTimelineWriter( - conf, ugi, webClient, uri); + conf, ugi, webClient, uri, retryPolicy); } else { - return new DirectTimelineWriter(ugi, webClient, uri); + return new DirectTimelineWriter(ugi, webClient, uri, retryPolicy); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java index c4ab169633421..6318458f1f896 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.client.api.impl; import java.io.IOException; -import java.io.InterruptedIOException; import java.lang.reflect.UndeclaredThrowableException; import java.net.ConnectException; import java.net.HttpURLConnection; @@ -30,10 +29,19 @@ import java.net.URLConnection; import java.security.GeneralSecurityException; import java.security.PrivilegedExceptionAction; +import java.time.Duration; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLSocketFactory; +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; + +import org.glassfish.jersey.client.ClientConfig; +import net.jodah.failsafe.Failsafe; +import net.jodah.failsafe.RetryPolicy; +import org.glassfish.jersey.client.HttpUrlConnectorProvider; import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; @@ -58,15 +66,6 @@ import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientRequest; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.api.client.filter.ClientFilter; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; /** * Utility Connector class which is used by timeline clients to securely get @@ -87,9 +86,8 @@ public class TimelineConnector extends AbstractService { private DelegationTokenAuthenticatedURL.Token token; private UserGroupInformation authUgi; private String doAsUser; - @VisibleForTesting - TimelineClientConnectionRetry connectionRetry; private boolean requireConnectionRetry; + private RetryPolicy retryPolicy; public TimelineConnector(boolean requireConnectionRetry, UserGroupInformation authUgi, String doAsUser, @@ -104,8 +102,8 @@ public TimelineConnector(boolean requireConnectionRetry, @Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); - ClientConfig cc = new DefaultClientConfig(); - cc.getClasses().add(YarnJacksonJaxbJsonProvider.class); + ClientConfig cc = new ClientConfig(); + cc.register(YarnJacksonJaxbJsonProvider.class); if (YarnConfiguration.useHttps(conf)) { // If https is chosen, configures SSL client. @@ -126,17 +124,13 @@ protected void serviceInit(Configuration conf) throws Exception { } authenticator.setConnectionConfigurator(connConfigurator); - connectionRetry = new TimelineClientConnectionRetry(conf); - client = - new Client( - new URLConnectionClientHandler(new TimelineURLConnectionFactory( - authUgi, authenticator, connConfigurator, token, doAsUser)), - cc); - if (requireConnectionRetry) { - TimelineJerseyRetryFilter retryFilter = - new TimelineJerseyRetryFilter(connectionRetry); - client.addFilter(retryFilter); - } + retryPolicy = createRetryPolicy(conf); + + cc.connectorProvider(new HttpUrlConnectorProvider().connectionFactory( + new TimelineURLConnectionFactory( + authUgi, authenticator, connConfigurator, token, doAsUser))); + + client = ClientBuilder.newClient(cc); } private static final ConnectionConfigurator DEFAULT_TIMEOUT_CONN_CONFIGURATOR @@ -208,7 +202,7 @@ DelegationTokenAuthenticatedURL getDelegationTokenAuthenticatedURL() { protected void serviceStop() { if (this.client != null) { - this.client.destroy(); + this.client.close(); } if (this.sslFactory != null) { this.sslFactory.destroy(); @@ -225,8 +219,7 @@ public Object operateDelegationToken( // Set up the retry operation TimelineClientRetryOp tokenRetryOp = createRetryOpForOperateDelegationToken(action); - - return connectionRetry.retryOn(tokenRetryOp); + return Failsafe.with(retryPolicy).get(tokenRetryOp::run); } @Private @@ -245,20 +238,17 @@ TimelineClientRetryOp createRetryOpForOperateDelegationToken( public static abstract class TimelineClientRetryOp { // The operation that should be retried public abstract Object run() throws IOException; - - // The method to indicate if we should retry given the incoming exception - public abstract boolean shouldRetryOn(Exception e); } private static class TimelineURLConnectionFactory - implements HttpURLConnectionFactory { + implements HttpUrlConnectorProvider.ConnectionFactory { private DelegationTokenAuthenticator authenticator; private UserGroupInformation authUgi; private ConnectionConfigurator connConfigurator; private Token token; private String doAsUser; - public TimelineURLConnectionFactory(UserGroupInformation authUgi, + TimelineURLConnectionFactory(UserGroupInformation authUgi, DelegationTokenAuthenticator authenticator, ConnectionConfigurator connConfigurator, DelegationTokenAuthenticatedURL.Token token, String doAsUser) { @@ -270,8 +260,7 @@ public TimelineURLConnectionFactory(UserGroupInformation authUgi, } @Override - public HttpURLConnection getHttpURLConnection(final URL url) - throws IOException { + public HttpURLConnection getConnection(URL url) throws IOException { authUgi.checkTGTAndReloginFromKeytab(); try { return new DelegationTokenAuthenticatedURL(authenticator, @@ -282,143 +271,43 @@ public HttpURLConnection getHttpURLConnection(final URL url) throw new IOException(ae); } } - } - // Class to handle retry - // Outside this class, only visible to tests - @Private - @VisibleForTesting - static class TimelineClientConnectionRetry { - - // maxRetries < 0 means keep trying - @Private - @VisibleForTesting - public int maxRetries; - - @Private - @VisibleForTesting - public long retryInterval; - - // Indicates if retries happened last time. Only tests should read it. - // In unit tests, retryOn() calls should _not_ be concurrent. - private boolean retried = false; - - @Private - @VisibleForTesting - boolean getRetired() { - return retried; - } - - // Constructor with default retry settings - public TimelineClientConnectionRetry(Configuration conf) { - Preconditions.checkArgument( - conf.getInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES) - >= -1, - "%s property value should be greater than or equal to -1", - YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES); - Preconditions.checkArgument( - conf.getLong( - YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, - YarnConfiguration. + private RetryPolicy createRetryPolicy(Configuration conf) { + Preconditions.checkArgument( + conf.getInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES) + >= -1, + "%s property value should be greater than or equal to -1", + YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES); + Preconditions.checkArgument( + conf.getLong( + YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, + YarnConfiguration. DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS) > 0, "%s property value should be greater than zero", YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS); - maxRetries = - conf.getInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES); - retryInterval = conf.getLong( - YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS); - } - - public Object retryOn(TimelineClientRetryOp op) - throws RuntimeException, IOException { - int leftRetries = maxRetries; - retried = false; - - // keep trying - while (true) { - try { - // try perform the op, if fail, keep retrying - return op.run(); - } catch (IOException | RuntimeException e) { - // break if there's no retries left - if (leftRetries == 0) { - break; - } - if (op.shouldRetryOn(e)) { - logException(e, leftRetries); - } else { - throw e; - } - } - if (leftRetries > 0) { - leftRetries--; - } - retried = true; - try { - // sleep for the given time interval - Thread.sleep(retryInterval); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new InterruptedIOException("Client retry sleep interrupted!"); - } - } - throw new RuntimeException("Failed to connect to timeline server. " - + "Connection retries limit exceeded. " - + "The posted timeline event may be missing"); - }; - - private void logException(Exception e, int leftRetries) { - if (leftRetries > 0) { - LOG.info( - "Exception caught by TimelineClientConnectionRetry," + " will try " - + leftRetries + " more time(s).\nMessage: " + e.getMessage()); - } else { - // note that maxRetries may be -1 at the very beginning - LOG.info("ConnectionException caught by TimelineClientConnectionRetry," - + " will keep retrying.\nMessage: " + e.getMessage()); - } + int maxRetries = 0; + if (requireConnectionRetry) { + // maxRetries < 0 means keep trying + maxRetries = conf.getInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES); } + long retryInterval = conf.getLong( + YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS); + return retryPolicy = new RetryPolicy<>() + .handle(IOException.class, RuntimeException.class) + .handleIf(e -> e instanceof ProcessingException + && (e.getCause() instanceof ConnectException + || e.getCause() instanceof SocketTimeoutException + || e.getCause() instanceof SocketException)) + .withDelay(Duration.ofMillis(retryInterval)) + .withMaxRetries(maxRetries); } - private static class TimelineJerseyRetryFilter extends ClientFilter { - private TimelineClientConnectionRetry connectionRetry; - - public TimelineJerseyRetryFilter( - TimelineClientConnectionRetry connectionRetry) { - this.connectionRetry = connectionRetry; - } - - @Override - public ClientResponse handle(final ClientRequest cr) - throws ClientHandlerException { - // Set up the retry operation - TimelineClientRetryOp jerseyRetryOp = new TimelineClientRetryOp() { - @Override - public Object run() { - // Try pass the request, if fail, keep retrying - return getNext().handle(cr); - } - - @Override - public boolean shouldRetryOn(Exception e) { - // Only retry on connection exceptions - return (e instanceof ClientHandlerException) - && (e.getCause() instanceof ConnectException - || e.getCause() instanceof SocketTimeoutException - || e.getCause() instanceof SocketException); - } - }; - try { - return (ClientResponse) connectionRetry.retryOn(jerseyRetryOp); - } catch (IOException e) { - throw new ClientHandlerException( - "Jersey retry failed!\nMessage: " + e.getMessage()); - } - } + RetryPolicy getRetryPolicy() { + return retryPolicy; } @Private @@ -447,13 +336,5 @@ public Object run() throws IOException { throw new IOException(e); } } - - @Override - public boolean shouldRetryOn(Exception e) { - // retry on connection exceptions - // and SocketTimeoutException - return (e instanceof ConnectException - || e instanceof SocketTimeoutException); - } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java index d5af8a0df5692..b5f98f73cdd03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java @@ -17,8 +17,9 @@ */ package org.apache.hadoop.yarn.client.api.impl; +import net.jodah.failsafe.Failsafe; + import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.core.util.MultivaluedMapImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -31,12 +32,15 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.client.api.TimelineReaderClient; -import com.sun.jersey.api.client.ClientResponse; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import javax.ws.rs.client.Invocation; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedHashMap; import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; import java.io.IOException; import java.net.URI; import java.util.Arrays; @@ -108,12 +112,12 @@ public TimelineEntity getApplicationEntity(ApplicationId appId, String fields, if (fields == null || fields.isEmpty()) { fields = "INFO"; } - MultivaluedMap params = new MultivaluedMapImpl(); + MultivaluedMap params = new MultivaluedHashMap<>(); params.add("fields", fields); mergeFilters(params, filters); - ClientResponse response = doGetUri(baseUri, path, params); - TimelineEntity entity = response.getEntity(TimelineEntity.class); + Response response = doGetUri(baseUri, path, params); + TimelineEntity entity = response.readEntity(TimelineEntity.class); return entity; } @@ -128,12 +132,12 @@ public TimelineEntity getApplicationAttemptEntity( if (fields == null || fields.isEmpty()) { fields = "INFO"; } - MultivaluedMap params = new MultivaluedMapImpl(); + MultivaluedMap params = new MultivaluedHashMap<>(); params.add("fields", fields); mergeFilters(params, filters); - ClientResponse response = doGetUri(baseUri, path, params); - TimelineEntity entity = response.getEntity(TimelineEntity.class); + Response response = doGetUri(baseUri, path, params); + TimelineEntity entity = response.readEntity(TimelineEntity.class); return entity; } @@ -147,7 +151,7 @@ public List getApplicationAttemptEntities( if (fields == null || fields.isEmpty()) { fields = "INFO"; } - MultivaluedMap params = new MultivaluedMapImpl(); + MultivaluedMap params = new MultivaluedHashMap<>(); params.add("fields", fields); if (limit > 0) { params.add("limit", Long.toString(limit)); @@ -157,8 +161,8 @@ public List getApplicationAttemptEntities( } mergeFilters(params, filters); - ClientResponse response = doGetUri(baseUri, path, params); - TimelineEntity[] entities = response.getEntity(TimelineEntity[].class); + Response response = doGetUri(baseUri, path, params); + TimelineEntity[] entities = response.readEntity(TimelineEntity[].class); return Arrays.asList(entities); } @@ -173,12 +177,12 @@ public TimelineEntity getContainerEntity(ContainerId containerId, if (fields == null || fields.isEmpty()) { fields = "INFO"; } - MultivaluedMap params = new MultivaluedMapImpl(); + MultivaluedMap params = new MultivaluedHashMap<>(); params.add("fields", fields); mergeFilters(params, filters); - ClientResponse response = doGetUri(baseUri, path, params); - TimelineEntity entity = response.getEntity(TimelineEntity.class); + Response response = doGetUri(baseUri, path, params); + TimelineEntity entity = response.readEntity(TimelineEntity.class); return entity; } @@ -193,7 +197,7 @@ public List getContainerEntities( if (fields == null || fields.isEmpty()) { fields = "INFO"; } - MultivaluedMap params = new MultivaluedMapImpl(); + MultivaluedMap params = new MultivaluedHashMap<>(); params.add("fields", fields); if (limit > 0) { params.add("limit", Long.toString(limit)); @@ -203,8 +207,8 @@ public List getContainerEntities( } mergeFilters(params, filters); - ClientResponse response = doGetUri(baseUri, path, params); - TimelineEntity[] entity = response.getEntity(TimelineEntity[].class); + Response response = doGetUri(baseUri, path, params); + TimelineEntity[] entity = response.readEntity(TimelineEntity[].class); return Arrays.asList(entity); } @@ -220,19 +224,23 @@ private void mergeFilters(MultivaluedMap defaults, } @VisibleForTesting - protected ClientResponse doGetUri(URI base, String path, + protected Response doGetUri(URI base, String path, MultivaluedMap params) throws IOException { - ClientResponse resp = connector.getClient().resource(base).path(path) - .queryParams(params).accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + WebTarget target = connector.getClient().target(base).path(path); + for(Map.Entry> param : params.entrySet()) { + target = target.queryParam(param.getKey(), param.getValue()); + } + Invocation.Builder builder = target.request(MediaType.APPLICATION_JSON); + Response resp = Failsafe.with(connector.getRetryPolicy()) + .get(() -> builder.get(Response.class)); if (resp == null || - resp.getStatusInfo().getStatusCode() != ClientResponse.Status.OK + resp.getStatusInfo().getStatusCode() != Response.Status.OK .getStatusCode()) { String msg = "Response from the timeline reader server is " + ((resp == null) ? "null" : "not successful," + " HTTP error code: " + resp.getStatus() + - ", Server response:\n" + resp.getEntity(String.class)); + ", Server response:\n" + resp.readEntity(String.class)); LOG.error(msg); throw new IOException(msg); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java index c5aef4e1ef4fd..3acf19a9dc16c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java @@ -24,8 +24,9 @@ import java.net.InetSocketAddress; import java.net.URI; import java.security.PrivilegedExceptionAction; +import java.util.List; +import java.util.Map; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -34,8 +35,13 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.CancellationException; +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedHashMap; import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,10 +63,6 @@ import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.core.util.MultivaluedMapImpl; /** * Implementation of timeline v2 client interface. @@ -288,24 +290,23 @@ private void checkRetryWithSleep(int retries, IOException e) } } - private ClientResponse doPutObjects(URI base, String path, - MultivaluedMap params, Object obj) { - return connector.getClient().resource(base).path(path).queryParams(params) - .accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON) - .put(ClientResponse.class, obj); + private Response doPutObjects(URI base, String path, + MultivaluedMap params, Object obj) { + WebTarget target = connector.getClient().target(base).path(path); + for(Map.Entry> param : params.entrySet()) { + target = target.queryParam(param.getKey(), param.getValue()); + } + return target.request(MediaType.APPLICATION_JSON) + .post(Entity.json(obj), Response.class); } protected void putObjects(URI base, String path, MultivaluedMap params, Object obj) throws IOException, YarnException { - ClientResponse resp = null; + Response resp = null; try { - resp = authUgi.doAs(new PrivilegedExceptionAction() { - @Override - public ClientResponse run() throws Exception { - return doPutObjects(base, path, params, obj); - } - }); + resp = authUgi.doAs((PrivilegedExceptionAction) () + -> doPutObjects(base, path, params, obj)); } catch (UndeclaredThrowableException ue) { Throwable cause = ue.getCause(); if (cause instanceof IOException) { @@ -325,23 +326,20 @@ public ClientResponse run() throws Exception { LOG.error(msg); throw new YarnException(msg); } else if (resp.getStatusInfo().getStatusCode() - == ClientResponse.Status.OK.getStatusCode()) { + == Response.Status.OK.getStatusCode()) { try { resp.close(); - } catch(ClientHandlerException che) { - LOG.warn("Error closing the HTTP response's inputstream. ", che); + } catch(ProcessingException e) { + LOG.warn("Error closing the HTTP response's inputstream. ", e); } } else { String msg = ""; try { - String stringType = resp.getEntity(String.class); + String stringType = resp.readEntity(String.class); msg = "Server response:\n" + stringType; - } catch (ClientHandlerException | UniformInterfaceException chuie) { - msg = "Error getting entity from the HTTP response." - + chuie.getLocalizedMessage(); - } catch (Throwable t) { + } catch (ProcessingException | IllegalStateException e) { msg = "Error getting entity from the HTTP response." - + t.getLocalizedMessage(); + + e.getLocalizedMessage(); } finally { msg = "Response from the timeline server is not successful" + ", HTTP error code: " + resp.getStatus() @@ -394,16 +392,14 @@ private final class EntitiesHolder extends FutureTask { EntitiesHolder(final TimelineEntities entities, final boolean isSync, final boolean subappwrite) { - super(new Callable() { - // publishEntities() - public Void call() throws Exception { - MultivaluedMap params = new MultivaluedMapImpl(); - params.add("appid", getContextAppId().toString()); - params.add("async", Boolean.toString(!isSync)); - params.add("subappwrite", Boolean.toString(subappwrite)); - putObjects("entities", params, entities); - return null; - } + // publishEntities() + super(() -> { + MultivaluedMap params = new MultivaluedHashMap<>(); + params.add("appid", getContextAppId().toString()); + params.add("async", Boolean.toString(!isSync)); + params.add("subappwrite", Boolean.toString(subappwrite)); + putObjects("entities", params, entities); + return null; }); this.entities = entities; this.isSync = isSync; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineWriter.java index 26c0877207eed..1154dbfc4a862 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineWriter.java @@ -24,8 +24,14 @@ import java.lang.reflect.UndeclaredThrowableException; import java.net.URI; import java.security.PrivilegedExceptionAction; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import net.jodah.failsafe.Failsafe; +import net.jodah.failsafe.RetryPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -41,9 +47,6 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; /** * Base writer class to write the Timeline data. @@ -58,12 +61,14 @@ public abstract class TimelineWriter implements Flushable { private UserGroupInformation authUgi; private Client client; private URI resURI; + private final RetryPolicy retryPolicy; public TimelineWriter(UserGroupInformation authUgi, Client client, - URI resURI) { + URI resURI, RetryPolicy retryPolicy) { this.authUgi = authUgi; this.client = client; this.resURI = resURI; + this.retryPolicy = retryPolicy; } public void close() throws Exception { @@ -89,8 +94,8 @@ public TimelinePutResponse putEntities( } entitiesContainer.addEntity(entity); } - ClientResponse resp = doPosting(entitiesContainer, null); - return resp.getEntity(TimelinePutResponse.class); + Response resp = doPosting(entitiesContainer, null); + return resp.readEntity(TimelinePutResponse.class); } public void putDomain(TimelineDomain domain) throws IOException, @@ -105,16 +110,12 @@ public abstract TimelinePutResponse putEntities( public abstract void putDomain(ApplicationAttemptId appAttemptId, TimelineDomain domain) throws IOException, YarnException; - private ClientResponse doPosting(final Object obj, final String path) + private Response doPosting(final Object obj, final String path) throws IOException, YarnException { - ClientResponse resp; + Response resp; try { - resp = authUgi.doAs(new PrivilegedExceptionAction() { - @Override - public ClientResponse run() throws Exception { - return doPostingObject(obj, path); - } - }); + resp = authUgi.doAs((PrivilegedExceptionAction) + () -> doPostingObject(obj, path)); } catch (UndeclaredThrowableException e) { Throwable cause = e.getCause(); if (cause instanceof IOException) { @@ -127,14 +128,14 @@ public ClientResponse run() throws Exception { } if (resp == null || resp.getStatusInfo().getStatusCode() - != ClientResponse.Status.OK.getStatusCode()) { + != Response.Status.OK.getStatusCode()) { String msg = "Failed to get the response from the timeline server."; LOG.error(msg); if (resp != null) { msg += " HTTP error code: " + resp.getStatus(); LOG.debug("HTTP error code: {} Server response : \n{}", - resp.getStatus(), resp.getEntity(String.class)); + resp.getStatus(), resp.readEntity(String.class)); } throw new YarnException(msg); } @@ -143,20 +144,20 @@ public ClientResponse run() throws Exception { @Private @VisibleForTesting - public ClientResponse doPostingObject(Object object, String path) { - WebResource webResource = client.resource(resURI); + public Response doPostingObject(Object object, String path) { + final WebTarget webTarget = client.target(resURI); if (path == null) { LOG.debug("POST to {}", resURI); - ClientResponse r = webResource.accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON) - .post(ClientResponse.class, object); + Response r = Failsafe.with(retryPolicy).get( + () -> webTarget.request(MediaType.APPLICATION_JSON) + .post(Entity.json(object), Response.class)); r.bufferEntity(); return r; } else if (path.equals("domain")) { LOG.debug("PUT to {}/{}", resURI, path); - ClientResponse r = webResource.path(path).accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON) - .put(ClientResponse.class, object); + Response r = Failsafe.with(retryPolicy).get( + () -> webTarget.path(path).request(MediaType.APPLICATION_JSON) + .post(Entity.json(object), Response.class)); r.bufferEntity(); return r; } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java index 3c56b0290d74e..229daad597c6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java @@ -29,15 +29,15 @@ import java.nio.file.Files; import java.nio.file.Paths; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; /** * This class contains several utility function which could be used in different @@ -201,15 +201,15 @@ public static PrintStream createPrintStream(String localDir, String nodeId, * @param logFile name of the log file * @return response from NMWebServices */ - public static ClientResponse getResponseFromNMWebService(Configuration conf, + public static Response getResponseFromNMWebService(Configuration conf, Client webServiceClient, ContainerLogsRequest request, String logFile) { - WebResource webResource = - webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf) + WebTarget target = + webServiceClient.target(WebAppUtils.getHttpSchemePrefix(conf) + request.getNodeHttpAddress()); - return webResource.path("ws").path("v1").path("node") + return target.path("ws").path("v1").path("node") .path("containers").path(request.getContainerId()).path("logs") .path(logFile) .queryParam("size", Long.toString(request.getBytes())) - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + .request(MediaType.TEXT_PLAIN).get(Response.class); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java index 2bad02c9f088e..e0763507f2d94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java @@ -58,8 +58,8 @@ public Response toResponse(Exception e) { // Don't catch this as filter forward on 404 // (ServletContainer.FEATURE_FILTER_FORWARD_ON_404) // won't work and the web UI won't work! - if (e instanceof com.sun.jersey.api.NotFoundException) { - return ((com.sun.jersey.api.NotFoundException) e).getResponse(); + if (e instanceof javax.ws.rs.NotFoundException) { + return ((javax.ws.rs.NotFoundException) e).getResponse(); } // clear content type response.setContentType(null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java index f6511b3c9c852..ab9bef59f5a8a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java @@ -22,9 +22,7 @@ import java.net.InetSocketAddress; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; @@ -37,13 +35,11 @@ import org.apache.hadoop.thirdparty.com.google.common.base.Splitter; import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import com.google.inject.Provides; +import com.google.inject.Scopes; import com.google.inject.servlet.GuiceFilter; import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.container.filter.GZIPContentEncodingFilter; -import com.sun.jersey.api.core.ResourceConfig; -import com.sun.jersey.core.util.FeaturesAndProperties; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.spi.container.servlet.ServletContainer; + +import javax.servlet.Filter; /** * @see WebApps for a usage example @@ -181,19 +177,12 @@ protected void configureWebAppServlets() { // from /* below - that doesn't work. String regex = "(?!/" + this.wsName + ")"; serveRegex(regex).with(DefaultWrapperServlet.class); - - Map params = new HashMap(); - params.put(ResourceConfig.FEATURE_IMPLICIT_VIEWABLES, "true"); - params.put(ServletContainer.FEATURE_FILTER_FORWARD_ON_404, "true"); - params.put(FeaturesAndProperties.FEATURE_XMLROOTELEMENT_PROCESSING, "true"); - params.put(ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS, GZIPContentEncodingFilter.class.getName()); - params.put(ResourceConfig.PROPERTY_CONTAINER_RESPONSE_FILTERS, GZIPContentEncodingFilter.class.getName()); - filter("/*").through(getWebAppFilterClass(), params); + bind(getWebAppFilterClass()).in(Scopes.SINGLETON); } } - protected Class getWebAppFilterClass() { - return GuiceContainer.class; + protected Class getWebAppFilterClass() { + return GuiceFilter.class; } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java index 2f02fd7a0762f..672d23d9de7d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.eclipse.jetty.webapp.WebAppContext; +import org.glassfish.jersey.servlet.ServletProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -404,8 +405,11 @@ public void setup() { new String[] {"/*"}); } + final Map guiceFilterParams = new HashMap<>(); + guiceFilterParams.put(ServletProperties.FILTER_FORWARD_ON_404, "true"); HttpServer2.defineFilter(server.getWebAppContext(), "guice", - GuiceFilter.class.getName(), null, new String[] { "/*" }); + GuiceFilter.class.getName(), guiceFilterParams, + new String[] { "/*" }); webapp.setConf(conf); webapp.setHttpServer(server); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 09daf424d154f..7d9d48db29a02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -24,6 +24,7 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -48,6 +49,7 @@ import org.apache.http.client.utils.URLEncodedUtils; import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.container.ContainerRequestContext; @Private @Evolving @@ -562,6 +564,20 @@ private static String getURLEncodedQueryString(HttpServletRequest request, return null; } + private static String getURLEncodedQueryString(ContainerRequestContext request, + String parameterToRemove) { + String queryString = request.getUriInfo().getPath(); + if (queryString != null && !queryString.isEmpty()) { + List params = URLEncodedUtils.parse(queryString, + StandardCharsets.ISO_8859_1); + if (parameterToRemove != null && !parameterToRemove.isEmpty()) { + params.removeIf(current -> current.getName().equals(parameterToRemove)); + } + return URLEncodedUtils.format(params, StandardCharsets.ISO_8859_1); + } + return null; + } + /** * Get a query string. * @param request HttpServletRequest with the request details @@ -583,6 +599,20 @@ public static List getURLEncodedQueryParam( return null; } + /** + * Get a query string. + * @param request ContainerRequestContext with the request details + * @return the query parameter string + */ + public static List getURLEncodedQueryParam( + ContainerRequestContext request) { + String queryString = request.getUriInfo().getPath(); + if (queryString != null && !queryString.isEmpty()) { + return URLEncodedUtils.parse(queryString, StandardCharsets.ISO_8859_1); + } + return null; + } + /** * Get a query string which removes the passed parameter. * @param httpRequest HttpServletRequest with the request details @@ -624,4 +654,20 @@ public static String appendQueryParams(HttpServletRequest request, } return ret; } + + /** + * Add the query params from a ContainerRequestContext to the target uri passed. + * @param request ContainerRequestContext with the request details + * @param targetUri the uri to which the query params must be added + * @return URL encoded string containing the targetUri + "?" + query string + */ + public static String appendQueryParams(ContainerRequestContext request, + String targetUri) { + String ret = targetUri; + String urlEncodedQueryString = getURLEncodedQueryString(request, null); + if (urlEncodedQueryString != null) { + ret += "?" + urlEncodedQueryString; + } + return ret; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebServiceClient.java index 1dce5af7bef95..3cdf0e1462c79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebServiceClient.java @@ -19,7 +19,12 @@ import java.io.IOException; import java.net.HttpURLConnection; -import java.net.URL; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; + +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.HttpUrlConnectorProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authentication.client.AuthenticatedURL; @@ -29,9 +34,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; /** * Utility for handling Web client. @@ -92,33 +94,31 @@ private static SSLFactory createSSLFactory(Configuration conf) * @return Client */ public Client createClient() { - return new Client( - new URLConnectionClientHandler(getHttpURLConnectionFactory())); + final ClientConfig cc = new ClientConfig(); + cc.connectorProvider(getHttpURLConnectionFactory()); + return ClientBuilder.newClient(cc); } @VisibleForTesting - protected HttpURLConnectionFactory getHttpURLConnectionFactory() { - return new HttpURLConnectionFactory() { - @Override - public HttpURLConnection getHttpURLConnection(URL url) - throws IOException { - AuthenticatedURL.Token token = new AuthenticatedURL.Token(); - HttpURLConnection conn = null; - try { - HttpURLConnection.setFollowRedirects(false); - // If https is chosen, configures SSL client. - if (isHttps) { - conn = new AuthenticatedURL(new KerberosAuthenticator(), - sslFactory).openConnection(url, token); - } else { - conn = new AuthenticatedURL().openConnection(url, token); + protected HttpUrlConnectorProvider getHttpURLConnectionFactory() { + return new HttpUrlConnectorProvider().connectionFactory( + url -> { + AuthenticatedURL.Token token = new AuthenticatedURL.Token(); + HttpURLConnection conn; + try { + HttpURLConnection.setFollowRedirects(false); + // If https is chosen, configures SSL client. + if (isHttps) { + conn = new AuthenticatedURL(new KerberosAuthenticator(), + sslFactory).openConnection(url, token); + } else { + conn = new AuthenticatedURL().openConnection(url, token); + } + } catch (AuthenticationException e) { + throw new IOException(e); } - } catch (AuthenticationException e) { - throw new IOException(e); - } - return conn; - } - }; + return conn; + }); } public synchronized static void destroy() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java index fccb3e1415f8e..634fa359616fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java @@ -17,19 +17,16 @@ */ package org.apache.hadoop.yarn.webapp.util; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource.Builder; +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; -import com.sun.jersey.api.json.JSONJAXBContext; -import com.sun.jersey.api.json.JSONMarshaller; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.hadoop.conf.Configuration; -import org.codehaus.jettison.json.JSONObject; -import java.io.StringWriter; +import org.codehaus.jettison.json.JSONObject; /** * This class contains several utility function which could be used to generate @@ -40,57 +37,41 @@ public final class YarnWebServiceUtils { private YarnWebServiceUtils() {} + private static final ObjectMapper mapper = new ObjectMapper(); + /** * Utility function to get NodeInfo by calling RM WebService. * @param conf the configuration - * @param nodeId the nodeId + * @param nodeId the node * @return a JSONObject which contains the NodeInfo - * @throws ClientHandlerException if there is an error - * processing the response. - * @throws UniformInterfaceException if the response status - * is 204 (No Content). */ public static JSONObject getNodeInfoFromRMWebService(Configuration conf, - String nodeId) throws ClientHandlerException, - UniformInterfaceException { + String nodeId) throws ProcessingException, IllegalStateException { try { return WebAppUtils.execOnActiveRM(conf, YarnWebServiceUtils::getNodeInfoFromRM, nodeId); + } catch (ProcessingException | IllegalStateException e) { + throw e; } catch (Exception e) { - if (e instanceof ClientHandlerException) { - throw ((ClientHandlerException) e); - } else if (e instanceof UniformInterfaceException) { - throw ((UniformInterfaceException) e); - } else { - throw new RuntimeException(e); - } + throw new RuntimeException(e); } } private static JSONObject getNodeInfoFromRM(String webAppAddress, - String nodeId) throws ClientHandlerException, UniformInterfaceException { - Client webServiceClient = Client.create(); - ClientResponse response = null; - try { - Builder builder = webServiceClient.resource(webAppAddress) - .path("ws").path("v1").path("cluster") - .path("nodes").path(nodeId).accept(MediaType.APPLICATION_JSON); - response = builder.get(ClientResponse.class); - return response.getEntity(JSONObject.class); + String nodeId) { + Client webServiceClient = ClientBuilder.newClient(); + try (Response response = webServiceClient.target(webAppAddress) + .path("ws").path("v1").path("cluster") + .path("nodes").path(nodeId) + .request(MediaType.APPLICATION_JSON) + .get(Response.class)) { + return response.readEntity(JSONObject.class); } finally { - if (response != null) { - response.close(); - } - webServiceClient.destroy(); + webServiceClient.close(); } } - @SuppressWarnings("rawtypes") - public static String toJson(Object nsli, Class klass) throws Exception { - StringWriter sw = new StringWriter(); - JSONJAXBContext ctx = new JSONJAXBContext(klass); - JSONMarshaller jm = ctx.createJSONMarshaller(); - jm.marshallToJSON(nsli, sw); - return sw.toString(); + public static String toJson(Object obj, Class klass) throws Exception { + return mapper.writerFor(klass).writeValueAsString(obj); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index 9f9564a310641..86623380879b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -33,6 +33,7 @@ import java.net.URI; import java.security.PrivilegedExceptionAction; +import net.jodah.failsafe.RetryPolicy; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.http.HttpConfig.Policy; @@ -42,6 +43,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.test.TestGenericTestUtils; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; @@ -50,14 +52,15 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.glassfish.jersey.client.ClientResponse; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientResponse; +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.core.Response; public class TestTimelineClient { @@ -86,7 +89,7 @@ public void tearDown() throws Exception { @Test public void testPostEntities() throws Exception { - mockEntityClientResponse(spyTimelineWriter, ClientResponse.Status.OK, + mockEntityClientResponse(spyTimelineWriter, Response.Status.OK, false, false); try { TimelinePutResponse response = client.putEntities(generateEntity()); @@ -98,7 +101,7 @@ public void testPostEntities() throws Exception { @Test public void testPostEntitiesWithError() throws Exception { - mockEntityClientResponse(spyTimelineWriter, ClientResponse.Status.OK, true, + mockEntityClientResponse(spyTimelineWriter, Response.Status.OK, true, false); try { TimelinePutResponse response = client.putEntities(generateEntity()); @@ -126,7 +129,7 @@ public void testPostIncompleteEntities() throws Exception { @Test public void testPostEntitiesNoResponse() throws Exception { mockEntityClientResponse(spyTimelineWriter, - ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false); + Response.Status.INTERNAL_SERVER_ERROR, false, false); try { client.putEntities(generateEntity()); Assert.fail("Exception is expected"); @@ -143,13 +146,13 @@ public void testPostEntitiesConnectionRefused() throws Exception { client.putEntities(generateEntity()); Assert.fail("RuntimeException is expected"); } catch (RuntimeException re) { - Assert.assertTrue(re instanceof ClientHandlerException); + Assert.assertTrue(re instanceof ProcessingException); } } @Test public void testPutDomain() throws Exception { - mockDomainClientResponse(spyTimelineWriter, ClientResponse.Status.OK, false); + mockDomainClientResponse(spyTimelineWriter, Response.Status.OK, false); try { client.putDomain(generateDomain()); } catch (YarnException e) { @@ -160,7 +163,7 @@ public void testPutDomain() throws Exception { @Test public void testPutDomainNoResponse() throws Exception { mockDomainClientResponse(spyTimelineWriter, - ClientResponse.Status.FORBIDDEN, false); + Response.Status.FORBIDDEN, false); try { client.putDomain(generateDomain()); Assert.fail("Exception is expected"); @@ -177,7 +180,7 @@ public void testPutDomainConnectionRefused() throws Exception { client.putDomain(generateDomain()); Assert.fail("RuntimeException is expected"); } catch (RuntimeException re) { - Assert.assertTrue(re instanceof ClientHandlerException); + Assert.assertTrue(re instanceof ProcessingException); } } @@ -215,18 +218,17 @@ public void testCheckRetryCount() throws Exception { newIntervalMs); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); TimelineClientImpl client = createTimelineClient(conf); + long start = Time.monotonicNow(); try { // This call should fail because there is no timeline server client.putEntities(generateEntity()); Assert.fail("Exception expected! " + "Timeline server should be off to run this test. "); } catch (RuntimeException ce) { - Assert.assertTrue( - "Handler exception for reason other than retry: " + ce.getMessage(), - ce.getMessage().contains("Connection retries limit exceeded")); - // we would expect this exception here, check if the client has retried - Assert.assertTrue("Retry filter didn't perform any retries! ", - client.connector.connectionRetry.getRetired()); + long end = Time.monotonicNow(); + ce.printStackTrace(); + Assert.assertTrue("Failed without retries.", + end - start >= newIntervalMs * newMaxRetries); } } @@ -261,7 +263,6 @@ public void testDelegationTokenOperationsRetry() throws Exception { UserGroupInformation.getCurrentUser().getShortUserName()); assertFail(); } catch (RuntimeException ce) { - assertException(client, ce); } try { @@ -276,7 +277,6 @@ public void testDelegationTokenOperationsRetry() throws Exception { new Text("0.0.0.0:8188"))); assertFail(); } catch (RuntimeException ce) { - assertException(client, ce); } try { @@ -291,7 +291,6 @@ public void testDelegationTokenOperationsRetry() throws Exception { new Text("0.0.0.0:8188"))); assertFail(); } catch (RuntimeException ce) { - assertException(client, ce); } // Test DelegationTokenOperationsRetry on SocketTimeoutException @@ -306,7 +305,6 @@ public void testDelegationTokenOperationsRetry() throws Exception { new Text("0.0.0.0:8188"))); assertFail(); } catch (RuntimeException ce) { - assertException(clientFake, ce); } } finally { client.stop(); @@ -320,21 +318,12 @@ private static void assertFail() { + "Timeline server should be off to run this test."); } - private void assertException(TimelineClientImpl client, RuntimeException ce) { - Assert.assertTrue( - "Handler exception for reason other than retry: " + ce.toString(), ce - .getMessage().contains("Connection retries limit exceeded")); - // we would expect this exception here, check if the client has retried - Assert.assertTrue("Retry filter didn't perform any retries! ", - client.connector.connectionRetry.getRetired()); - } - public static ClientResponse mockEntityClientResponse( - TimelineWriter spyTimelineWriter, ClientResponse.Status status, + TimelineWriter spyTimelineWriter, Response.Status status, boolean hasError, boolean hasRuntimeError) { ClientResponse response = mock(ClientResponse.class); if (hasRuntimeError) { - doThrow(new ClientHandlerException(new ConnectException())).when( + doThrow(new ProcessingException(new ConnectException())).when( spyTimelineWriter).doPostingObject( any(TimelineEntities.class), any()); return response; @@ -351,16 +340,16 @@ public static ClientResponse mockEntityClientResponse( if (hasError) { putResponse.addError(error); } - when(response.getEntity(TimelinePutResponse.class)).thenReturn(putResponse); + when(response.readEntity(TimelinePutResponse.class)).thenReturn(putResponse); return response; } private static ClientResponse mockDomainClientResponse( - TimelineWriter spyTimelineWriter, ClientResponse.Status status, + TimelineWriter spyTimelineWriter, Response.Status status, boolean hasRuntimeError) { ClientResponse response = mock(ClientResponse.class); if (hasRuntimeError) { - doThrow(new ClientHandlerException(new ConnectException())).when( + doThrow(new ProcessingException(new ConnectException())).when( spyTimelineWriter).doPostingObject(any(TimelineDomain.class), any(String.class)); return response; @@ -411,10 +400,10 @@ private TimelineClientImpl createTimelineClient( TimelineClientImpl client = new TimelineClientImpl() { @Override protected TimelineWriter createTimelineWriter(Configuration conf, - UserGroupInformation authUgi, Client client, URI resURI) - throws IOException { + UserGroupInformation authUgi, Client client, URI resURI, + RetryPolicy retryPolicy) { TimelineWriter timelineWriter = - new DirectTimelineWriter(authUgi, client, resURI); + new DirectTimelineWriter(authUgi, client, resURI, retryPolicy); spyTimelineWriter = spy(timelineWriter); return spyTimelineWriter; } @@ -503,7 +492,7 @@ public void testTimelineConnectorDestroy() { Client mockJerseyClient = mock(Client.class); client.connector.client = mockJerseyClient; client.stop(); - verify(mockJerseyClient, times(1)).destroy(); + verify(mockJerseyClient, times(1)).close(); } private void setupSSLConfig(YarnConfiguration conf) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientForATS1_5.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientForATS1_5.java index 26dd7f42bc5f8..6f8b3ff655d1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientForATS1_5.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientForATS1_5.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.net.URI; +import net.jodah.failsafe.RetryPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -47,8 +48,8 @@ import org.junit.Before; import org.junit.Test; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; +import javax.ws.rs.client.Client; +import javax.ws.rs.core.Response; public class TestTimelineClientForATS1_5 { @@ -251,14 +252,15 @@ private TimelineClientImpl createTimelineClient(YarnConfiguration conf) { TimelineClientImpl client = new TimelineClientImpl() { @Override protected TimelineWriter createTimelineWriter(Configuration conf, - UserGroupInformation authUgi, Client client, URI resURI) + UserGroupInformation authUgi, Client client, URI resURI, + RetryPolicy retryPolicy) throws IOException { TimelineWriter timelineWriter = - new FileSystemTimelineWriter(conf, authUgi, client, resURI) { - public ClientResponse doPostingObject(Object object, String path) { - ClientResponse response = mock(ClientResponse.class); + new FileSystemTimelineWriter(conf, authUgi, client, resURI, retryPolicy) { + public Response doPostingObject(Object object, String path) { + Response response = mock(Response.class); when(response.getStatusInfo()).thenReturn( - ClientResponse.Status.OK); + Response.Status.OK); return response; } }; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java index f668472256a7b..6c5223d9c49d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java @@ -23,7 +23,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.sun.jersey.api.client.ClientResponse; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -37,7 +36,7 @@ import org.junit.Test; import javax.ws.rs.core.MultivaluedMap; -import java.io.IOException; +import javax.ws.rs.core.Response; import java.net.URI; import java.util.ArrayList; import java.util.List; @@ -132,23 +131,23 @@ private static TimelineEntity[] createTimelineEntities(String... ids) { private class MockTimelineReaderClient extends TimelineReaderClientImpl { @Override - protected ClientResponse doGetUri(URI base, String path, - MultivaluedMap params) throws IOException { - ClientResponse mockClientResponse = mock(ClientResponse.class); + protected Response doGetUri(URI base, String path, + MultivaluedMap params) { + Response mockClientResponse = mock(Response.class); if (path.contains(YARN_CONTAINER.toString())) { - when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity.class)).thenReturn( createTimelineEntity("mockContainer1")); - when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity[].class)).thenReturn( createTimelineEntities("mockContainer1", "mockContainer2")); } else if (path.contains(YARN_APPLICATION_ATTEMPT.toString())) { - when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity.class)).thenReturn( createTimelineEntity("mockAppAttempt1")); - when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity[].class)).thenReturn( createTimelineEntities("mockAppAttempt1", "mockAppAttempt2")); } else { - when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity.class)).thenReturn( createTimelineEntity("mockApp1")); - when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity[].class)).thenReturn( createTimelineEntities("mockApp1", "mockApp2")); } return mockClientResponse; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java deleted file mode 100644 index 6f6ee5d4b53ab..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by joblicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.webapp; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -import javax.ws.rs.ext.ContextResolver; -import javax.ws.rs.ext.Provider; -import javax.xml.bind.JAXBContext; - -import org.apache.hadoop.yarn.webapp.MyTestWebService.MyInfo; - -import com.google.inject.Singleton; -import com.sun.jersey.api.json.JSONConfiguration; -import com.sun.jersey.api.json.JSONJAXBContext; - -@Singleton -@Provider -public class MyTestJAXBContextResolver implements ContextResolver { - - private JAXBContext context; - private final Set types; - - // you have to specify all the dao classes here - private final Class[] cTypes = { MyInfo.class }; - - public MyTestJAXBContextResolver() throws Exception { - this.types = new HashSet(Arrays.asList(cTypes)); - this.context = - new JSONJAXBContext(JSONConfiguration.natural().rootUnwrapping(false) - .build(), cTypes); - } - - @Override - public JAXBContext getContext(Class objectType) { - return (types.contains(objectType)) ? context : null; - } -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestObjectWriterContextResolver.java similarity index 53% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestObjectWriterContextResolver.java index d537fa748f9bc..46d93ea7e9291 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestObjectWriterContextResolver.java @@ -15,32 +15,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.yarn.webapp; -import java.io.IOException; -import java.util.Random; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; + +import javax.ws.rs.ext.ContextResolver; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; -import org.apache.hadoop.net.ServerSocketUtil; +public class MyTestObjectWriterContextResolver implements ContextResolver { -import com.sun.jersey.test.framework.JerseyTest; -import com.sun.jersey.test.framework.WebAppDescriptor; + private final ObjectWriter writer; + private final Set types; + private final Class[] cTypes = { MyTestWebService.MyInfo.class }; -public abstract class JerseyTestBase extends JerseyTest { - public JerseyTestBase(WebAppDescriptor appDescriptor) { - super(appDescriptor); + public MyTestObjectWriterContextResolver() throws Exception { + this.types = new HashSet<>(Arrays.asList(cTypes)); + this.writer = new ObjectMapper().writerFor(MyTestWebService.MyInfo.class); } @Override - protected int getPort(int port) { - Random rand = new Random(); - int jerseyPort = port + rand.nextInt(1000); - try { - jerseyPort = ServerSocketUtil.getPort(jerseyPort, 10); - } catch (IOException e) { - // Ignore exception even after 10 times free port is - // not received. - } - return super.getPort(jerseyPort); + public ObjectWriter getContext(Class type) { + return (types.contains(type)) ? writer : null; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java index 98b7505426896..c741ab6595337 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java @@ -237,7 +237,7 @@ public void testCreateWithNonZeroPort() { WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() { @Override public void setup() { - bind(MyTestJAXBContextResolver.class); + bind(MyTestObjectWriterContextResolver.class); bind(MyTestWebService.class); route("/:foo", FooController.class); @@ -268,7 +268,7 @@ public void setup() { WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() { @Override public void setup() { - bind(MyTestJAXBContextResolver.class); + bind(MyTestObjectWriterContextResolver.class); bind(MyTestWebService.class); route("/:foo", FooController.class); @@ -297,7 +297,7 @@ public void setup() { WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() { @Override public void setup() { - bind(MyTestJAXBContextResolver.class); + bind(MyTestObjectWriterContextResolver.class); bind(MyTestWebService.class); } }); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java index 99a5783628e9a..b38ed3c7084a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java @@ -32,6 +32,8 @@ import org.junit.Assert; import org.junit.Test; +import javax.ws.rs.core.Response; + public class TestWebServiceClient { private static final String BASEDIR = System.getProperty("test.build.dir", @@ -89,9 +91,8 @@ public void testCreateClient() throws Exception { URL u = new URL(baseUrl, SERVLET_PATH_ECHO + "?a=b&c=d"); WebServiceClient.initialize(sslConf); WebServiceClient client = WebServiceClient.getWebServiceClient(); - HttpURLConnection conn = client.getHttpURLConnectionFactory() - .getHttpURLConnection(u); - Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + Response resp = client.createClient().target(u.toURI()).request().get(); + Assert.assertEquals(HttpURLConnection.HTTP_OK, resp.getStatus()); WebServiceClient.destroy(); server.stop(); FileUtil.fullyDelete(new File(BASEDIR)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java index e7b8afe149058..16f78e8873bfd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java @@ -17,34 +17,28 @@ package org.apache.hadoop.yarn.server.federation.policies.dao; -import java.io.StringReader; -import java.io.StringWriter; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.ObjectWriter; import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.sun.jersey.api.json.JSONConfiguration; -import com.sun.jersey.api.json.JSONJAXBContext; -import com.sun.jersey.api.json.JSONMarshaller; -import com.sun.jersey.api.json.JSONUnmarshaller; /** - * This is a DAO class for the configuration of parameteres for federation + * This is a DAO class for the configuration of parameters for federation * policies. This generalizes several possible configurations as two lists of * {@link SubClusterIdInfo} and corresponding weights as a {@link Float}. The * interpretation of the weight is left to the logic in the policy. @@ -56,25 +50,16 @@ @XmlAccessorType(XmlAccessType.FIELD) public class WeightedPolicyInfo { - private static final Logger LOG = - LoggerFactory.getLogger(WeightedPolicyInfo.class); - private static JSONJAXBContext jsonjaxbContext = initContext(); + private static final ObjectWriter writer = new ObjectMapper() + .writerFor(WeightedPolicyInfo.class).withDefaultPrettyPrinter(); + private static final ObjectReader reader = + new ObjectMapper().readerFor(WeightedPolicyInfo.class); private Map routerPolicyWeights = new HashMap<>(); private Map amrmPolicyWeights = new HashMap<>(); private float headroomAlpha; public WeightedPolicyInfo() { - // JAXB needs this - } - - private static JSONJAXBContext initContext() { - try { - return new JSONJAXBContext(JSONConfiguration.DEFAULT, - WeightedPolicyInfo.class); - } catch (JAXBException e) { - LOG.error("Error parsing the policy.", e); - } - return null; + // Jackson requires this } /** @@ -91,22 +76,18 @@ private static JSONJAXBContext initContext() { public static WeightedPolicyInfo fromByteBuffer(ByteBuffer bb) throws FederationPolicyInitializationException { - if (jsonjaxbContext == null) { + if (reader == null || writer == null) { throw new FederationPolicyInitializationException( - "JSONJAXBContext should" + " not be null."); + "ObjectWriter and ObjectReader are not initialized."); } try { - JSONUnmarshaller unmarshaller = jsonjaxbContext.createJSONUnmarshaller(); final byte[] bytes = new byte[bb.remaining()]; bb.get(bytes); String params = new String(bytes, StandardCharsets.UTF_8); - - WeightedPolicyInfo weightedPolicyInfo = unmarshaller.unmarshalFromJSON( - new StringReader(params), WeightedPolicyInfo.class); - return weightedPolicyInfo; - } catch (JAXBException j) { - throw new FederationPolicyInitializationException(j); + return reader.readValue(params); + } catch (IOException e) { + throw new FederationPolicyInitializationException(e); } } @@ -158,24 +139,20 @@ public void setAMRMPolicyWeights(Map policyWeights) { */ public ByteBuffer toByteBuffer() throws FederationPolicyInitializationException { - if (jsonjaxbContext == null) { + if (reader == null || writer == null) { throw new FederationPolicyInitializationException( - "JSONJAXBContext should" + " not be null."); + "ObjectWriter and ObjectReader are not initialized."); } try { String s = toJSONString(); return ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8)); - } catch (JAXBException j) { - throw new FederationPolicyInitializationException(j); + } catch (JsonProcessingException e) { + throw new FederationPolicyInitializationException(e); } } - private String toJSONString() throws JAXBException { - JSONMarshaller marshaller = jsonjaxbContext.createJSONMarshaller(); - marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); - StringWriter sw = new StringWriter(256); - marshaller.marshallToJSON(this, sw); - return sw.toString(); + private String toJSONString() throws JsonProcessingException { + return writer.writeValueAsString(this); } @Override @@ -243,7 +220,7 @@ public void setHeadroomAlpha(float headroomAlpha) { public String toString() { try { return toJSONString(); - } catch (JAXBException e) { + } catch (JsonProcessingException e) { e.printStackTrace(); return "Error serializing to string."; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java index e2260a1f457fd..c722701adb703 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java @@ -72,4 +72,8 @@ public int hashCode() { return id.hashCode(); } + @Override + public String toString() { + return id; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java index fb8ad602669d6..988393d3228d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java @@ -20,8 +20,6 @@ import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.UniformInterfaceException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.security.UserGroupInformation; @@ -46,6 +44,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.ProcessingException; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.GenericEntity; import javax.ws.rs.core.Response; @@ -88,7 +87,7 @@ private LogAggregationFileControllerFactory getOrCreateFactory() { @VisibleForTesting public String getNMWebAddressFromRM(String nodeId) - throws ClientHandlerException, UniformInterfaceException, JSONException { + throws ProcessingException, IllegalStateException, JSONException { return LogWebServiceUtils.getNMWebAddressFromRM(getConf(), nodeId); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java index 49c72009a4b19..4087a27da61c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebService.java @@ -21,13 +21,7 @@ import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import com.google.inject.Singleton; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; -import com.sun.jersey.core.util.MultivaluedMapImpl; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -44,6 +38,8 @@ import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.HttpUrlConnectorProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,13 +51,16 @@ import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedHashMap; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; import java.io.IOException; import java.net.HttpURLConnection; -import java.net.URL; import java.security.PrivilegedExceptionAction; /** @@ -104,25 +103,22 @@ public LogWebService() { } private Client createTimelineWebClient() { - ClientConfig cfg = new DefaultClientConfig(); - cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class); - Client client = new Client( - new URLConnectionClientHandler(new HttpURLConnectionFactory() { - @Override public HttpURLConnection getHttpURLConnection(URL url) - throws IOException { - AuthenticatedURL.Token token = new AuthenticatedURL.Token(); - HttpURLConnection conn = null; - try { - conn = new AuthenticatedURL().openConnection(url, token); - LOG.info("LogWeService:Connecetion created."); - } catch (AuthenticationException e) { - throw new IOException(e); - } - return conn; + ClientConfig cfg = new ClientConfig(); + cfg.register(YarnJacksonJaxbJsonProvider.class); + cfg.connectorProvider(new HttpUrlConnectorProvider().connectionFactory( + url -> { + AuthenticatedURL.Token token = new AuthenticatedURL.Token(); + HttpURLConnection conn = null; + try { + conn = new AuthenticatedURL().openConnection(url, token); + LOG.info("LogWeService:Connecetion created."); + } catch (AuthenticationException e) { + throw new IOException(e); } - }), cfg); - - return client; + return conn; + } + )); + return ClientBuilder.newClient(cfg); } private void initForReadableEndpoints(HttpServletResponse response) { @@ -166,7 +162,7 @@ public String getNodeHttpAddress(HttpServletRequest req, String appId, String appAttemptId, String containerId, String clusterId) { UserGroupInformation callerUGI = LogWebServiceUtils.getUser(req); String cId = clusterId != null ? clusterId : defaultClusterid; - MultivaluedMap params = new MultivaluedMapImpl(); + MultivaluedMap params = new MultivaluedHashMap<>(); params.add("fields", "INFO"); String path = JOINER.join("clusters/", cId, "/apps/", appId, "/entities/", TimelineEntityType.YARN_CONTAINER.toString(), "/", containerId); @@ -199,7 +195,7 @@ public BasicAppInfo getApp(HttpServletRequest req, String appId, UserGroupInformation callerUGI = LogWebServiceUtils.getUser(req); String cId = clusterId != null ? clusterId : defaultClusterid; - MultivaluedMap params = new MultivaluedMapImpl(); + MultivaluedMap params = new MultivaluedHashMap<>(); params.add("fields", "INFO"); String path = JOINER.join("clusters/", cId, "/apps/", appId); TimelineEntity appEntity = null; @@ -292,22 +288,23 @@ public Response getLogs(@Context HttpServletRequest req, @VisibleForTesting protected TimelineEntity getEntity(String path, MultivaluedMap params) throws IOException { - ClientResponse resp = - getClient().resource(base).path(path).queryParams(params) - .accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + WebTarget t = getClient().target(base).path(path); + params.forEach(t::queryParam); + Response resp = t.request(MediaType.APPLICATION_JSON) + .accept(MediaType.APPLICATION_JSON) + .get(Response.class); if (resp == null - || resp.getStatusInfo().getStatusCode() != ClientResponse.Status.OK + || resp.getStatusInfo().getStatusCode() != Response.Status.OK .getStatusCode()) { String msg = "Response from the timeline reader server is " + ((resp == null) ? "null" : "not successful," + " HTTP error code: " + resp.getStatus() - + ", Server response:\n" + resp.getEntity(String.class)); + + ", Server response:\n" + resp.readEntity(String.class)); LOG.error(msg); throw new IOException(msg); } - TimelineEntity entity = resp.getEntity(TimelineEntity.class); + TimelineEntity entity = resp.readEntity(TimelineEntity.class); return entity; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java index c4568be8011a7..e37e1cd00d9ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java @@ -19,8 +19,6 @@ package org.apache.hadoop.yarn.server.webapp; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.UniformInterfaceException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -40,6 +38,7 @@ import org.slf4j.LoggerFactory; import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.ProcessingException; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; @@ -188,7 +187,7 @@ protected static UserGroupInformation getUser(HttpServletRequest req) { public static String getNMWebAddressFromRM(Configuration yarnConf, String nodeId) - throws ClientHandlerException, UniformInterfaceException, JSONException { + throws ProcessingException, IllegalStateException, JSONException { JSONObject nodeInfo = YarnWebServiceUtils.getNodeInfoFromRMWebService(yarnConf, nodeId) .getJSONObject("node"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 34741e4de9b87..562f0c5fd8c97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -73,11 +73,11 @@ commons-codec - com.sun.jersey - jersey-core + org.glassfish.jersey.core + jersey-common - com.sun.jersey + org.glassfish.jersey.core jersey-client @@ -142,17 +142,19 @@ guice - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 + org.glassfish.jersey.test-framework + jersey-test-framework-core test - com.sun.jersey - jersey-json + org.glassfish.jersey.test-framework.providers + jersey-test-framework-provider-grizzly2 + test - com.sun.jersey.contribs - jersey-guice + org.glassfish.grizzly + grizzly-http-server + test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/JAXBContextResolver.java index 89a5a7b12530f..b3ee689e21fc1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/JAXBContextResolver.java @@ -22,8 +22,6 @@ import java.util.HashSet; import java.util.Arrays; -import com.sun.jersey.api.json.JSONConfiguration; -import com.sun.jersey.api.json.JSONJAXBContext; import com.google.inject.Singleton; import javax.ws.rs.ext.ContextResolver; @@ -38,6 +36,7 @@ import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.webapp.RemoteExceptionData; +import org.glassfish.jersey.jettison.JettisonJaxbContext; @Singleton @Provider @@ -53,11 +52,8 @@ public class JAXBContextResolver implements ContextResolver { RemoteExceptionData.class}; public JAXBContextResolver() throws Exception { - this.types = new HashSet(Arrays.asList(cTypes)); - // sets the json configuration so that the json output looks like - // the xml output - this.context = new JSONJAXBContext(JSONConfiguration.natural(). - rootUnwrapping(false).build(), cTypes); + this.types = new HashSet<>(Arrays.asList(cTypes)); + this.context = new JettisonJaxbContext(cTypes); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebAppFilter.java index 961ea7d4c29b5..cd64562aa8a21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebAppFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebAppFilter.java @@ -19,15 +19,19 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; import java.io.IOException; -import java.io.PrintWriter; import java.util.List; import javax.inject.Inject; import javax.inject.Singleton; +import javax.servlet.Filter; import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.google.inject.Injector; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HtmlQuoting; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -36,13 +40,11 @@ import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.webapp.Controller.RequestContext; -import com.google.inject.Injector; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.http.NameValuePair; @Singleton -public class NMWebAppFilter extends GuiceContainer{ +public class NMWebAppFilter implements Filter { private Injector injector; private Context nmContext; @@ -51,26 +53,26 @@ public class NMWebAppFilter extends GuiceContainer{ @Inject public NMWebAppFilter(Injector injector, Context nmContext) { - super(injector); this.injector = injector; this.nmContext = nmContext; } @Override - public void doFilter(HttpServletRequest request, - HttpServletResponse response, FilterChain chain) throws IOException, - ServletException { + public void init(FilterConfig filterConfig) throws ServletException { + } + + @Override + public void doFilter(ServletRequest servletRequest, + ServletResponse servletResponse, + FilterChain chain) throws IOException, ServletException { + HttpServletRequest request = (HttpServletRequest) servletRequest; + HttpServletResponse response = (HttpServletResponse) servletResponse; + String redirectPath = containerLogPageRedirectPath(request); if (redirectPath != null) { - String redirectMsg = - "Redirecting to log server" + " : " + redirectPath; - PrintWriter out = response.getWriter(); - out.println(redirectMsg); - response.setHeader("Location", redirectPath); - response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT); - return; + response.sendRedirect(redirectPath); } - super.doFilter(request, response, chain); + chain.doFilter(request, response); } private String containerLogPageRedirectPath(HttpServletRequest request) { @@ -89,7 +91,7 @@ private String containerLogPageRedirectPath(HttpServletRequest request) { try { containerId = ContainerId.fromString(containerIdStr); } catch (IllegalArgumentException ex) { - return redirectPath; + return null; } ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId(); @@ -142,4 +144,8 @@ private String containerLogPageRedirectPath(HttpServletRequest request) { } return redirectPath; } + + @Override + public void destroy() { + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java index eab20accf2c68..ab1055ccfd1bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java @@ -39,7 +39,7 @@ import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import javax.servlet.Filter; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -182,7 +182,7 @@ public void setup() { } @Override - protected Class getWebAppFilterClass() { + protected Class getWebAppFilterClass() { return NMWebAppFilter.class; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index 30b73c1acc7ab..f62e2b533fec8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -18,22 +18,22 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; -import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import com.google.inject.AbstractModule; import com.google.inject.Guice; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Scopes; +import com.google.inject.servlet.GuiceFilter; +import com.google.inject.servlet.GuiceServletContextListener; import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.ClientResponse.Status; -import com.sun.jersey.api.client.GenericType; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; + import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.http.JettyUtils; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -69,22 +69,37 @@ import org.apache.hadoop.yarn.util.YarnVersionInfo; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; -import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import org.glassfish.hk2.api.ServiceLocator; +import org.glassfish.hk2.api.ServiceLocatorFactory; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.test.DeploymentContext; +import org.glassfish.jersey.test.JerseyTest; +import org.glassfish.jersey.test.ServletDeploymentContext; +import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory; +import org.glassfish.jersey.test.spi.TestContainerFactory; import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; +import org.jvnet.hk2.guice.bridge.api.GuiceBridge; +import org.jvnet.hk2.guice.bridge.api.GuiceIntoHK2Bridge; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.ApplicationPath; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.GenericType; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import java.io.File; @@ -95,6 +110,7 @@ import java.net.URL; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -111,7 +127,7 @@ /** * Test the nodemanager node info web services api's */ -public class TestNMWebServices extends JerseyTestBase { +public class TestNMWebServices extends JerseyTest { private static final long NM_RESOURCE_VALUE = 1000L; private static NodeManager.NMContext nmContext; @@ -129,6 +145,47 @@ public class TestNMWebServices extends JerseyTestBase { private static File testRemoteLogDir = new File("target", TestNMWebServices.class.getSimpleName() + "remote-log-dir"); + @Override + protected TestContainerFactory getTestContainerFactory() { + return new GrizzlyWebTestContainerFactory(); + } + + @Override + protected DeploymentContext configureDeployment() { + return ServletDeploymentContext + .forPackages("org.apache.hadoop.yarn.server.nodemanager.webapp") + .addListener(GuiceServletContextListener.class) + .filterClass(GuiceFilter.class) + .contextPath("jersey-guice-filter") + .servletPath("/") + .build(); + + } + + /* + @Override + protected Application configure() { + return new MyApplication(ServiceLocatorFactory.getInstance() + .create("test")).packages(); + }*/ + + static { + GuiceServletConfig.setInjector( + Guice.createInjector(new WebServletModule())); + } + + @ApplicationPath("/") + private class MyApplication extends ResourceConfig { + @Inject + public MyApplication(ServiceLocator serviceLocator) { + GuiceBridge.getGuiceBridge().initializeGuiceBridge(serviceLocator); + GuiceIntoHK2Bridge guiceBridge = serviceLocator + .getService(GuiceIntoHK2Bridge.class); + guiceBridge.bridgeGuiceInjector(Guice.createInjector( + new WebServletModule())); + } + } + private static class WebServletModule extends ServletModule { @Override @@ -184,14 +241,8 @@ public boolean isPmemCheckEnabled() { bind(ResourceView.class).toInstance(resourceView); bind(ApplicationACLsManager.class).toInstance(aclsManager); bind(LocalDirsHandlerService.class).toInstance(dirsHandler); - - serve("/*").with(GuiceContainer.class); + bind(GuiceFilter.class).in(Scopes.SINGLETON); } - }; - - static { - GuiceServletConfig.setInjector( - Guice.createInjector(new WebServletModule())); } private void setupMockPluginsWithNmResourceInfo() throws YarnException { @@ -250,42 +301,44 @@ private ResourcePluginManager createResourceManagerWithPlugins( return pluginManager; } - private void assertNMResourceInfoResponse(ClientResponse response, long value) + private void assertNMResourceInfoResponse(Response response, long value) throws JSONException { assertEquals("MediaType of the response is not the expected!", MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); assertEquals("Unexpected value in the json response!", (int) value, json.get("a")); } - private void assertEmptyNMResourceInfo(ClientResponse response) { + private void assertEmptyNMResourceInfo(Response response) { assertEquals("MediaType of the response is not the expected!", MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); assertEquals("Unexpected value in the json response!", 0, json.length()); } - private ClientResponse getNMResourceResponse(WebResource resource, - String resourceName) { - return resource.path("ws").path("v1").path("node").path("resources") - .path(resourceName).accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + private Response getNMResourceResponse(WebTarget target, + String resourceName) { + return target.path("ws").path("v1").path("node").path("resources") + .path(resourceName).request(MediaType.APPLICATION_JSON) + .get(Response.class); } - - @Before @Override public void setUp() throws Exception { super.setUp(); + GuiceServletConfig.setInjector( + Guice.createInjector(new WebServletModule())); + } + + @Before + public void createTesetDir() throws Exception { testRemoteLogDir.mkdir(); testRootDir.mkdirs(); testLogDir.mkdir(); - GuiceServletConfig.setInjector( - Guice.createInjector(new WebServletModule())); } @AfterClass @@ -295,41 +348,34 @@ static public void stop() { FileUtil.fullyDelete(testRemoteLogDir); } - public TestNMWebServices() { - super(new WebAppDescriptor.Builder( - "org.apache.hadoop.yarn.server.nodemanager.webapp") - .contextListenerClass(GuiceServletConfig.class) - .filterClass(com.google.inject.servlet.GuiceFilter.class) - .contextPath("jersey-guice-filter").servletPath("/").build()); - } - @Test - public void testInvalidUri() throws JSONException, Exception { - WebResource r = resource(); + public void testInvalidUri() { + WebTarget t = target(); String responseStr = ""; try { - responseStr = r.path("ws").path("v1").path("node").path("bogus") - .accept(MediaType.APPLICATION_JSON).get(String.class); + responseStr = t.path("ws").path("v1").path("node").path("bogus") + .request(MediaType.APPLICATION_JSON).get(String.class); fail("should have thrown exception on invalid uri"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo()); + } catch (WebApplicationException we) { + Response response = we.getResponse(); + assertResponseStatusCode(Response.Status.NOT_FOUND, + response.getStatusInfo()); WebServicesTestUtils.checkStringMatch( "error string exists and shouldn't", "", responseStr); } } @Test - public void testInvalidAccept() throws JSONException, Exception { - WebResource r = resource(); + public void testInvalidAccept() { + WebTarget t = target(); String responseStr = ""; try { - responseStr = r.path("ws").path("v1").path("node") - .accept(MediaType.TEXT_PLAIN).get(String.class); + responseStr = t.path("ws").path("v1").path("node") + .request(MediaType.TEXT_PLAIN).get(String.class); fail("should have thrown exception on invalid uri"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - assertResponseStatusCode(Status.INTERNAL_SERVER_ERROR, + } catch (WebApplicationException we) { + Response response = we.getResponse(); + assertResponseStatusCode(Response.Status.INTERNAL_SERVER_ERROR, response.getStatusInfo()); WebServicesTestUtils.checkStringMatch( "error string exists and shouldn't", "", responseStr); @@ -337,101 +383,102 @@ public void testInvalidAccept() throws JSONException, Exception { } @Test - public void testInvalidUri2() throws JSONException, Exception { - WebResource r = resource(); + public void testInvalidUri2() { + WebTarget t = target(); String responseStr = ""; try { - responseStr = r.accept(MediaType.APPLICATION_JSON).get(String.class); + responseStr = t.request(MediaType.APPLICATION_JSON).get(String.class); fail("should have thrown exception on invalid uri"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo()); + } catch (WebApplicationException we) { + Response response = we.getResponse(); + assertResponseStatusCode(Response.Status.NOT_FOUND, + response.getStatusInfo()); WebServicesTestUtils.checkStringMatch( "error string exists and shouldn't", "", responseStr); } } @Test - public void testNode() throws JSONException, Exception { - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("node") - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + public void testNode() throws Exception { + WebTarget t = target(); + Response response = t.path("ws").path("v1").path("node") + .request(MediaType.APPLICATION_JSON).get(Response.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); verifyNodeInfo(json); } @Test - public void testNodeSlash() throws JSONException, Exception { - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("node/") - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + public void testNodeSlash() throws Exception { + WebTarget t = target(); + Response response = t.path("ws").path("v1").path("node/") + .request(MediaType.APPLICATION_JSON).get(Response.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); verifyNodeInfo(json); } // make sure default is json output @Test - public void testNodeDefault() throws JSONException, Exception { - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("node") - .get(ClientResponse.class); + public void testNodeDefault() throws Exception { + WebTarget t = target(); + Response response = t.path("ws").path("v1").path("node") + .request().get(Response.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); verifyNodeInfo(json); } @Test - public void testNodeInfo() throws JSONException, Exception { - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("node").path("info") - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + public void testNodeInfo() throws Exception { + WebTarget t = target(); + Response response = t.path("ws").path("v1").path("node").path("info") + .request(MediaType.APPLICATION_JSON).get(Response.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); verifyNodeInfo(json); } @Test - public void testNodeInfoSlash() throws JSONException, Exception { - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("node") - .path("info/").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + public void testNodeInfoSlash() throws Exception { + WebTarget t = target(); + Response response = t.path("ws").path("v1").path("node") + .path("info/").request(MediaType.APPLICATION_JSON) + .get(Response.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); verifyNodeInfo(json); } // make sure default is json output @Test - public void testNodeInfoDefault() throws JSONException, Exception { - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("node").path("info") - .get(ClientResponse.class); + public void testNodeInfoDefault() throws Exception { + WebTarget t = target(); + Response response = t.path("ws").path("v1").path("node").path("info") + .request().get(Response.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); verifyNodeInfo(json); } @Test - public void testSingleNodesXML() throws JSONException, Exception { - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("node") - .path("info/").accept(MediaType.APPLICATION_XML) - .get(ClientResponse.class); + public void testSingleNodesXML() throws Exception { + WebTarget t = target(); + Response response = t.path("ws").path("v1").path("node") + .path("info/").request(MediaType.APPLICATION_XML) + .get(Response.class); assertEquals(MediaType.APPLICATION_XML+ "; " + JettyUtils.UTF_8, - response.getType().toString()); - String xml = response.getEntity(String.class); + response.getMediaType().toString()); + String xml = response.readEntity(String.class); DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); InputSource is = new InputSource(); @@ -445,25 +492,25 @@ public void testSingleNodesXML() throws JSONException, Exception { @Test (timeout = 5000) public void testContainerLogsWithNewAPI() throws Exception { ContainerId containerId0 = BuilderUtils.newContainerId(0, 0, 0, 0); - WebResource r0 = resource(); - r0 = r0.path("ws").path("v1").path("node").path("containers") + WebTarget t0 = target(); + t0 = t0.path("ws").path("v1").path("node").path("containers") .path(containerId0.toString()).path("logs"); - testContainerLogs(r0, containerId0, LOG_MESSAGE); + testContainerLogs(t0, containerId0, LOG_MESSAGE); ContainerId containerId1 = BuilderUtils.newContainerId(0, 0, 0, 1); - WebResource r1 = resource(); - r1 = r1.path("ws").path("v1").path("node").path("containers") + WebTarget t1 = target(); + t1 = t1.path("ws").path("v1").path("node").path("containers") .path(containerId1.toString()).path("logs"); - testContainerLogs(r1, containerId1, ""); + testContainerLogs(t1, containerId1, ""); } @Test (timeout = 5000) public void testContainerLogsWithOldAPI() throws Exception { final ContainerId containerId2 = BuilderUtils.newContainerId(1, 1, 0, 2); - WebResource r = resource(); - r = r.path("ws").path("v1").path("node").path("containerlogs") + WebTarget t = target(); + t = t.path("ws").path("v1").path("node").path("containerlogs") .path(containerId2.toString()); - testContainerLogs(r, containerId2, LOG_MESSAGE); + testContainerLogs(t, containerId2, LOG_MESSAGE); } @Test (timeout = 10000) @@ -475,16 +522,16 @@ public void testNMRedirect() { ContainerId noExistContainerId = ContainerId.newContainerId( noExistAttemptId, 250); String fileName = "syslog"; - WebResource r = resource(); + WebTarget t = target(); // check the old api - URI requestURI = r.path("ws").path("v1").path("node") + URI requestURI = t.path("ws").path("v1").path("node") .path("containerlogs").path(noExistContainerId.toString()) .path(fileName).queryParam("user.name", "user") .queryParam(YarnWebServiceParams.NM_ID, "localhost:1111") - .getURI(); + .getUri(); String redirectURL = getRedirectURL(requestURI.toString()); - assertTrue(redirectURL != null); + assertThat(redirectURL).isNotNull(); assertTrue(redirectURL.contains(LOGSERVICEWSADDR)); assertTrue(redirectURL.contains(noExistContainerId.toString())); assertTrue(redirectURL.contains("/logs/" + fileName)); @@ -494,13 +541,13 @@ public void testNMRedirect() { assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); // check the new api - requestURI = r.path("ws").path("v1").path("node") + requestURI = t.path("ws").path("v1").path("node") .path("containers").path(noExistContainerId.toString()) .path("logs").path(fileName).queryParam("user.name", "user") .queryParam(YarnWebServiceParams.NM_ID, "localhost:1111") - .getURI(); + .getUri(); redirectURL = getRedirectURL(requestURI.toString()); - assertTrue(redirectURL != null); + assertThat(redirectURL).isNotNull(); assertTrue(redirectURL.contains(LOGSERVICEWSADDR)); assertTrue(redirectURL.contains(noExistContainerId.toString())); assertTrue(redirectURL.contains("/logs/" + fileName)); @@ -509,13 +556,13 @@ public void testNMRedirect() { YarnWebServiceParams.REDIRECTED_FROM_NODE + "=true")); assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); - requestURI = r.path("ws").path("v1").path("node") + requestURI = t.path("ws").path("v1").path("node") .path("containers").path(noExistContainerId.toString()) .path("logs").queryParam("user.name", "user") .queryParam(YarnWebServiceParams.NM_ID, "localhost:1111") - .getURI(); + .getUri(); redirectURL = getRedirectURL(requestURI.toString()); - assertTrue(redirectURL != null); + assertThat(redirectURL).isNotNull(); assertTrue(redirectURL.contains(LOGSERVICEWSADDR)); assertTrue(redirectURL.contains(noExistContainerId.toString())); assertTrue(redirectURL.contains("user.name=" + "user")); @@ -529,8 +576,8 @@ public void testGetNMResourceInfoSuccessful() throws YarnException, JSONException { setupMockPluginsWithNmResourceInfo(); - WebResource r = resource(); - ClientResponse response = getNMResourceResponse(r, "resource-1"); + WebTarget t = target(); + Response response = getNMResourceResponse(t, "resource-1"); assertNMResourceInfoResponse(response, NM_RESOURCE_VALUE); } @@ -540,8 +587,8 @@ public void testGetNMResourceInfoEncodedIsSuccessful() setupMockPluginsWithNmResourceInfo(); //test encoded yarn.io/resource-1 path - WebResource r = resource(); - ClientResponse response = getNMResourceResponse(r, "yarn.io%2Fresource-1"); + WebTarget t = target(); + Response response = getNMResourceResponse(t, "yarn.io%2Fresource-1"); assertNMResourceInfoResponse(response, NM_RESOURCE_VALUE); } @@ -550,8 +597,8 @@ public void testGetNMResourceInfoFailBecauseOfEmptyResourceInfo() throws YarnException { setupMockPluginsWithNmResourceInfo(); - WebResource r = resource(); - ClientResponse response = getNMResourceResponse(r, "resource-2"); + WebTarget t = target(); + Response response = getNMResourceResponse(t, "resource-2"); assertEmptyNMResourceInfo(response); } @@ -560,8 +607,8 @@ public void testGetNMResourceInfoWhenPluginIsUnknown() throws YarnException { setupMockPluginsWithNmResourceInfo(); - WebResource r = resource(); - ClientResponse response = getNMResourceResponse(r, "resource-3"); + WebTarget t = target(); + Response response = getNMResourceResponse(t, "resource-3"); assertEmptyNMResourceInfo(response); } @@ -577,12 +624,12 @@ public void testGetYarnGpuResourceInfo() throws YarnException, JSONException { setupMockPluginsWithGpuResourceInfo(); - WebResource r = resource(); - ClientResponse response = getNMResourceResponse(r, "resource-1"); + WebTarget t = target(); + Response response = getNMResourceResponse(t, "resource-1"); assertEquals("MediaType of the response is not the expected!", MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); assertEquals("Unexpected driverVersion in the json response!", "1.2.3", json.getJSONObject("gpuDeviceInformation").get("driverVersion")); @@ -592,7 +639,7 @@ public void testGetYarnGpuResourceInfo() 2, json.getJSONArray("assignedGpuDevices").length()); } - private void testContainerLogs(WebResource r, ContainerId containerId, + private void testContainerLogs(WebTarget t, ContainerId containerId, String logMessage) throws Exception { final String containerIdStr = containerId.toString(); final ApplicationAttemptId appAttemptId = containerId @@ -624,9 +671,9 @@ private void testContainerLogs(WebResource r, ContainerId containerId, pw.close(); // ask for it - ClientResponse response = r.path(filename) - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); - String responseText = response.getEntity(String.class); + Response response = t.path(filename) + .request(MediaType.TEXT_PLAIN).get(Response.class); + String responseText = response.readEntity(String.class); String responseLogMessage = getLogContext(responseText); assertEquals(logMessage, responseLogMessage); int fullTextSize = responseLogMessage.getBytes().length; @@ -634,10 +681,10 @@ private void testContainerLogs(WebResource r, ContainerId containerId, // specify how many bytes we should get from logs // specify a position number, it would get the first n bytes from // container log - response = r.path(filename) + response = t.path(filename) .queryParam("size", "5") - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); - responseText = response.getEntity(String.class); + .request(MediaType.TEXT_PLAIN).get(Response.class); + responseText = response.readEntity(String.class); responseLogMessage = getLogContext(responseText); int truncatedLength = Math.min(5, logMessage.getBytes().length); assertEquals(truncatedLength, responseLogMessage.getBytes().length); @@ -647,20 +694,20 @@ private void testContainerLogs(WebResource r, ContainerId containerId, // specify the bytes which is larger than the actual file size, // we would get the full logs - response = r.path(filename) + response = t.path(filename) .queryParam("size", "10000") - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); - responseText = response.getEntity(String.class); + .request(MediaType.TEXT_PLAIN).get(Response.class); + responseText = response.readEntity(String.class); responseLogMessage = getLogContext(responseText); assertEquals(fullTextSize, responseLogMessage.getBytes().length); assertEquals(logMessage, responseLogMessage); // specify a negative number, it would get the last n bytes from // container log - response = r.path(filename) + response = t.path(filename) .queryParam("size", "-5") - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); - responseText = response.getEntity(String.class); + .request(MediaType.TEXT_PLAIN).get(Response.class); + responseText = response.readEntity(String.class); responseLogMessage = getLogContext(responseText); assertEquals(truncatedLength, responseLogMessage.getBytes().length); assertEquals(new String(logMessage.getBytes(), @@ -668,57 +715,57 @@ private void testContainerLogs(WebResource r, ContainerId containerId, responseLogMessage); assertTrue(fullTextSize >= responseLogMessage.getBytes().length); - response = r.path(filename) + response = t.path(filename) .queryParam("size", "-10000") - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); - responseText = response.getEntity(String.class); + .request(MediaType.TEXT_PLAIN).get(Response.class); + responseText = response.readEntity(String.class); responseLogMessage = getLogContext(responseText); - assertEquals("text/plain; charset=utf-8", response.getType().toString()); + assertEquals("text/plain; charset=utf-8", response.getMediaType().toString()); assertEquals(fullTextSize, responseLogMessage.getBytes().length); assertEquals(logMessage, responseLogMessage); // ask and download it - response = r.path(filename) + response = t.path(filename) .queryParam("format", "octet-stream") - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); - responseText = response.getEntity(String.class); + .request(MediaType.TEXT_PLAIN).get(Response.class); + responseText = response.readEntity(String.class); responseLogMessage = getLogContext(responseText); assertEquals(logMessage, responseLogMessage); assertEquals(200, response.getStatus()); assertEquals("application/octet-stream; charset=utf-8", - response.getType().toString()); + response.getMediaType().toString()); // specify a invalid format value - response = r.path(filename) + response = t.path(filename) .queryParam("format", "123") - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); - responseText = response.getEntity(String.class); + .request(MediaType.TEXT_PLAIN).get(Response.class); + responseText = response.readEntity(String.class); assertEquals("The valid values for the parameter : format are " + WebAppUtils.listSupportedLogContentType(), responseText); assertEquals(400, response.getStatus()); // ask for file that doesn't exist and it will re-direct to // the log server - URI requestURI = r.path("uhhh").getURI(); + URI requestURI = t.path("uhhh").getUri(); String redirectURL = getRedirectURL(requestURI.toString()); - assertTrue(redirectURL != null); + assertThat(redirectURL).isNotNull(); assertTrue(redirectURL.contains(LOGSERVICEWSADDR)); // Get container log files' name - WebResource r1 = resource(); - response = r1.path("ws").path("v1").path("node") + WebTarget t1 = target(); + response = t1.path("ws").path("v1").path("node") .path("containers").path(containerIdStr) - .path("logs").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + .path("logs").request(MediaType.APPLICATION_JSON) + .get(Response.class); assertEquals(200, response.getStatus()); - List responseList = response.getEntity(new GenericType< - List>(){}); - assertTrue(responseList.size() == 1); + List responseList = response.readEntity( + new GenericType>(){}); + assertThat(responseList).hasSize(1); assertEquals(responseList.get(0).getLogType(), ContainerLogAggregationType.LOCAL.toString()); List logMeta = responseList.get(0) .getContainerLogsInfo(); - assertTrue(logMeta.size() == 1); + assertThat(logMeta).hasSize(1); assertThat(logMeta.get(0).getFileName()).isEqualTo(filename); // now create an aggregated log in Remote File system @@ -732,26 +779,26 @@ private void testContainerLogs(WebResource r, ContainerId containerId, tempLogDir.getAbsolutePath(), appId, Collections.singletonMap(containerId, aggregatedLogMessage), nmContext.getNodeId(), aggregatedLogFile, "user", true); - r1 = resource(); - response = r1.path("ws").path("v1").path("node") + t1 = target(); + response = t1.path("ws").path("v1").path("node") .path("containers").path(containerIdStr) - .path("logs").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + .path("logs").request(MediaType.APPLICATION_JSON) + .get(Response.class); assertEquals(200, response.getStatus()); - responseList = response.getEntity(new GenericType< + responseList = response.readEntity(new GenericType< List>(){}); assertThat(responseList).hasSize(2); for (ContainerLogsInfo logInfo : responseList) { if(logInfo.getLogType().equals( ContainerLogAggregationType.AGGREGATED.toString())) { List meta = logInfo.getContainerLogsInfo(); - assertTrue(meta.size() == 1); + assertThat(meta).hasSize(1); assertThat(meta.get(0).getFileName()).isEqualTo(aggregatedLogFile); } else { assertEquals(logInfo.getLogType(), ContainerLogAggregationType.LOCAL.toString()); List meta = logInfo.getContainerLogsInfo(); - assertTrue(meta.size() == 1); + assertThat(meta).hasSize(1); assertThat(meta.get(0).getFileName()).isEqualTo(filename); } } @@ -762,9 +809,9 @@ private void testContainerLogs(WebResource r, ContainerId containerId, tempLogDir.getAbsolutePath(), appId, Collections.singletonMap(containerId, aggregatedLogMessage), nmContext.getNodeId(), filename, "user", true); - response = r.path(filename) - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); - responseText = response.getEntity(String.class); + response = t.path(filename) + .request(MediaType.TEXT_PLAIN).get(Response.class); + responseText = response.readEntity(String.class); assertTrue(responseText.contains("LogAggregationType: " + ContainerLogAggregationType.AGGREGATED)); assertTrue(responseText.contains(aggregatedLogMessage)); @@ -778,9 +825,9 @@ private void testContainerLogs(WebResource r, ContainerId containerId, nmContext.getContainers().remove(containerId); assertNull(nmContext.getContainers().get(containerId)); response = - r.path(filename).accept(MediaType.TEXT_PLAIN) - .get(ClientResponse.class); - responseText = response.getEntity(String.class); + t.path(filename).request(MediaType.TEXT_PLAIN) + .get(Response.class); + responseText = response.readEntity(String.class); assertTrue(responseText.contains(logMessage)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java index ab06c0f9f3396..42d8432dd1462 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java @@ -33,7 +33,10 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -60,12 +63,12 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; -import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import org.glassfish.jersey.test.JerseyTest; import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; @@ -76,15 +79,11 @@ import org.xml.sax.InputSource; import com.google.inject.Guice; +import com.google.inject.Scopes; +import com.google.inject.servlet.GuiceFilter; import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.ClientResponse.Status; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; -public class TestNMWebServicesApps extends JerseyTestBase { +public class TestNMWebServicesApps extends JerseyTest { private static Context nmContext; private static ResourceView resourceView; @@ -151,8 +150,7 @@ public boolean isPmemCheckEnabled() { bind(ResourceView.class).toInstance(resourceView); bind(ApplicationACLsManager.class).toInstance(aclsManager); bind(LocalDirsHandlerService.class).toInstance(dirsHandler); - - serve("/*").with(GuiceContainer.class); + bind(GuiceFilter.class).in(Scopes.SINGLETON); } } @@ -178,21 +176,23 @@ static public void cleanup() { } public TestNMWebServicesApps() { + /* TODO: Remove or fix this super(new WebAppDescriptor.Builder( "org.apache.hadoop.yarn.server.nodemanager.webapp") .contextListenerClass(GuiceServletConfig.class) .filterClass(com.google.inject.servlet.GuiceFilter.class) .contextPath("jersey-guice-filter").servletPath("/").build()); + */ } @Test - public void testNodeAppsNone() throws JSONException, Exception { - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("node").path("apps") - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + public void testNodeAppsNone() throws Exception { + WebTarget t = target(); + Response response = t.path("ws").path("v1").path("node").path("apps") + .request(MediaType.APPLICATION_JSON).get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); assertEquals("apps isn't empty", new JSONObject().toString(), json.get("apps").toString()); } @@ -225,25 +225,25 @@ private HashMap addAppContainers(Application app, } @Test - public void testNodeApps() throws JSONException, Exception { + public void testNodeApps() throws Exception { testNodeHelper("apps", MediaType.APPLICATION_JSON); } @Test - public void testNodeAppsSlash() throws JSONException, Exception { + public void testNodeAppsSlash() throws Exception { testNodeHelper("apps/", MediaType.APPLICATION_JSON); } // make sure default is json output @Test - public void testNodeAppsDefault() throws JSONException, Exception { + public void testNodeAppsDefault() throws Exception { testNodeHelper("apps/", ""); } public void testNodeHelper(String path, String media) throws JSONException, Exception { - WebResource r = resource(); + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); HashMap hash = addAppContainers(app); @@ -251,11 +251,11 @@ public void testNodeHelper(String path, String media) throws JSONException, nmContext.getApplications().put(app2.getAppId(), app2); HashMap hash2 = addAppContainers(app2); - ClientResponse response = r.path("ws").path("v1").path("node").path(path) - .accept(media).get(ClientResponse.class); + Response response = t.path("ws").path("v1").path("node").path(path) + .request(media).get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); JSONObject info = json.getJSONObject("apps"); assertEquals("incorrect number of elements", 1, info.length()); JSONArray appInfo = info.getJSONArray("app"); @@ -271,8 +271,8 @@ public void testNodeHelper(String path, String media) throws JSONException, } @Test - public void testNodeAppsUser() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeAppsUser() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); HashMap hash = addAppContainers(app); @@ -280,12 +280,12 @@ public void testNodeAppsUser() throws JSONException, Exception { nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); - ClientResponse response = r.path("ws").path("v1").path("node").path("apps") - .queryParam("user", "mockUser").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + Response response = t.path("ws").path("v1").path("node").path("apps") + .queryParam("user", "mockUser").request(MediaType.APPLICATION_JSON) + .get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); JSONObject info = json.getJSONObject("apps"); assertEquals("incorrect number of elements", 1, info.length()); @@ -295,8 +295,8 @@ public void testNodeAppsUser() throws JSONException, Exception { } @Test - public void testNodeAppsUserNone() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeAppsUserNone() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -304,19 +304,19 @@ public void testNodeAppsUserNone() throws JSONException, Exception { nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); - ClientResponse response = r.path("ws").path("v1").path("node").path("apps") - .queryParam("user", "george").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + Response response = t.path("ws").path("v1").path("node").path("apps") + .queryParam("user", "george").request(MediaType.APPLICATION_JSON) + .get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); assertEquals("apps is not empty", new JSONObject().toString(), json.get("apps").toString()); } @Test - public void testNodeAppsUserEmpty() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeAppsUserEmpty() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -325,16 +325,17 @@ public void testNodeAppsUserEmpty() throws JSONException, Exception { addAppContainers(app2); try { - r.path("ws").path("v1").path("node").path("apps").queryParam("user", "") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + t.path("ws").path("v1").path("node").path("apps").queryParam("user", "") + .request(MediaType.APPLICATION_JSON).get(JSONObject.class); fail("should have thrown exception on invalid user query"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); + } catch (WebApplicationException we) { + Response response = we.getResponse(); - assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo()); + assertResponseStatusCode(Response.Status.BAD_REQUEST, + response.getStatusInfo()); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject msg = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject msg = response.readEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); assertEquals("incorrect number of elements", 3, exception.length()); String message = exception.getString("message"); @@ -353,8 +354,8 @@ public void testNodeAppsUserEmpty() throws JSONException, Exception { } @Test - public void testNodeAppsState() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeAppsState() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -363,13 +364,13 @@ public void testNodeAppsState() throws JSONException, Exception { HashMap hash2 = addAppContainers(app2); app2.setState(ApplicationState.RUNNING); - ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + Response response = t.path("ws").path("v1").path("node").path("apps") .queryParam("state", ApplicationState.RUNNING.toString()) - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + .request(MediaType.APPLICATION_JSON).get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); JSONObject info = json.getJSONObject("apps"); assertEquals("incorrect number of elements", 1, info.length()); @@ -380,8 +381,8 @@ public void testNodeAppsState() throws JSONException, Exception { } @Test - public void testNodeAppsStateNone() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeAppsStateNone() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -389,20 +390,20 @@ public void testNodeAppsStateNone() throws JSONException, Exception { nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); - ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + Response response = t.path("ws").path("v1").path("node").path("apps") .queryParam("state", ApplicationState.INITING.toString()) - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + .request(MediaType.APPLICATION_JSON).get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); assertEquals("apps is not empty", new JSONObject().toString(), json.get("apps").toString()); } @Test - public void testNodeAppsStateInvalid() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeAppsStateInvalid() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -411,17 +412,18 @@ public void testNodeAppsStateInvalid() throws JSONException, Exception { addAppContainers(app2); try { - r.path("ws").path("v1").path("node").path("apps") - .queryParam("state", "FOO_STATE").accept(MediaType.APPLICATION_JSON) + t.path("ws").path("v1").path("node").path("apps") + .queryParam("state", "FOO_STATE").request(MediaType.APPLICATION_JSON) .get(JSONObject.class); fail("should have thrown exception on invalid user query"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); + } catch (WebApplicationException we) { + Response response = we.getResponse(); - assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo()); + assertResponseStatusCode(Response.Status.BAD_REQUEST, + response.getStatusInfo()); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject msg = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject msg = response.readEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); assertEquals("incorrect number of elements", 3, exception.length()); String message = exception.getString("message"); @@ -433,8 +435,8 @@ public void testNodeAppsStateInvalid() throws JSONException, Exception { // verify the exception object default format is JSON @Test - public void testNodeAppsStateInvalidDefault() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeAppsStateInvalidDefault() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -443,16 +445,18 @@ public void testNodeAppsStateInvalidDefault() throws JSONException, Exception { addAppContainers(app2); try { - r.path("ws").path("v1").path("node").path("apps") - .queryParam("state", "FOO_STATE").get(JSONObject.class); + t.path("ws").path("v1").path("node").path("apps") + .queryParam("state", "FOO_STATE") + .request().get(JSONObject.class); fail("should have thrown exception on invalid user query"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); + } catch (WebApplicationException we) { + Response response = we.getResponse(); - assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo()); + assertResponseStatusCode(Response.Status.BAD_REQUEST, + response.getStatusInfo()); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject msg = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject msg = response.readEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); assertEquals("incorrect number of elements", 3, exception.length()); String message = exception.getString("message"); @@ -464,8 +468,8 @@ public void testNodeAppsStateInvalidDefault() throws JSONException, Exception { // test that the exception output also returns XML @Test - public void testNodeAppsStateInvalidXML() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeAppsStateInvalidXML() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -474,17 +478,18 @@ public void testNodeAppsStateInvalidXML() throws JSONException, Exception { addAppContainers(app2); try { - r.path("ws").path("v1").path("node").path("apps") - .queryParam("state", "FOO_STATE").accept(MediaType.APPLICATION_XML) + t.path("ws").path("v1").path("node").path("apps") + .queryParam("state", "FOO_STATE").request(MediaType.APPLICATION_XML) .get(JSONObject.class); fail("should have thrown exception on invalid user query"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); + } catch (WebApplicationException we) { + Response response = we.getResponse(); - assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo()); + assertResponseStatusCode(Response.Status.BAD_REQUEST, + response.getStatusInfo()); assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - String msg = response.getEntity(String.class); + response.getMediaType().toString()); + String msg = response.readEntity(String.class); DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); @@ -515,19 +520,18 @@ private void verifyStateInvalidException(String message, String type, } @Test - public void testNodeSingleApps() throws JSONException, Exception { + public void testNodeSingleApps() throws Exception { testNodeSingleAppHelper(MediaType.APPLICATION_JSON); } // make sure default is json output @Test - public void testNodeSingleAppsDefault() throws JSONException, Exception { + public void testNodeSingleAppsDefault() throws Exception { testNodeSingleAppHelper(""); } - public void testNodeSingleAppHelper(String media) throws JSONException, - Exception { - WebResource r = resource(); + public void testNodeSingleAppHelper(String media) throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); HashMap hash = addAppContainers(app); @@ -535,37 +539,37 @@ public void testNodeSingleAppHelper(String media) throws JSONException, nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); - ClientResponse response = r.path("ws").path("v1").path("node").path("apps") - .path(app.getAppId().toString()).accept(media) - .get(ClientResponse.class); + Response response = t.path("ws").path("v1").path("node").path("apps") + .path(app.getAppId().toString()).request(media) + .get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); verifyNodeAppInfo(json.getJSONObject("app"), app, hash); } @Test - public void testNodeSingleAppsSlash() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeSingleAppsSlash() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); HashMap hash = addAppContainers(app); Application app2 = new MockApp(2); nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); - ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + Response response = t.path("ws").path("v1").path("node").path("apps") .path(app.getAppId().toString() + "/") - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + .request(MediaType.APPLICATION_JSON).get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); verifyNodeAppInfo(json.getJSONObject("app"), app, hash); } @Test - public void testNodeSingleAppsInvalid() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeSingleAppsInvalid() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -574,15 +578,16 @@ public void testNodeSingleAppsInvalid() throws JSONException, Exception { addAppContainers(app2); try { - r.path("ws").path("v1").path("node").path("apps").path("app_foo_0000") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + t.path("ws").path("v1").path("node").path("apps").path("app_foo_0000") + .request(MediaType.APPLICATION_JSON).get(JSONObject.class); fail("should have thrown exception on invalid user query"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo()); + } catch (WebApplicationException we) { + Response response = we.getResponse(); + assertResponseStatusCode(Response.Status.BAD_REQUEST, + response.getStatusInfo()); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject msg = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject msg = response.readEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); assertEquals("incorrect number of elements", 3, exception.length()); String message = exception.getString("message"); @@ -601,8 +606,8 @@ public void testNodeSingleAppsInvalid() throws JSONException, Exception { } @Test - public void testNodeSingleAppsMissing() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeSingleAppsMissing() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -611,16 +616,17 @@ public void testNodeSingleAppsMissing() throws JSONException, Exception { addAppContainers(app2); try { - r.path("ws").path("v1").path("node").path("apps") - .path("application_1234_0009").accept(MediaType.APPLICATION_JSON) + t.path("ws").path("v1").path("node").path("apps") + .path("application_1234_0009").request(MediaType.APPLICATION_JSON) .get(JSONObject.class); fail("should have thrown exception on invalid user query"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo()); + } catch (WebApplicationException we) { + Response response = we.getResponse(); + assertResponseStatusCode(Response.Status.NOT_FOUND, + response.getStatusInfo()); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject msg = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject msg = response.readEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); assertEquals("incorrect number of elements", 3, exception.length()); String message = exception.getString("message"); @@ -637,8 +643,8 @@ public void testNodeSingleAppsMissing() throws JSONException, Exception { } @Test - public void testNodeAppsXML() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeAppsXML() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -646,11 +652,11 @@ public void testNodeAppsXML() throws JSONException, Exception { nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); - ClientResponse response = r.path("ws").path("v1").path("node").path("apps") - .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + Response response = t.path("ws").path("v1").path("node").path("apps") + .request(MediaType.APPLICATION_XML).get(Response.class); assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - String xml = response.getEntity(String.class); + response.getMediaType().toString()); + String xml = response.readEntity(String.class); DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); InputSource is = new InputSource(); @@ -661,8 +667,8 @@ public void testNodeAppsXML() throws JSONException, Exception { } @Test - public void testNodeSingleAppsXML() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeSingleAppsXML() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); HashMap hash = addAppContainers(app); @@ -670,12 +676,12 @@ public void testNodeSingleAppsXML() throws JSONException, Exception { nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); - ClientResponse response = r.path("ws").path("v1").path("node").path("apps") + Response response = t.path("ws").path("v1").path("node").path("apps") .path(app.getAppId().toString() + "/") - .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); + .request(MediaType.APPLICATION_XML).get(Response.class); assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - String xml = response.getEntity(String.class); + response.getMediaType().toString()); + String xml = response.readEntity(String.class); DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); InputSource is = new InputSource(); @@ -732,7 +738,7 @@ public void verifyNodeAppInfoGeneric(Application app, String id, } @Test - public void testNodeAppsUserFiltering() throws JSONException, Exception { + public void testNodeAppsUserFiltering() throws Exception { Configuration yarnConf = new Configuration(); yarnConf.setBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, true); yarnConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java index 7ec8fcd47d30b..d3ef5b2b66db2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java @@ -30,13 +30,15 @@ import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Date; +import java.util.logging.Logger; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.filter.LoggingFilter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.http.JettyUtils; @@ -53,12 +55,13 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; -import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import org.glassfish.jersey.logging.LoggingFeature; +import org.glassfish.jersey.test.JerseyTest; import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; @@ -68,17 +71,17 @@ import org.xml.sax.InputSource; import com.google.inject.Guice; +import com.google.inject.Scopes; +import com.google.inject.servlet.GuiceFilter; import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; /** * Basic sanity Tests for AuxServices. * */ -public class TestNMWebServicesAuxServices extends JerseyTestBase { +public class TestNMWebServicesAuxServices extends JerseyTest { + private static final Logger LOG = + Logger.getLogger(TestNMWebServicesAuxServices.class.getName()); private static final String AUX_SERVICES_PATH = "auxiliaryservices"; private static Context nmContext; private static Configuration conf = new Configuration(); @@ -148,8 +151,7 @@ public int getHttpPort() { bind(ResourceView.class).toInstance(resourceView); bind(ApplicationACLsManager.class).toInstance(aclsManager); bind(LocalDirsHandlerService.class).toInstance(dirsHandler); - - serve("/*").with(GuiceContainer.class); + bind(GuiceFilter.class).in(Scopes.SINGLETON); } } @@ -175,23 +177,26 @@ static public void cleanup() { } public TestNMWebServicesAuxServices() { + /* TODO: fix or remove this super(new WebAppDescriptor.Builder( "org.apache.hadoop.yarn.server.nodemanager.webapp") .contextListenerClass(GuiceServletConfig.class) .filterClass(com.google.inject.servlet.GuiceFilter.class) .contextPath("jersey-guice-filter").servletPath("/").build()); + + */ } @Test public void testNodeAuxServicesNone() throws Exception { addAuxServices(); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("node") - .path(AUX_SERVICES_PATH).accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + WebTarget t = target(); + Response response = t.path("ws").path("v1").path("node") + .path(AUX_SERVICES_PATH).request(MediaType.APPLICATION_JSON) + .get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); assertEquals("aux services isn't empty", new JSONObject().toString(), json.get("services").toString()); } @@ -225,14 +230,14 @@ public void testNodeHelper(String path, String media) throws Exception { AuxServiceRecord r2 = new AuxServiceRecord().name("name2").launchTime(new Date(456L)); addAuxServices(r1, r2); - WebResource r = resource(); - client().addFilter(new LoggingFilter()); + WebTarget t = target(); + client().register(new LoggingFeature(LOG)); - ClientResponse response = r.path("ws").path("v1").path("node").path(path) - .accept(media).get(ClientResponse.class); + Response response = t.path("ws").path("v1").path("node").path(path) + .request(media).get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); JSONObject info = json.getJSONObject("services"); assertEquals("incorrect number of elements", 1, info.length()); JSONArray auxInfo = info.getJSONArray("service"); @@ -249,14 +254,14 @@ public void testNodeAuxServicesXML() throws Exception { AuxServiceRecord r2 = new AuxServiceRecord().name("name2").launchTime(new Date(456L)); addAuxServices(r1, r2); - WebResource r = resource(); + WebTarget t = target(); - ClientResponse response = r.path("ws").path("v1").path("node") - .path(AUX_SERVICES_PATH).accept(MediaType.APPLICATION_XML) - .get(ClientResponse.class); + Response response = t.path("ws").path("v1").path("node") + .path(AUX_SERVICES_PATH).request(MediaType.APPLICATION_XML) + .get(Response.class); assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - String xml = response.getEntity(String.class); + response.getMediaType().toString()); + String xml = response.readEntity(String.class); DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); InputSource is = new InputSource(); @@ -272,18 +277,18 @@ public void testAuxServicesDisabled() throws JSONException, Exception { AuxServices auxServices = mock(AuxServices.class); when(auxServices.isManifestEnabled()).thenReturn(false); nmContext.setAuxServices(auxServices); - WebResource r = resource(); + WebTarget t = target(); try { - r.path("ws").path("v1").path("node").path(AUX_SERVICES_PATH) - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + t.path("ws").path("v1").path("node").path(AUX_SERVICES_PATH) + .request(MediaType.APPLICATION_JSON).get(JSONObject.class); fail("should have thrown exception on invalid user query"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - assertResponseStatusCode(ClientResponse.Status.BAD_REQUEST, + } catch (WebApplicationException we) { + Response response = we.getResponse(); + assertResponseStatusCode(Response.Status.BAD_REQUEST, response.getStatusInfo()); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject msg = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject msg = response.readEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); assertEquals("incorrect number of elements", 3, exception.length()); String message = exception.getString("message"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java index 175a0b02470ba..dd93771a9f346 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java @@ -30,12 +30,17 @@ import java.io.StringReader; import java.util.HashMap; import java.util.List; +import java.util.logging.Logger; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import com.sun.jersey.api.client.filter.LoggingFilter; +import com.google.inject.Scopes; +import com.google.inject.servlet.GuiceFilter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.http.JettyUtils; @@ -58,12 +63,13 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; -import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import org.glassfish.jersey.logging.LoggingFeature; +import org.glassfish.jersey.test.JerseyTest; import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; @@ -74,15 +80,11 @@ import com.google.inject.Guice; import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.ClientResponse.Status; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; -public class TestNMWebServicesContainers extends JerseyTestBase { +public class TestNMWebServicesContainers extends JerseyTest { + private static final Logger LOG = + Logger.getLogger(TestNMWebServicesContainers.class.getName()); private static Context nmContext; private static ResourceView resourceView; private static ApplicationACLsManager aclsManager; @@ -153,8 +155,7 @@ public int getHttpPort() { bind(ResourceView.class).toInstance(resourceView); bind(ApplicationACLsManager.class).toInstance(aclsManager); bind(LocalDirsHandlerService.class).toInstance(dirsHandler); - - serve("/*").with(GuiceContainer.class); + bind(GuiceFilter.class).in(Scopes.SINGLETON); } } @@ -180,22 +181,24 @@ static public void cleanup() { } public TestNMWebServicesContainers() { + /* TODO: Remove or fix this super(new WebAppDescriptor.Builder( "org.apache.hadoop.yarn.server.nodemanager.webapp") .contextListenerClass(GuiceServletConfig.class) .filterClass(com.google.inject.servlet.GuiceFilter.class) .contextPath("jersey-guice-filter").servletPath("/").build()); + */ } @Test - public void testNodeContainersNone() throws JSONException, Exception { - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("node") - .path("containers").accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + public void testNodeContainersNone() throws Exception { + WebTarget t = target(); + Response response = t.path("ws").path("v1").path("node") + .path("containers").request(MediaType.APPLICATION_JSON) + .get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); assertEquals("apps isn't empty", new JSONObject().toString(), json.get("containers").toString()); } @@ -240,38 +243,37 @@ private HashMap addAppContainers(Application app) } @Test - public void testNodeContainers() throws JSONException, Exception { + public void testNodeContainers() throws Exception { testNodeHelper("containers", MediaType.APPLICATION_JSON); } @Test - public void testNodeContainersSlash() throws JSONException, Exception { + public void testNodeContainersSlash() throws Exception { testNodeHelper("containers/", MediaType.APPLICATION_JSON); } // make sure default is json output @Test - public void testNodeContainersDefault() throws JSONException, Exception { + public void testNodeContainersDefault() throws Exception { testNodeHelper("containers/", ""); } - public void testNodeHelper(String path, String media) throws JSONException, - Exception { - WebResource r = resource(); + public void testNodeHelper(String path, String media) throws Exception { + WebTarget r = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); Application app2 = new MockApp(2); nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); - client().addFilter(new LoggingFilter()); + client().register(new LoggingFeature(LOG)); - ClientResponse response = r.path("ws").path("v1").path("node").path(path) - .accept(media).get(ClientResponse.class); + Response response = r.path("ws").path("v1").path("node").path(path) + .request(media).get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); JSONObject info = json.getJSONObject("containers"); assertEquals("incorrect number of elements", 1, info.length()); JSONArray conInfo = info.getJSONArray("container"); @@ -287,23 +289,23 @@ public void testNodeHelper(String path, String media) throws JSONException, } @Test - public void testNodeSingleContainers() throws JSONException, Exception { + public void testNodeSingleContainers() throws Exception { testNodeSingleContainersHelper(MediaType.APPLICATION_JSON); } @Test - public void testNodeSingleContainersSlash() throws JSONException, Exception { + public void testNodeSingleContainersSlash() throws Exception { testNodeSingleContainersHelper(MediaType.APPLICATION_JSON); } @Test - public void testNodeSingleContainersDefault() throws JSONException, Exception { + public void testNodeSingleContainersDefault() throws Exception { testNodeSingleContainersHelper(""); } public void testNodeSingleContainersHelper(String media) throws JSONException, Exception { - WebResource r = resource(); + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); HashMap hash = addAppContainers(app); @@ -312,19 +314,19 @@ public void testNodeSingleContainersHelper(String media) addAppContainers(app2); for (String id : hash.keySet()) { - ClientResponse response = r.path("ws").path("v1").path("node") - .path("containers").path(id).accept(media).get(ClientResponse.class); + Response response = t.path("ws").path("v1").path("node") + .path("containers").path(id).request(media).get(Response.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject json = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject json = response.readEntity(JSONObject.class); verifyNodeContainerInfo(json.getJSONObject("container"), nmContext .getContainers().get(ContainerId.fromString(id))); } } @Test - public void testSingleContainerInvalid() throws JSONException, Exception { - WebResource r = resource(); + public void testSingleContainerInvalid() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -332,16 +334,17 @@ public void testSingleContainerInvalid() throws JSONException, Exception { nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); try { - r.path("ws").path("v1").path("node").path("containers") - .path("container_foo_1234").accept(MediaType.APPLICATION_JSON) + t.path("ws").path("v1").path("node").path("containers") + .path("container_foo_1234").request(MediaType.APPLICATION_JSON) .get(JSONObject.class); fail("should have thrown exception on invalid user query"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo()); + } catch (WebApplicationException we) { + Response response = we.getResponse(); + assertResponseStatusCode(Response.Status.BAD_REQUEST, + response.getStatusInfo()); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject msg = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject msg = response.readEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); assertEquals("incorrect number of elements", 3, exception.length()); String message = exception.getString("message"); @@ -358,8 +361,8 @@ public void testSingleContainerInvalid() throws JSONException, Exception { } @Test - public void testSingleContainerInvalid2() throws JSONException, Exception { - WebResource r = resource(); + public void testSingleContainerInvalid2() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -367,16 +370,17 @@ public void testSingleContainerInvalid2() throws JSONException, Exception { nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); try { - r.path("ws").path("v1").path("node").path("containers") - .path("container_1234_0001").accept(MediaType.APPLICATION_JSON) + t.path("ws").path("v1").path("node").path("containers") + .path("container_1234_0001").request(MediaType.APPLICATION_JSON) .get(JSONObject.class); fail("should have thrown exception on invalid user query"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo()); + } catch (WebApplicationException we) { + Response response = we.getResponse(); + assertResponseStatusCode(Response.Status.BAD_REQUEST, + response.getStatusInfo()); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject msg = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject msg = response.readEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); assertEquals("incorrect number of elements", 3, exception.length()); String message = exception.getString("message"); @@ -393,8 +397,8 @@ public void testSingleContainerInvalid2() throws JSONException, Exception { } @Test - public void testSingleContainerWrong() throws JSONException, Exception { - WebResource r = resource(); + public void testSingleContainerWrong() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -402,16 +406,17 @@ public void testSingleContainerWrong() throws JSONException, Exception { nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); try { - r.path("ws").path("v1").path("node").path("containers") + t.path("ws").path("v1").path("node").path("containers") .path("container_1234_0001_01_000005") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + .request(MediaType.APPLICATION_JSON).get(JSONObject.class); fail("should have thrown exception on invalid user query"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo()); + } catch (WebApplicationException we) { + Response response = we.getResponse(); + assertResponseStatusCode(Response.Status.NOT_FOUND, + response.getStatusInfo()); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - JSONObject msg = response.getEntity(JSONObject.class); + response.getMediaType().toString()); + JSONObject msg = response.readEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); assertEquals("incorrect number of elements", 3, exception.length()); String message = exception.getString("message"); @@ -430,23 +435,23 @@ public void testSingleContainerWrong() throws JSONException, Exception { } @Test - public void testNodeSingleContainerXML() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeSingleContainerXML() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); HashMap hash = addAppContainers(app); Application app2 = new MockApp(2); nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); - client().addFilter(new LoggingFilter(System.out)); + client().register(new LoggingFeature(LOG)); for (String id : hash.keySet()) { - ClientResponse response = r.path("ws").path("v1").path("node") - .path("containers").path(id).accept(MediaType.APPLICATION_XML) - .get(ClientResponse.class); + Response response = t.path("ws").path("v1").path("node") + .path("containers").path(id).request(MediaType.APPLICATION_XML) + .get(Response.class); assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - String xml = response.getEntity(String.class); + response.getMediaType().toString()); + String xml = response.readEntity(String.class); DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); InputSource is = new InputSource(); @@ -461,8 +466,8 @@ public void testNodeSingleContainerXML() throws JSONException, Exception { } @Test - public void testNodeContainerXML() throws JSONException, Exception { - WebResource r = resource(); + public void testNodeContainerXML() throws Exception { + WebTarget t = target(); Application app = new MockApp(1); nmContext.getApplications().put(app.getAppId(), app); addAppContainers(app); @@ -470,12 +475,12 @@ public void testNodeContainerXML() throws JSONException, Exception { nmContext.getApplications().put(app2.getAppId(), app2); addAppContainers(app2); - ClientResponse response = r.path("ws").path("v1").path("node") - .path("containers").accept(MediaType.APPLICATION_XML) - .get(ClientResponse.class); + Response response = t.path("ws").path("v1").path("node") + .path("containers").request(MediaType.APPLICATION_XML) + .get(Response.class); assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, - response.getType().toString()); - String xml = response.getEntity(String.class); + response.getMediaType().toString()); + String xml = response.readEntity(String.class); DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); InputSource is = new InputSource(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebTerminal.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebTerminal.java index d4180e4825186..a9c93f0db6836 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebTerminal.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebTerminal.java @@ -22,7 +22,10 @@ import java.io.File; import java.io.IOException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.JettyUtils; @@ -37,10 +40,6 @@ import org.junit.Before; import org.junit.Test; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource.Builder; - /** * Unit test for hosting web terminal servlet in node manager. */ @@ -110,11 +109,11 @@ public void tearDown() throws IOException { @Test public void testWebTerminal() { - Client client = Client.create(); - Builder builder = client.resource("http://127.0.0.1:" + port + - "/terminal/terminal.template").accept("text/html"); - ClientResponse response = builder.get(ClientResponse.class); + Client client = ClientBuilder.newClient(); + Response response = client.target("http://127.0.0.1:" + port + + "/terminal/terminal.template").request("text/html") + .get(Response.class); assertEquals(MediaType.TEXT_HTML + "; " + JettyUtils.UTF_8, - response.getType().toString()); + response.getMediaType().toString()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index 3ed53f6dcc669..d0d874d695ebd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -34,8 +34,6 @@ import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.YarnWebParams; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; - /** * The RM webapp */