diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 208345d5f5a53..3e18dc421bd02 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -406,69 +406,6 @@ - - com.sun.jersey - jersey-core - true - - - javax.ws.rs - jsr311-api - - - - - com.sun.jersey - jersey-client - true - - - com.github.pjfanning - jersey-json - true - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - - javax.xml.bind - jaxb-api - - - - - com.sun.jersey - jersey-server - true - - - com.sun.jersey - jersey-servlet - true - - - javax.servlet - servlet-api - - - javax.enterprise - cdi-api - - - ch.qos.cal10n - cal10n-api - - - net.sf.kosmosfs @@ -611,24 +548,6 @@ - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - true - - - - org.glassfish - javax.servlet - - - - - - com.sun.jersey.contribs - jersey-guice - true - diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 41707f5002414..c11210282797e 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -132,49 +132,6 @@ jsp-api runtime - - com.sun.jersey - jersey-core - compile - - - com.sun.jersey - jersey-servlet - compile - - - javax.enterprise - cdi-api - - - javax.servlet - servlet-api - - - ch.qos.cal10n - cal10n-api - - - - - com.github.pjfanning - jersey-json - compile - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - - - 1.19.4 + 2.39.1 - 2.12.7 - 2.12.7.1 + 2.15.2 + 2.15.2 + + + 2.1.1 4.5.13 @@ -828,8 +831,13 @@ javax.ws.rs - jsr311-api - 1.1.1 + javax.ws.rs-api + ${javax.ws.rs-api.version} + + + net.jodah + failsafe + 2.4.4 org.eclipse.jetty @@ -915,46 +923,57 @@ 43.0 - com.sun.jersey - jersey-core - ${jersey.version} - - - org.osgi - org.osgi.core - - + org.glassfish.jersey.core + jersey-common + ${jersey2.version} - com.sun.jersey - jersey-servlet - ${jersey.version} + org.glassfish.jersey.core + jersey-server + ${jersey2.version} - com.github.pjfanning - jersey-json - 1.20 - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - + org.glassfish.jersey.core + jersey-client + ${jersey2.version} - com.sun.jersey - jersey-server - ${jersey.version} + org.glassfish.jersey.containers + jersey-container-servlet-core + ${jersey2.version} + + + org.glassfish.jersey.inject + jersey-hk2 + ${jersey2.version} + + + org.glassfish.jersey.media + jersey-media-json-jackson + ${jersey2.version} + + + org.glassfish.jersey.media + jersey-media-json-jettison + ${jersey2.version} + + + org.glassfish.jersey.test-framework + jersey-test-framework-core + ${jersey2.version} + + + org.glassfish.jersey.test-framework.providers + jersey-test-framework-provider-grizzly2 + ${jersey2.version} + + + org.glassfish.hk2 + guice-bridge + 2.6.1 + com.google.inject guice @@ -973,36 +992,6 @@ ${guice.version} - - com.sun.jersey.contribs - jersey-guice - ${jersey.version} - - - - com.sun.jersey.jersey-test-framework - jersey-test-framework-core - ${jersey.version} - test - - - javax.servlet - javax.servlet-api - - - - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - ${jersey.version} - - - javax.servlet - javax.servlet-api - - - - io.netty netty-all @@ -1528,11 +1517,6 @@ - - com.sun.jersey - jersey-client - ${jersey.version} - ${leveldbjni.group} @@ -1906,6 +1890,10 @@ org.osgi org.osgi.core + + javax.ws.rs + jsr311-api + diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index e8c5fb78efd8d..f5964e312f88d 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -312,12 +312,6 @@ test - - javax.ws.rs - jsr311-api - test - - org.mockito mockito-core diff --git a/hadoop-tools/hadoop-resourceestimator/pom.xml b/hadoop-tools/hadoop-resourceestimator/pom.xml index 89e248f9a7efe..c46d7f22f6b56 100644 --- a/hadoop-tools/hadoop-resourceestimator/pom.xml +++ b/hadoop-tools/hadoop-resourceestimator/pom.xml @@ -65,37 +65,10 @@ javax.inject 1 - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - test - javax.servlet javax.servlet-api - - com.sun.jersey - jersey-server - - - com.github.pjfanning - jersey-json - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - - junit junit diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml index 3acd9ce0ea888..4b5e2186e3cfe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml @@ -107,25 +107,6 @@ test - - com.github.pjfanning - jersey-json - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - - - org.apache.solr solr-solrj diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/webapp/WEB-INF/web.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/webapp/WEB-INF/web.xml index d9b17ae590543..47b59d707cd43 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/webapp/WEB-INF/web.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/webapp/WEB-INF/web.xml @@ -74,15 +74,11 @@ REST_API - com.sun.jersey.spi.container.servlet.ServletContainer + org.glassfish.jersey.servlet.ServletContainer - com.sun.jersey.config.property.packages + jersey.config.server.provider.packages org.apache.hadoop.yarn.appcatalog.controller;com.wordnik.swagger.jaxrs.listing;com.wordnik.swagger.jaxrs.json - - com.sun.jersey.api.json.POJOMappingFeature - true - 1 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml index dbe0c69d5508b..c3ae8f5a0853e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml @@ -131,10 +131,6 @@ com.google.inject guice - - javax.ws.rs - jsr311-api - javax.servlet javax.servlet-api @@ -155,10 +151,6 @@ org.apache.hadoop.thirdparty hadoop-shaded-guava - - com.sun.jersey - jersey-client - org.eclipse.jetty jetty-server diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml index 1282c9f863565..9be06d1724613 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml @@ -18,15 +18,11 @@ Jersey REST API - com.sun.jersey.spi.container.servlet.ServletContainer + org.glassfish.jersey.servlet.ServletContainer - com.sun.jersey.config.property.packages + jersey.config.server.provider.packages org.apache.hadoop.yarn.service.webapp,org.apache.hadoop.yarn.service.api,org.apache.hadoop.yarn.service.api.records - - com.sun.jersey.api.json.POJOMappingFeature - true - 1 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index dbe4e9048b2df..afee12241ff1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -84,16 +84,6 @@ mockito-core test - - com.sun.jersey.jersey-test-framework - jersey-test-framework-core - test - - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - test - org.apache.hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index 9b4f4fd47bf95..1f2e34f0c15c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -76,14 +76,6 @@ org.eclipse.jetty jetty-util - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-client - org.apache.hadoop.thirdparty hadoop-shaded-guava @@ -158,48 +150,17 @@ junit-platform-launcher test - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - test - commons-io commons-io - com.google.inject - guice - - - com.sun.jersey - jersey-server + net.jodah + failsafe - com.github.pjfanning - jersey-json - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - - org.codehaus.jettison - jettison - - - - - com.sun.jersey.contribs - jersey-guice + com.google.inject + guice log4j @@ -221,6 +182,34 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + org.glassfish.jersey.test-framework + jersey-test-framework-core + + + org.glassfish.jersey.test-framework.providers + jersey-test-framework-provider-grizzly2 + + + org.glassfish.hk2 + guice-bridge + + + org.glassfish.jersey.containers + jersey-container-servlet-core + + + org.glassfish.jersey.inject + jersey-hk2 + + + org.glassfish.jersey.media + jersey-media-json-jackson + + + org.glassfish.jersey.media + jersey-media-json-jettison + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/DirectTimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/DirectTimelineWriter.java index 7fea1dbcdbea5..ce0bb760835d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/DirectTimelineWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/DirectTimelineWriter.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.net.URI; +import javax.ws.rs.client.Client; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -33,8 +35,6 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.exceptions.YarnException; -import com.sun.jersey.api.client.Client; - /** * A simple writer class for storing Timeline data into Leveldb store. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java index b92f4e412347c..24a0854952e47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java @@ -38,6 +38,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import javax.ws.rs.client.Client; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -67,7 +69,6 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.type.TypeFactory; import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; -import com.sun.jersey.api.client.Client; /** * A simple writer class for storing Timeline data in any storage that diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 2b9ce4fa8f2ad..d519ed11e71ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -24,6 +24,8 @@ import java.net.URI; import java.security.PrivilegedExceptionAction; +import javax.ws.rs.client.Client; + import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; @@ -54,7 +56,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.hadoop.classification.VisibleForTesting; -import com.sun.jersey.api.client.Client; @Private @Evolving diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java index 5a216d20d35d6..458d1bfd81413 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.client.api.impl; import java.io.IOException; -import java.io.InterruptedIOException; import java.lang.reflect.UndeclaredThrowableException; import java.net.ConnectException; import java.net.HttpURLConnection; @@ -30,13 +29,22 @@ import java.net.URLConnection; import java.security.GeneralSecurityException; import java.security.PrivilegedExceptionAction; +import java.time.Duration; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLSocketFactory; +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; + +import net.jodah.failsafe.Failsafe; +import net.jodah.failsafe.RetryPolicy; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.HttpUrlConnectorProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -58,15 +66,6 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import org.apache.hadoop.util.Preconditions; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientRequest; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.api.client.filter.ClientFilter; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; /** * Utility Connector class which is used by timeline clients to securely get @@ -87,9 +86,8 @@ public class TimelineConnector extends AbstractService { private DelegationTokenAuthenticatedURL.Token token; private UserGroupInformation authUgi; private String doAsUser; - @VisibleForTesting - TimelineClientConnectionRetry connectionRetry; private boolean requireConnectionRetry; + private RetryPolicy retryPolicy; public TimelineConnector(boolean requireConnectionRetry, UserGroupInformation authUgi, String doAsUser, @@ -104,8 +102,8 @@ public TimelineConnector(boolean requireConnectionRetry, @Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); - ClientConfig cc = new DefaultClientConfig(); - cc.getClasses().add(YarnJacksonJaxbJsonProvider.class); + ClientConfig clientConfig = new ClientConfig(); + clientConfig.register(YarnJacksonJaxbJsonProvider.class); if (YarnConfiguration.useHttps(conf)) { // If https is chosen, configures SSL client. @@ -126,17 +124,11 @@ protected void serviceInit(Configuration conf) throws Exception { } authenticator.setConnectionConfigurator(connConfigurator); - connectionRetry = new TimelineClientConnectionRetry(conf); - client = - new Client( - new URLConnectionClientHandler(new TimelineURLConnectionFactory( - authUgi, authenticator, connConfigurator, token, doAsUser)), - cc); - if (requireConnectionRetry) { - TimelineJerseyRetryFilter retryFilter = - new TimelineJerseyRetryFilter(connectionRetry); - client.addFilter(retryFilter); - } + retryPolicy = createRetryPolicy(conf); + clientConfig.connectorProvider(new HttpUrlConnectorProvider().connectionFactory( + new TimelineURLConnectionFactory( + authUgi, authenticator, connConfigurator, token, doAsUser))); + client = ClientBuilder.newClient(clientConfig); } private static final ConnectionConfigurator DEFAULT_TIMEOUT_CONN_CONFIGURATOR @@ -208,7 +200,7 @@ DelegationTokenAuthenticatedURL getDelegationTokenAuthenticatedURL() { protected void serviceStop() { if (this.client != null) { - this.client.destroy(); + this.client.close(); } if (this.sslFactory != null) { this.sslFactory.destroy(); @@ -226,7 +218,7 @@ public Object operateDelegationToken( TimelineClientRetryOp tokenRetryOp = createRetryOpForOperateDelegationToken(action); - return connectionRetry.retryOn(tokenRetryOp); + return Failsafe.with(retryPolicy).get(tokenRetryOp::run); } @Private @@ -245,13 +237,10 @@ TimelineClientRetryOp createRetryOpForOperateDelegationToken( public static abstract class TimelineClientRetryOp { // The operation that should be retried public abstract Object run() throws IOException; - - // The method to indicate if we should retry given the incoming exception - public abstract boolean shouldRetryOn(Exception e); } private static class TimelineURLConnectionFactory - implements HttpURLConnectionFactory { + implements HttpUrlConnectorProvider.ConnectionFactory { private DelegationTokenAuthenticator authenticator; private UserGroupInformation authUgi; private ConnectionConfigurator connConfigurator; @@ -270,8 +259,7 @@ public TimelineURLConnectionFactory(UserGroupInformation authUgi, } @Override - public HttpURLConnection getHttpURLConnection(final URL url) - throws IOException { + public HttpURLConnection getConnection(URL url) throws IOException { authUgi.checkTGTAndReloginFromKeytab(); try { return new DelegationTokenAuthenticatedURL(authenticator, @@ -282,143 +270,42 @@ public HttpURLConnection getHttpURLConnection(final URL url) throw new IOException(ae); } } - } - // Class to handle retry - // Outside this class, only visible to tests - @Private - @VisibleForTesting - static class TimelineClientConnectionRetry { - - // maxRetries < 0 means keep trying - @Private - @VisibleForTesting - public int maxRetries; - - @Private - @VisibleForTesting - public long retryInterval; - - // Indicates if retries happened last time. Only tests should read it. - // In unit tests, retryOn() calls should _not_ be concurrent. - private boolean retried = false; - - @Private - @VisibleForTesting - boolean getRetired() { - return retried; - } - - // Constructor with default retry settings - public TimelineClientConnectionRetry(Configuration conf) { - Preconditions.checkArgument( - conf.getInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES) - >= -1, - "%s property value should be greater than or equal to -1", - YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES); - Preconditions.checkArgument( - conf.getLong( - YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, - YarnConfiguration. + private RetryPolicy createRetryPolicy(Configuration conf) { + Preconditions.checkArgument( + conf.getInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES) + >= -1, + "%s property value should be greater than or equal to -1", + YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES); + Preconditions.checkArgument( + conf.getLong( + YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, + YarnConfiguration. DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS) > 0, - "%s property value should be greater than zero", - YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS); - maxRetries = - conf.getInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES); - retryInterval = conf.getLong( - YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS); - } - - public Object retryOn(TimelineClientRetryOp op) - throws RuntimeException, IOException { - int leftRetries = maxRetries; - retried = false; - - // keep trying - while (true) { - try { - // try perform the op, if fail, keep retrying - return op.run(); - } catch (IOException | RuntimeException e) { - // break if there's no retries left - if (leftRetries == 0) { - break; - } - if (op.shouldRetryOn(e)) { - logException(e, leftRetries); - } else { - throw e; - } - } - if (leftRetries > 0) { - leftRetries--; - } - retried = true; - try { - // sleep for the given time interval - Thread.sleep(retryInterval); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new InterruptedIOException("Client retry sleep interrupted!"); - } - } - throw new RuntimeException("Failed to connect to timeline server. " - + "Connection retries limit exceeded. " - + "The posted timeline event may be missing"); - }; - - private void logException(Exception e, int leftRetries) { - if (leftRetries > 0) { - LOG.info( - "Exception caught by TimelineClientConnectionRetry," + " will try " - + leftRetries + " more time(s).\nMessage: " + e.getMessage()); - } else { - // note that maxRetries may be -1 at the very beginning - LOG.info("ConnectionException caught by TimelineClientConnectionRetry," - + " will keep retrying.\nMessage: " + e.getMessage()); - } + "%s property value should be greater than zero", + YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS); + int maxRetries = 0; + if (requireConnectionRetry) { + maxRetries = conf.getInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES); } + long retryInterval = conf.getLong( + YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS); + return retryPolicy = new RetryPolicy<>() + .handle(IOException.class, RuntimeException.class) + .handleIf(e -> e instanceof ProcessingException + && (e.getCause() instanceof ConnectException + || e.getCause() instanceof SocketTimeoutException + || e.getCause() instanceof SocketException)) + .withDelay(Duration.ofMillis(retryInterval)) + .withMaxRetries(maxRetries); } - private static class TimelineJerseyRetryFilter extends ClientFilter { - private TimelineClientConnectionRetry connectionRetry; - - public TimelineJerseyRetryFilter( - TimelineClientConnectionRetry connectionRetry) { - this.connectionRetry = connectionRetry; - } - - @Override - public ClientResponse handle(final ClientRequest cr) - throws ClientHandlerException { - // Set up the retry operation - TimelineClientRetryOp jerseyRetryOp = new TimelineClientRetryOp() { - @Override - public Object run() { - // Try pass the request, if fail, keep retrying - return getNext().handle(cr); - } - - @Override - public boolean shouldRetryOn(Exception e) { - // Only retry on connection exceptions - return (e instanceof ClientHandlerException) - && (e.getCause() instanceof ConnectException - || e.getCause() instanceof SocketTimeoutException - || e.getCause() instanceof SocketException); - } - }; - try { - return (ClientResponse) connectionRetry.retryOn(jerseyRetryOp); - } catch (IOException e) { - throw new ClientHandlerException( - "Jersey retry failed!\nMessage: " + e.getMessage()); - } - } + RetryPolicy getRetryPolicy() { + return retryPolicy; } @Private @@ -447,13 +334,5 @@ public Object run() throws IOException { throw new IOException(e); } } - - @Override - public boolean shouldRetryOn(Exception e) { - // retry on connection exceptions - // and SocketTimeoutException - return (e instanceof ConnectException - || e instanceof SocketTimeoutException); - } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java index 71bf13220b016..802804fba7f5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.client.api.impl; import org.apache.hadoop.classification.VisibleForTesting; -import com.sun.jersey.core.util.MultivaluedMapImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -31,12 +30,14 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.client.api.TimelineReaderClient; -import com.sun.jersey.api.client.ClientResponse; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedHashMap; import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URI; @@ -111,12 +112,12 @@ public TimelineEntity getApplicationEntity(ApplicationId appId, String fields, if (fields == null || fields.isEmpty()) { fields = "INFO"; } - MultivaluedMap params = new MultivaluedMapImpl(); + MultivaluedMap params = new MultivaluedHashMap<>(); params.add("fields", fields); mergeFilters(params, filters); - ClientResponse response = doGetUri(baseUri, path, params); - TimelineEntity entity = response.getEntity(TimelineEntity.class); + Response response = doGetUri(baseUri, path, params); + TimelineEntity entity = response.readEntity(TimelineEntity.class); return entity; } @@ -131,12 +132,12 @@ public TimelineEntity getApplicationAttemptEntity( if (fields == null || fields.isEmpty()) { fields = "INFO"; } - MultivaluedMap params = new MultivaluedMapImpl(); + MultivaluedMap params = new MultivaluedHashMap<>(); params.add("fields", fields); mergeFilters(params, filters); - ClientResponse response = doGetUri(baseUri, path, params); - TimelineEntity entity = response.getEntity(TimelineEntity.class); + Response response = doGetUri(baseUri, path, params); + TimelineEntity entity = response.readEntity(TimelineEntity.class); return entity; } @@ -150,7 +151,7 @@ public List getApplicationAttemptEntities( if (fields == null || fields.isEmpty()) { fields = "INFO"; } - MultivaluedMap params = new MultivaluedMapImpl(); + MultivaluedMap params = new MultivaluedHashMap<>(); params.add("fields", fields); if (limit > 0) { params.add("limit", Long.toString(limit)); @@ -160,8 +161,8 @@ public List getApplicationAttemptEntities( } mergeFilters(params, filters); - ClientResponse response = doGetUri(baseUri, path, params); - TimelineEntity[] entities = response.getEntity(TimelineEntity[].class); + Response response = doGetUri(baseUri, path, params); + TimelineEntity[] entities = response.readEntity(TimelineEntity[].class); return Arrays.asList(entities); } @@ -176,12 +177,12 @@ public TimelineEntity getContainerEntity(ContainerId containerId, if (fields == null || fields.isEmpty()) { fields = "INFO"; } - MultivaluedMap params = new MultivaluedMapImpl(); + MultivaluedMap params = new MultivaluedHashMap<>(); params.add("fields", fields); mergeFilters(params, filters); - ClientResponse response = doGetUri(baseUri, path, params); - TimelineEntity entity = response.getEntity(TimelineEntity.class); + Response response = doGetUri(baseUri, path, params); + TimelineEntity entity = response.readEntity(TimelineEntity.class); return entity; } @@ -196,7 +197,7 @@ public List getContainerEntities( if (fields == null || fields.isEmpty()) { fields = "INFO"; } - MultivaluedMap params = new MultivaluedMapImpl(); + MultivaluedMap params = new MultivaluedHashMap<>(); params.add("fields", fields); if (limit > 0) { params.add("limit", Long.toString(limit)); @@ -206,8 +207,8 @@ public List getContainerEntities( } mergeFilters(params, filters); - ClientResponse response = doGetUri(baseUri, path, params); - TimelineEntity[] entity = response.getEntity(TimelineEntity[].class); + Response response = doGetUri(baseUri, path, params); + TimelineEntity[] entity = response.readEntity(TimelineEntity[].class); return Arrays.asList(entity); } @@ -232,19 +233,22 @@ private void mergeFilters(MultivaluedMap defaults, } @VisibleForTesting - protected ClientResponse doGetUri(URI base, String path, + protected Response doGetUri(URI base, String path, MultivaluedMap params) throws IOException { - ClientResponse resp = connector.getClient().resource(base).path(path) - .queryParams(params).accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - if (resp == null || - resp.getStatusInfo().getStatusCode() != ClientResponse.Status.OK - .getStatusCode()) { - String msg = - "Response from the timeline reader server is " + - ((resp == null) ? "null" : "not successful," + - " HTTP error code: " + resp.getStatus() + - ", Server response:\n" + resp.getEntity(String.class)); + WebTarget webTarget = connector.getClient() + .target(base) + .path(path); + for (Map.Entry> param : params.entrySet()) { + webTarget.queryParam(param.getKey(), param.getValue()); + } + Response resp = webTarget + .request(MediaType.APPLICATION_JSON) + .get(Response.class); + if (resp == null + || resp.getStatusInfo().getStatusCode() != Response.Status.OK.getStatusCode()) { + String msg = "Response from the timeline reader server is " + ((resp == null) ? + "null" : "not successful, HTTP error code: " + resp.getStatus() + + ", Server response:\n" + resp.readEntity(String.class)); LOG.error(msg); throw new IOException(msg); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java index ed74addd162c1..85cff9a4ae5b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java @@ -24,8 +24,9 @@ import java.net.InetSocketAddress; import java.net.URI; import java.security.PrivilegedExceptionAction; +import java.util.List; +import java.util.Map; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -34,8 +35,13 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.CancellationException; +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedHashMap; import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,10 +63,6 @@ import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.classification.VisibleForTesting; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.core.util.MultivaluedMapImpl; /** * Implementation of timeline v2 client interface. @@ -288,24 +290,26 @@ private void checkRetryWithSleep(int retries, IOException e) } } - private ClientResponse doPutObjects(URI base, String path, - MultivaluedMap params, Object obj) { - return connector.getClient().resource(base).path(path).queryParams(params) - .accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON) - .put(ClientResponse.class, obj); + private Response doPutObjects(URI base, String path, + MultivaluedMap params, + Object obj) { + WebTarget webTarget = connector.getClient() + .target(base) + .path(path); + for (Map.Entry> param : params.entrySet()) { + webTarget = webTarget.queryParam(param.getKey(), param.getValue()); + } + return webTarget.request(MediaType.APPLICATION_JSON) + .put(Entity.json(obj), Response.class); } protected void putObjects(URI base, String path, MultivaluedMap params, Object obj) throws IOException, YarnException { - ClientResponse resp = null; + Response resp; try { - resp = authUgi.doAs(new PrivilegedExceptionAction() { - @Override - public ClientResponse run() throws Exception { - return doPutObjects(base, path, params, obj); - } - }); + resp = authUgi.doAs( + (PrivilegedExceptionAction) () -> doPutObjects(base, path, params, obj)); } catch (UndeclaredThrowableException ue) { Throwable cause = ue.getCause(); if (cause instanceof IOException) { @@ -324,24 +328,19 @@ public ClientResponse run() throws Exception { String msg = "Error getting HTTP response from the timeline server."; LOG.error(msg); throw new YarnException(msg); - } else if (resp.getStatusInfo().getStatusCode() - == ClientResponse.Status.OK.getStatusCode()) { + } else if (resp.getStatusInfo().getStatusCode() == Response.Status.OK.getStatusCode()) { try { resp.close(); - } catch(ClientHandlerException che) { - LOG.warn("Error closing the HTTP response's inputstream. ", che); + } catch (ProcessingException e) { + LOG.warn("Error closing the HTTP response's inputstream. ", e); } } else { String msg = ""; try { - String stringType = resp.getEntity(String.class); + String stringType = resp.readEntity(String.class); msg = "Server response:\n" + stringType; - } catch (ClientHandlerException | UniformInterfaceException chuie) { - msg = "Error getting entity from the HTTP response." - + chuie.getLocalizedMessage(); - } catch (Throwable t) { - msg = "Error getting entity from the HTTP response." - + t.getLocalizedMessage(); + } catch (Throwable e) { + msg = "Error getting entity from the HTTP response." + e.getLocalizedMessage(); } finally { msg = "Response from the timeline server is not successful" + ", HTTP error code: " + resp.getStatus() @@ -394,16 +393,14 @@ private final class EntitiesHolder extends FutureTask { EntitiesHolder(final TimelineEntities entities, final boolean isSync, final boolean subappwrite) { - super(new Callable() { - // publishEntities() - public Void call() throws Exception { - MultivaluedMap params = new MultivaluedMapImpl(); - params.add("appid", getContextAppId().toString()); - params.add("async", Boolean.toString(!isSync)); - params.add("subappwrite", Boolean.toString(subappwrite)); - putObjects("entities", params, entities); - return null; - } + // publishEntities() + super(() -> { + MultivaluedMap params = new MultivaluedHashMap<>(); + params.add("appid", getContextAppId().toString()); + params.add("async", Boolean.toString(!isSync)); + params.add("subappwrite", Boolean.toString(subappwrite)); + putObjects("entities", params, entities); + return null; }); this.entities = entities; this.isSync = isSync; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineWriter.java index 957501cb48632..faecb56a22e7d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineWriter.java @@ -24,7 +24,11 @@ import java.lang.reflect.UndeclaredThrowableException; import java.net.URI; import java.security.PrivilegedExceptionAction; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,9 +45,6 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.classification.VisibleForTesting; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; /** * Base writer class to write the Timeline data. @@ -89,8 +90,8 @@ public TimelinePutResponse putEntities( } entitiesContainer.addEntity(entity); } - ClientResponse resp = doPosting(entitiesContainer, null); - return resp.getEntity(TimelinePutResponse.class); + Response resp = doPosting(entitiesContainer, null); + return resp.readEntity(TimelinePutResponse.class); } public void putDomain(TimelineDomain domain) throws IOException, @@ -105,16 +106,11 @@ public abstract TimelinePutResponse putEntities( public abstract void putDomain(ApplicationAttemptId appAttemptId, TimelineDomain domain) throws IOException, YarnException; - private ClientResponse doPosting(final Object obj, final String path) + private Response doPosting(final Object obj, final String path) throws IOException, YarnException { - ClientResponse resp; + Response resp; try { - resp = authUgi.doAs(new PrivilegedExceptionAction() { - @Override - public ClientResponse run() throws Exception { - return doPostingObject(obj, path); - } - }); + resp = authUgi.doAs((PrivilegedExceptionAction) () -> doPostingObject(obj, path)); } catch (UndeclaredThrowableException e) { Throwable cause = e.getCause(); if (cause instanceof IOException) { @@ -125,16 +121,15 @@ public ClientResponse run() throws Exception { } catch (InterruptedException ie) { throw (IOException)new InterruptedIOException().initCause(ie); } - if (resp == null || - resp.getStatusInfo().getStatusCode() - != ClientResponse.Status.OK.getStatusCode()) { + if (resp == null + || resp.getStatusInfo().getStatusCode() != Response.Status.OK.getStatusCode()) { String msg = "Failed to get the response from the timeline server."; LOG.error(msg); if (resp != null) { msg += " HTTP error code: " + resp.getStatus(); - LOG.debug("HTTP error code: {} Server response : \n{}", - resp.getStatus(), resp.getEntity(String.class)); + LOG.debug("HTTP error code: {} Server response : \n{}", resp.getStatus(), + resp.readEntity(String.class)); } throw new YarnException(msg); } @@ -143,20 +138,18 @@ public ClientResponse run() throws Exception { @Private @VisibleForTesting - public ClientResponse doPostingObject(Object object, String path) { - WebResource webResource = client.resource(resURI); + public Response doPostingObject(Object object, String path) { + WebTarget webTarget = client.target(resURI); if (path == null) { LOG.debug("POST to {}", resURI); - ClientResponse r = webResource.accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON) - .post(ClientResponse.class, object); + Response r = webTarget.request(MediaType.APPLICATION_JSON) + .post(Entity.json(object), Response.class); r.bufferEntity(); return r; } else if (path.equals("domain")) { LOG.debug("PUT to {}/{}", resURI, path); - ClientResponse r = webResource.path(path).accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON) - .put(ClientResponse.class, object); + Response r = webTarget.path(path).request(MediaType.APPLICATION_JSON) + .put(Entity.json(object), Response.class); r.bufferEntity(); return r; } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java index 3c56b0290d74e..985935bfbb0ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java @@ -29,15 +29,15 @@ import java.nio.file.Files; import java.nio.file.Paths; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; /** * This class contains several utility function which could be used in different @@ -201,15 +201,16 @@ public static PrintStream createPrintStream(String localDir, String nodeId, * @param logFile name of the log file * @return response from NMWebServices */ - public static ClientResponse getResponseFromNMWebService(Configuration conf, + public static Response getResponseFromNMWebService(Configuration conf, Client webServiceClient, ContainerLogsRequest request, String logFile) { - WebResource webResource = - webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf) + WebTarget webResource = + webServiceClient.target(WebAppUtils.getHttpSchemePrefix(conf) + request.getNodeHttpAddress()); return webResource.path("ws").path("v1").path("node") .path("containers").path(request.getContainerId()).path("logs") .path(logFile) .queryParam("size", Long.toString(request.getBytes())) - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + .request(MediaType.TEXT_PLAIN) + .get(Response.class); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java index 7cb6018e92ae1..c2f26104bd4a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java @@ -58,8 +58,8 @@ public Response toResponse(Exception e) { // Don't catch this as filter forward on 404 // (ServletContainer.FEATURE_FILTER_FORWARD_ON_404) // won't work and the web UI won't work! - if (e instanceof com.sun.jersey.api.NotFoundException) { - return ((com.sun.jersey.api.NotFoundException) e).getResponse(); + if (e instanceof javax.ws.rs.NotFoundException) { + return ((javax.ws.rs.NotFoundException) e).getResponse(); } // clear content type response.setContentType(null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java index 6ef1c50cc6df5..2d3a064c4163a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java @@ -22,9 +22,9 @@ import java.net.InetSocketAddress; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; + +import javax.servlet.Filter; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; @@ -34,6 +34,7 @@ import org.apache.hadoop.util.Lists; import org.apache.hadoop.yarn.webapp.view.RobotsTextPage; +import com.google.inject.Scopes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,11 +43,6 @@ import com.google.inject.Provides; import com.google.inject.servlet.GuiceFilter; import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.container.filter.GZIPContentEncodingFilter; -import com.sun.jersey.api.core.ResourceConfig; -import com.sun.jersey.core.util.FeaturesAndProperties; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.spi.container.servlet.ServletContainer; /** * @see WebApps for a usage example @@ -185,18 +181,12 @@ protected void configureWebAppServlets() { String regex = "(?!/" + this.wsName + ")"; serveRegex(regex).with(DefaultWrapperServlet.class); - Map params = new HashMap(); - params.put(ResourceConfig.FEATURE_IMPLICIT_VIEWABLES, "true"); - params.put(ServletContainer.FEATURE_FILTER_FORWARD_ON_404, "true"); - params.put(FeaturesAndProperties.FEATURE_XMLROOTELEMENT_PROCESSING, "true"); - params.put(ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS, GZIPContentEncodingFilter.class.getName()); - params.put(ResourceConfig.PROPERTY_CONTAINER_RESPONSE_FILTERS, GZIPContentEncodingFilter.class.getName()); - filter("/*").through(getWebAppFilterClass(), params); + bind(getWebAppFilterClass()).in(Scopes.SINGLETON); } } - protected Class getWebAppFilterClass() { - return GuiceContainer.class; + protected Class getWebAppFilterClass() { + return GuiceFilter.class; } /** @@ -231,6 +221,10 @@ public void route(String pathSpec, Class cls, route(HTTP.GET, pathSpec, cls, action); } + public void addJerseyResourceConfigs(HttpServer2 server) { + + } + public void route(String pathSpec, Class cls) { List res = parseRoute(pathSpec); router.add(HTTP.GET, res.get(R_PATH), cls, res.get(R_ACTION), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java index 9fef076196e4e..6c18cb8de0ec6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java @@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.eclipse.jetty.webapp.WebAppContext; +import org.glassfish.jersey.servlet.ServletProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -407,8 +408,13 @@ public void setup() { new String[] {"/*"}); } + final Map guiceFilterParams = new HashMap<>(); + guiceFilterParams.put(ServletProperties.FILTER_FORWARD_ON_404, "true"); HttpServer2.defineFilter(server.getWebAppContext(), "guice", - GuiceFilter.class.getName(), null, new String[] { "/*" }); + GuiceFilter.class.getName(), + guiceFilterParams, new String[] {"/*"}); + + webapp.addJerseyResourceConfigs(server); webapp.setConf(conf); webapp.setHttpServer(server); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebServiceClient.java index 39cc2e361f1b2..fe4046e28b3eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebServiceClient.java @@ -19,7 +19,12 @@ import java.io.IOException; import java.net.HttpURLConnection; -import java.net.URL; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; + +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.HttpUrlConnectorProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authentication.client.AuthenticatedURL; @@ -29,9 +34,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.classification.VisibleForTesting; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; /** * Utility for handling Web client. @@ -92,33 +94,30 @@ private static SSLFactory createSSLFactory(Configuration conf) * @return Client */ public Client createClient() { - return new Client( - new URLConnectionClientHandler(getHttpURLConnectionFactory())); + ClientConfig clientConfig = new ClientConfig(); + clientConfig.connectorProvider(getHttpURLConnectionFactory()); + return ClientBuilder.newClient(clientConfig); } @VisibleForTesting - protected HttpURLConnectionFactory getHttpURLConnectionFactory() { - return new HttpURLConnectionFactory() { - @Override - public HttpURLConnection getHttpURLConnection(URL url) - throws IOException { - AuthenticatedURL.Token token = new AuthenticatedURL.Token(); - HttpURLConnection conn = null; - try { - HttpURLConnection.setFollowRedirects(false); - // If https is chosen, configures SSL client. - if (isHttps) { - conn = new AuthenticatedURL(new KerberosAuthenticator(), - sslFactory).openConnection(url, token); - } else { - conn = new AuthenticatedURL().openConnection(url, token); - } - } catch (AuthenticationException e) { - throw new IOException(e); + protected HttpUrlConnectorProvider getHttpURLConnectionFactory() { + return new HttpUrlConnectorProvider().connectionFactory(url -> { + AuthenticatedURL.Token token = new AuthenticatedURL.Token(); + HttpURLConnection conn; + try { + HttpURLConnection.setFollowRedirects(false); + // If https is chosen, configures SSL client. + if (isHttps) { + conn = new AuthenticatedURL(new KerberosAuthenticator(), sslFactory) + .openConnection(url, token); + } else { + conn = new AuthenticatedURL().openConnection(url, token); } - return conn; + } catch (AuthenticationException e) { + throw new IOException(e); } - }; + return conn; + }); } public synchronized static void destroy() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java index fccb3e1415f8e..8023b060b12b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java @@ -17,19 +17,16 @@ */ package org.apache.hadoop.yarn.webapp.util; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource.Builder; +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; -import com.sun.jersey.api.json.JSONJAXBContext; -import com.sun.jersey.api.json.JSONMarshaller; import org.apache.hadoop.conf.Configuration; -import org.codehaus.jettison.json.JSONObject; -import java.io.StringWriter; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.codehaus.jettison.json.JSONObject; /** * This class contains several utility function which could be used to generate @@ -40,57 +37,42 @@ public final class YarnWebServiceUtils { private YarnWebServiceUtils() {} + private static final ObjectMapper MAPPER = new ObjectMapper(); + /** * Utility function to get NodeInfo by calling RM WebService. * @param conf the configuration * @param nodeId the nodeId * @return a JSONObject which contains the NodeInfo - * @throws ClientHandlerException if there is an error - * processing the response. - * @throws UniformInterfaceException if the response status - * is 204 (No Content). */ - public static JSONObject getNodeInfoFromRMWebService(Configuration conf, - String nodeId) throws ClientHandlerException, - UniformInterfaceException { + public static JSONObject getNodeInfoFromRMWebService(Configuration conf, String nodeId) { try { - return WebAppUtils.execOnActiveRM(conf, - YarnWebServiceUtils::getNodeInfoFromRM, nodeId); + return WebAppUtils.execOnActiveRM(conf, YarnWebServiceUtils::getNodeInfoFromRM, nodeId); + } catch (ProcessingException | IllegalStateException e) { + throw e; } catch (Exception e) { - if (e instanceof ClientHandlerException) { - throw ((ClientHandlerException) e); - } else if (e instanceof UniformInterfaceException) { - throw ((UniformInterfaceException) e); - } else { - throw new RuntimeException(e); - } + throw new RuntimeException(e); } } - private static JSONObject getNodeInfoFromRM(String webAppAddress, - String nodeId) throws ClientHandlerException, UniformInterfaceException { - Client webServiceClient = Client.create(); - ClientResponse response = null; - try { - Builder builder = webServiceClient.resource(webAppAddress) - .path("ws").path("v1").path("cluster") - .path("nodes").path(nodeId).accept(MediaType.APPLICATION_JSON); - response = builder.get(ClientResponse.class); - return response.getEntity(JSONObject.class); + private static JSONObject getNodeInfoFromRM(String webAppAddress, String nodeId) { + Client webServiceClient = ClientBuilder.newClient(); + try (Response response = webServiceClient.target(webAppAddress) + .path("ws") + .path("v1") + .path("cluster") + .path("nodes") + .path(nodeId) + .request(MediaType.APPLICATION_JSON) + .get(Response.class)) { + return response.readEntity(JSONObject.class); } finally { - if (response != null) { - response.close(); - } - webServiceClient.destroy(); + webServiceClient.close(); } } @SuppressWarnings("rawtypes") public static String toJson(Object nsli, Class klass) throws Exception { - StringWriter sw = new StringWriter(); - JSONJAXBContext ctx = new JSONJAXBContext(klass); - JSONMarshaller jm = ctx.createJSONMarshaller(); - jm.marshallToJSON(nsli, sw); - return sw.toString(); + return MAPPER.writerFor(klass).writeValueAsString(nsli); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index 4b9b7c5f50308..89c2476922811 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -24,9 +24,11 @@ import java.net.URI; import java.security.PrivilegedExceptionAction; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientResponse; +import javax.ws.rs.ProcessingException; +import javax.ws.rs.client.Client; +import javax.ws.rs.core.Response; + +import net.jodah.failsafe.FailsafeException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -92,7 +94,7 @@ public void tearDown() throws Exception { @Test void testPostEntities() throws Exception { - mockEntityClientResponse(spyTimelineWriter, ClientResponse.Status.OK, + mockEntityClientResponse(spyTimelineWriter, Response.Status.OK, false, false); try { TimelinePutResponse response = client.putEntities(generateEntity()); @@ -104,7 +106,7 @@ void testPostEntities() throws Exception { @Test void testPostEntitiesWithError() throws Exception { - mockEntityClientResponse(spyTimelineWriter, ClientResponse.Status.OK, true, + mockEntityClientResponse(spyTimelineWriter, Response.Status.OK, true, false); try { TimelinePutResponse response = client.putEntities(generateEntity()); @@ -132,7 +134,7 @@ void testPostIncompleteEntities() throws Exception { @Test void testPostEntitiesNoResponse() throws Exception { mockEntityClientResponse(spyTimelineWriter, - ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false); + Response.Status.INTERNAL_SERVER_ERROR, false, false); try { client.putEntities(generateEntity()); fail("Exception is expected"); @@ -149,13 +151,13 @@ void testPostEntitiesConnectionRefused() throws Exception { client.putEntities(generateEntity()); fail("RuntimeException is expected"); } catch (RuntimeException re) { - assertTrue(re instanceof ClientHandlerException); + assertTrue(re instanceof ProcessingException); } } @Test void testPutDomain() throws Exception { - mockDomainClientResponse(spyTimelineWriter, ClientResponse.Status.OK, false); + mockDomainClientResponse(spyTimelineWriter, Response.Status.OK, false); try { client.putDomain(generateDomain()); } catch (YarnException e) { @@ -166,7 +168,7 @@ void testPutDomain() throws Exception { @Test void testPutDomainNoResponse() throws Exception { mockDomainClientResponse(spyTimelineWriter, - ClientResponse.Status.FORBIDDEN, false); + Response.Status.FORBIDDEN, false); try { client.putDomain(generateDomain()); fail("Exception is expected"); @@ -183,7 +185,7 @@ void testPutDomainConnectionRefused() throws Exception { client.putDomain(generateDomain()); fail("RuntimeException is expected"); } catch (RuntimeException re) { - assertTrue(re instanceof ClientHandlerException); + assertTrue(re instanceof ProcessingException); } } @@ -227,12 +229,7 @@ void testCheckRetryCount() throws Exception { fail("Exception expected! " + "Timeline server should be off to run this test. "); } catch (RuntimeException ce) { - assertTrue( - ce.getMessage().contains("Connection retries limit exceeded"), - "Handler exception for reason other than retry: " + ce.getMessage()); - // we would expect this exception here, check if the client has retried - assertTrue(client.connector.connectionRetry.getRetired(), - "Retry filter didn't perform any retries! "); + assertException(ce); } } @@ -267,7 +264,7 @@ void testDelegationTokenOperationsRetry() throws Exception { UserGroupInformation.getCurrentUser().getShortUserName()); assertFail(); } catch (RuntimeException ce) { - assertException(client, ce); + assertException(ce); } try { @@ -282,7 +279,7 @@ void testDelegationTokenOperationsRetry() throws Exception { new Text("0.0.0.0:8188"))); assertFail(); } catch (RuntimeException ce) { - assertException(client, ce); + assertException(ce); } try { @@ -297,7 +294,7 @@ void testDelegationTokenOperationsRetry() throws Exception { new Text("0.0.0.0:8188"))); assertFail(); } catch (RuntimeException ce) { - assertException(client, ce); + assertException(ce); } // Test DelegationTokenOperationsRetry on SocketTimeoutException @@ -312,7 +309,7 @@ void testDelegationTokenOperationsRetry() throws Exception { new Text("0.0.0.0:8188"))); assertFail(); } catch (RuntimeException ce) { - assertException(clientFake, ce); + assertException(ce); } } finally { client.stop(); @@ -364,22 +361,21 @@ private static void assertFail() { + "Timeline server should be off to run this test."); } - private void assertException(TimelineClientImpl client, RuntimeException ce) { - assertTrue(ce.getMessage().contains("Connection retries limit exceeded"), - "Handler exception for reason other than retry: " + ce.toString()); - // we would expect this exception here, check if the client has retried - assertTrue(client.connector.connectionRetry.getRetired(), - "Retry filter didn't perform any retries! "); + private void assertException(RuntimeException ce) { + assertTrue((ce instanceof FailsafeException || ce instanceof ProcessingException) && ( + ce.getCause() instanceof ConnectException + || ce.getCause() instanceof SocketTimeoutException), + "Cause should be of type connection exception"); } - public static ClientResponse mockEntityClientResponse( - TimelineWriter spyTimelineWriter, ClientResponse.Status status, - boolean hasError, boolean hasRuntimeError) { - ClientResponse response = mock(ClientResponse.class); + public static Response mockEntityClientResponse(TimelineWriter spyTimelineWriter, + Response.Status status, boolean hasError, + boolean hasRuntimeError) { + Response response = mock(Response.class); if (hasRuntimeError) { - doThrow(new ClientHandlerException(new ConnectException())).when( - spyTimelineWriter).doPostingObject( - any(TimelineEntities.class), any()); + doThrow(new ProcessingException(new ConnectException())) + .when(spyTimelineWriter) + .doPostingObject(any(TimelineEntities.class), any()); return response; } doReturn(response).when(spyTimelineWriter) @@ -394,18 +390,18 @@ public static ClientResponse mockEntityClientResponse( if (hasError) { putResponse.addError(error); } - when(response.getEntity(TimelinePutResponse.class)).thenReturn(putResponse); + when(response.readEntity(TimelinePutResponse.class)).thenReturn(putResponse); return response; } - private static ClientResponse mockDomainClientResponse( - TimelineWriter spyTimelineWriter, ClientResponse.Status status, + private static Response mockDomainClientResponse( + TimelineWriter spyTimelineWriter, Response.Status status, boolean hasRuntimeError) { - ClientResponse response = mock(ClientResponse.class); + Response response = mock(Response.class); if (hasRuntimeError) { - doThrow(new ClientHandlerException(new ConnectException())).when( - spyTimelineWriter).doPostingObject(any(TimelineDomain.class), - any(String.class)); + doThrow(new ProcessingException(new ConnectException())) + .when(spyTimelineWriter) + .doPostingObject(any(TimelineDomain.class), any(String.class)); return response; } doReturn(response).when(spyTimelineWriter) @@ -546,7 +542,7 @@ void testTimelineConnectorDestroy() { Client mockJerseyClient = mock(Client.class); client.connector.client = mockJerseyClient; client.stop(); - verify(mockJerseyClient, times(1)).destroy(); + verify(mockJerseyClient, times(1)).close(); } private void setupSSLConfig(YarnConfiguration conf) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientForATS1_5.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientForATS1_5.java index 2fdff72a4f440..002d8f732941b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientForATS1_5.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientForATS1_5.java @@ -22,8 +22,9 @@ import java.io.IOException; import java.net.URI; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; +import javax.ws.rs.client.Client; +import javax.ws.rs.core.Response; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -250,16 +251,15 @@ private static TimelineDomain generateDomain() { private TimelineClientImpl createTimelineClient(YarnConfiguration conf) { TimelineClientImpl client = new TimelineClientImpl() { + @Override - protected TimelineWriter createTimelineWriter(Configuration conf, - UserGroupInformation authUgi, Client client, URI resURI) - throws IOException { + protected TimelineWriter createTimelineWriter(Configuration conf, UserGroupInformation ugi, + Client webClient, URI uri) throws IOException { TimelineWriter timelineWriter = - new FileSystemTimelineWriter(conf, authUgi, client, resURI) { - public ClientResponse doPostingObject(Object object, String path) { - ClientResponse response = mock(ClientResponse.class); - when(response.getStatusInfo()).thenReturn( - ClientResponse.Status.OK); + new FileSystemTimelineWriter(conf, authUgi, webClient, uri) { + public Response doPostingObject(Object object, String path) { + Response response = mock(Response.class); + when(response.getStatusInfo()).thenReturn(Response.Status.OK); return response; } }; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java index 975f9c74f4e7f..612bd3d17ab74 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java @@ -23,9 +23,9 @@ import java.util.ArrayList; import java.util.List; import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; -import com.sun.jersey.api.client.ClientResponse; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -146,27 +146,27 @@ private static TimelineEntity[] createTimelineEntities(String... ids) { private class MockTimelineReaderClient extends TimelineReaderClientImpl { @Override - protected ClientResponse doGetUri(URI base, String path, - MultivaluedMap params) throws IOException { - ClientResponse mockClientResponse = mock(ClientResponse.class); + protected Response doGetUri(URI base, String path, MultivaluedMap params) + throws IOException { + Response mockClientResponse = mock(Response.class); if (path.contains(YARN_CONTAINER.toString()) && !params.containsKey("infofilters")) { - when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity.class)).thenReturn( createTimelineEntity("mockContainer1")); - when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity[].class)).thenReturn( createTimelineEntities("mockContainer1", "mockContainer2")); } else if (path.contains(YARN_CONTAINER.toString()) && params.containsKey("infofilters")) { assertEquals(encodeValue(appAttemptInfoFilter), params.get("infofilters").get(0)); - when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity[].class)).thenReturn( createTimelineEntities("mockContainer3", "mockContainer4")); } else if (path.contains(YARN_APPLICATION_ATTEMPT.toString())) { - when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity.class)).thenReturn( createTimelineEntity("mockAppAttempt1")); - when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity[].class)).thenReturn( createTimelineEntities("mockAppAttempt1", "mockAppAttempt2")); } else { - when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity.class)).thenReturn( createTimelineEntity("mockApp1")); - when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn( + when(mockClientResponse.readEntity(TimelineEntity[].class)).thenReturn( createTimelineEntities("mockApp1", "mockApp2")); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java index 6578248cae0d3..a4ee3bfc627c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java @@ -18,29 +18,8 @@ package org.apache.hadoop.yarn.webapp; -import java.io.IOException; -import java.util.Random; - -import com.sun.jersey.test.framework.JerseyTest; -import com.sun.jersey.test.framework.WebAppDescriptor; - -import org.apache.hadoop.net.ServerSocketUtil; +import org.glassfish.jersey.test.JerseyTest; public abstract class JerseyTestBase extends JerseyTest { - public JerseyTestBase(WebAppDescriptor appDescriptor) { - super(appDescriptor); - } - @Override - protected int getPort(int port) { - Random rand = new Random(); - int jerseyPort = port + rand.nextInt(1000); - try { - jerseyPort = ServerSocketUtil.getPort(jerseyPort, 10); - } catch (IOException e) { - // Ignore exception even after 10 times free port is - // not received. - } - return super.getPort(jerseyPort); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java index 7d7a1575b4724..cfc31bd5b5f0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java @@ -22,16 +22,21 @@ import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder; +import java.util.HashMap; import com.google.inject.Inject; +import org.glassfish.jersey.server.ResourceConfig; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.lang3.ArrayUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.yarn.MockApps; +import org.apache.hadoop.yarn.webapp.resource.MyTestObjectWriterContextResolver; +import org.apache.hadoop.yarn.webapp.resource.MyTestWebService; import org.apache.hadoop.yarn.webapp.view.HtmlPage; import org.apache.hadoop.yarn.webapp.view.JQueryUI; import org.apache.hadoop.yarn.webapp.view.RobotsTextPage; @@ -245,17 +250,27 @@ void testDefaultRoutes() throws Exception { @Test void testCustomRoutes() throws Exception { WebApp app = - WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() { + WebApps.$for("test", TestWebApp.class, this, "ws1").start(new WebApp() { @Override public void setup() { - bind(MyTestJAXBContextResolver.class); - bind(MyTestWebService.class); +// bind(MyTestJAXBContextResolver.class); +// bind(MyTestWebService.class); route("/:foo", FooController.class); route("/bar/foo", FooController.class, "bar"); route("/foo/:foo", DefaultController.class); route("/foo/bar/:foo", DefaultController.class, "index"); } + + @Override + public void addJerseyResourceConfigs(HttpServer2 server) { +// server.addJerseyResourcePackage(MyTestWebService.class.getPackage().getName() + ";" +// + MyTestJAXBContextResolver.class.getPackage().getName(), "/ws1/v1/test/*"); + ResourceConfig resourceConfig = new ResourceConfig(); + resourceConfig.register(MyTestObjectWriterContextResolver.class); + resourceConfig.register(MyTestWebService.class); + server.addJerseyResourceConfig(resourceConfig, "/ws/v1/test/*", new HashMap<>()); + } }); String baseUrl = baseUrl(app); try { @@ -280,7 +295,7 @@ void testEncodedUrl() throws Exception { WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() { @Override public void setup() { - bind(MyTestJAXBContextResolver.class); + bind(MyTestObjectWriterContextResolver.class); bind(MyTestWebService.class); route("/:foo", FooController.class); @@ -310,7 +325,7 @@ void testRobotsText() throws Exception { WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() { @Override public void setup() { - bind(MyTestJAXBContextResolver.class); + bind(MyTestObjectWriterContextResolver.class); bind(MyTestWebService.class); } }); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/resource/MyTestObjectWriterContextResolver.java similarity index 65% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/resource/MyTestObjectWriterContextResolver.java index 242bf047805d2..a0edb0a83e0d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/resource/MyTestObjectWriterContextResolver.java @@ -16,40 +16,37 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.webapp; +package org.apache.hadoop.yarn.webapp.resource; import java.util.Arrays; import java.util.HashSet; import java.util.Set; import javax.ws.rs.ext.ContextResolver; import javax.ws.rs.ext.Provider; -import javax.xml.bind.JAXBContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import com.google.inject.Singleton; -import com.sun.jersey.api.json.JSONConfiguration; -import com.sun.jersey.api.json.JSONJAXBContext; -import org.apache.hadoop.yarn.webapp.MyTestWebService.MyInfo; +import org.apache.hadoop.yarn.webapp.resource.MyTestWebService.MyInfo; @Singleton @Provider -public class MyTestJAXBContextResolver implements ContextResolver { +public class MyTestObjectWriterContextResolver implements ContextResolver { - private JAXBContext context; + private ObjectWriter context; private final Set types; // you have to specify all the dao classes here private final Class[] cTypes = { MyInfo.class }; - public MyTestJAXBContextResolver() throws Exception { - this.types = new HashSet(Arrays.asList(cTypes)); - this.context = - new JSONJAXBContext(JSONConfiguration.natural().rootUnwrapping(false) - .build(), cTypes); + public MyTestObjectWriterContextResolver() { + this.types = new HashSet<>(Arrays.asList(cTypes)); + this.context = new ObjectMapper().writerFor(MyInfo.class); } @Override - public JAXBContext getContext(Class objectType) { + public ObjectWriter getContext(Class objectType) { return (types.contains(objectType)) ? context : null; } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/resource/MyTestWebService.java similarity index 91% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestWebService.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/resource/MyTestWebService.java index 1d0a01ea53dea..26b059848b758 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestWebService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/resource/MyTestWebService.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.webapp; +package org.apache.hadoop.yarn.webapp.resource; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -26,12 +26,9 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; -import com.google.inject.Singleton; - import org.apache.hadoop.http.JettyUtils; -@Singleton -@Path("/ws/v1/test") +@Path("") public class MyTestWebService { @GET @Produces({ MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) @@ -41,7 +38,7 @@ public MyInfo get() { @XmlRootElement(name = "myInfo") @XmlAccessorType(XmlAccessType.FIELD) - static class MyInfo { + public static class MyInfo { public MyInfo() { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java index b51dcf88bcbf7..3076b51b78500 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java @@ -22,6 +22,8 @@ import java.net.URI; import java.net.URL; +import javax.ws.rs.core.Response; + import org.junit.jupiter.api.Test; import org.apache.hadoop.conf.Configuration; @@ -92,9 +94,8 @@ void testCreateClient() throws Exception { URL u = new URL(baseUrl, SERVLET_PATH_ECHO + "?a=b&c=d"); WebServiceClient.initialize(sslConf); WebServiceClient client = WebServiceClient.getWebServiceClient(); - HttpURLConnection conn = client.getHttpURLConnectionFactory() - .getHttpURLConnection(u); - assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + Response response = client.createClient().target(u.toURI()).request().get(); + assertEquals(HttpURLConnection.HTTP_OK, response.getStatus()); WebServiceClient.destroy(); server.stop(); FileUtil.fullyDelete(new File(BASEDIR)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index d0fd79aaa78e1..285711b5c07c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -88,41 +88,6 @@ com.google.inject guice - - com.sun.jersey.jersey-test-framework - jersey-test-framework-core - test - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-client - - - com.github.pjfanning - jersey-json - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - - - - com.sun.jersey.contribs - jersey-guice - org.apache.hadoop @@ -159,13 +124,6 @@ hadoop-yarn-server-common - - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - test - - commons-collections commons-collections diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index a66b666402094..b3d63f398416c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -71,14 +71,6 @@ commons-codec commons-codec - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-client - org.eclipse.jetty jetty-util @@ -145,33 +137,6 @@ com.google.inject guice - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - test - - - com.github.pjfanning - jersey-json - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - - - - com.sun.jersey.contribs - jersey-guice - org.apache.hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 9d096d20c5fdd..d1393509e1f3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -101,33 +101,6 @@ com.google.inject guice - - com.sun.jersey.jersey-test-framework - jersey-test-framework-core - test - - - com.github.pjfanning - jersey-json - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - - - - com.sun.jersey.contribs - jersey-guice - org.apache.hadoop @@ -153,14 +126,6 @@ org.codehaus.jettison jettison - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-client - org.eclipse.jetty jetty-util @@ -260,13 +225,6 @@ ${project.version} - - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - test - - com.github.stefanbirkner system-rules diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index 6f2fce097df73..af0a8a11d1392 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -202,18 +202,6 @@ runtime - - com.sun.jersey - jersey-client - test - - - - javax.ws.rs - jsr311-api - 1.1.1 - - org.apache.hbase hbase-common diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml index 5a2823ad5eff5..3ec267671520c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml @@ -101,22 +101,11 @@ jackson-databind - - com.sun.jersey - jersey-client - - org.apache.commons commons-csv - - javax.ws.rs - jsr311-api - 1.1.1 - - org.apache.hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml index 15df5456810ce..6927c3770e287 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml @@ -116,11 +116,6 @@ - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - test - org.bouncycastle bcprov-jdk15on