sync

LICENSE-binary

@@ -241,7 +241,7 @@ com.google.guava:guava:27.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.microsoft.azure:azure-storage:7.0.0
 com.nimbusds:nimbus-jose-jwt:9.8.1
-com.squareup.okhttp:okhttp:2.7.5
+com.squareup.okhttp3:okhttp:4.9.3
 com.squareup.okio:okio:1.6.0
 com.zaxxer:HikariCP:4.0.3
 commons-beanutils:commons-beanutils:1.9.3
@@ -324,10 +324,6 @@ org.apache.kerby:token-provider:1.0.1
 org.apache.solr:solr-solrj:8.8.2
 org.apache.yetus:audience-annotations:0.5.0
 org.apache.zookeeper:zookeeper:3.6.3
-org.codehaus.jackson:jackson-core-asl:1.9.13
-org.codehaus.jackson:jackson-jaxrs:1.9.13
-org.codehaus.jackson:jackson-mapper-asl:1.9.13
-org.codehaus.jackson:jackson-xc:1.9.13
 org.codehaus.jettison:jettison:1.1
 org.eclipse.jetty:jetty-annotations:9.4.44.v20210927
 org.eclipse.jetty:jetty-http:9.4.44.v20210927
@@ -484,12 +480,12 @@ org.slf4j:slf4j-log4j12:1.7.25
 CDDL 1.1 + GPLv2 with classpath exception
 -----------------------------------------
 
-com.sun.jersey:jersey-client:1.19
-com.sun.jersey:jersey-core:1.19
-com.sun.jersey:jersey-guice:1.19
-com.sun.jersey:jersey-json:1.19
-com.sun.jersey:jersey-server:1.19
-com.sun.jersey:jersey-servlet:1.19
+com.github.pjfanning:jersey-json:1.20
+com.sun.jersey:jersey-client:1.19.4
+com.sun.jersey:jersey-core:1.19.4
+com.sun.jersey:jersey-guice:1.19.4
+com.sun.jersey:jersey-server:1.19.4
+com.sun.jersey:jersey-servlet:1.19.4
 com.sun.xml.bind:jaxb-impl:2.2.3-1
 javax.annotation:javax.annotation-api:1.3.2
 javax.servlet:javax.servlet-api:3.1.0

dev-support/Jenkinsfile

@@ -55,7 +55,7 @@ pipeline {
     environment {
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
-        YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
+        YETUS_VERSION='rel/0.14.0'
     }
 
     parameters {

dev-support/bin/create-release

@@ -293,6 +293,7 @@ function usage
   echo "--security              Emergency security release"
   echo "--sign                  Use .gnupg dir to sign the artifacts and jars"
   echo "--version=[version]     Use an alternative version string"
+  echo "--mvnargs=[args]        Extra Maven args to be provided when running mvn commands"
 }
 
 function option_parse
@@ -347,6 +348,9 @@ function option_parse
       --version=*)
         HADOOP_VERSION=${i#*=}
       ;;
+      --mvnargs=*)
+        MVNEXTRAARGS=${i#*=}
+      ;;
     esac
   done
 
@@ -413,6 +417,9 @@ function option_parse
       MVN_ARGS=("-Dmaven.repo.local=${MVNCACHE}")
     fi
   fi
+  if [ -n "$MVNEXTRAARGS" ]; then
+    MVN_ARGS+=("$MVNEXTRAARGS")
+  fi
 
   if [[ "${SECURITYRELEASE}" = true ]]; then
     if [[ ! -d "${BASEDIR}/hadoop-common-project/hadoop-common/src/site/markdown/release/${HADOOP_VERSION}" ]]; then
@@ -535,6 +542,10 @@ function makearelease
 
   big_console_header "Cleaning the Source Tree"
 
+  # Since CVE-2022-24765 in April 2022, git refuses to work in directories
+  # whose owner != the current user, unless explicitly told to trust it.
+  git config --global --add safe.directory /build/source
+
   # git clean to clear any remnants from previous build
   run "${GIT}" clean -xdf -e /patchprocess
 

dev-support/bin/yetus-wrapper

@@ -77,7 +77,7 @@ WANTED="$1"
 shift
 ARGV=("$@")
 
-HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.13.0}
+HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.14.0}
 BIN=$(yetus_abs "${BASH_SOURCE-$0}")
 BINDIR=$(dirname "${BIN}")
 

dev-support/docker/Dockerfile_debian_10

@@ -82,6 +82,7 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true
 ####
 # Install packages
 ####
+RUN pkg-resolver/install-cmake.sh debian:10
 RUN pkg-resolver/install-spotbugs.sh debian:10
 RUN pkg-resolver/install-boost.sh debian:10
 RUN pkg-resolver/install-protobuf.sh debian:10

dev-support/docker/pkg-resolver/install-maven.sh

@@ -40,7 +40,7 @@ fi
 
 if [ "$version_to_install" == "3.6.3" ]; then
   mkdir -p /opt/maven /tmp/maven &&
-    curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
+    curl -L -s -S https://dlcdn.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
       -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz &&
     tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
 else

dev-support/docker/pkg-resolver/packages.json

@@ -62,7 +62,6 @@
     "centos:8": "clang"
   },
   "cmake": {
-    "debian:10": "cmake",
     "ubuntu:focal": "cmake",
     "ubuntu:focal::arch64": "cmake"
   },

dev-support/git-jira-validation/git_jira_fix_version_check.py

@@ -71,10 +71,9 @@
         print("Commit seems reverted. \t\t\t Commit: " + commit)
         continue
     ACTUAL_PROJECT_JIRA = None
-    for project_jira in project_jira_keys:
-        if project_jira in commit:
-            ACTUAL_PROJECT_JIRA = project_jira
-            break
+    matches = re.findall('|'.join(project_jira_keys), commit)
+    if matches:
+        ACTUAL_PROJECT_JIRA = matches[0]
     if not ACTUAL_PROJECT_JIRA:
         print("WARN: Jira not found. \t\t\t Commit: " + commit)
         continue

dev-support/hadoop-vote.sh

@@ -189,13 +189,16 @@ pushd "${OUTPUT_DIR}"
 download_and_import_keys
 download_release_candidate
 
+pushd "${HADOOP_RC_VERSION}"
+
 execute verify_signatures
 execute verify_checksums
 execute unzip_from_source
 execute rat_test
 execute build_from_source
 execute build_tar_from_source
 
+popd
 popd
 
 print_when_exit

hadoop-client-modules/hadoop-client-minicluster/pom.xml

@@ -423,29 +423,25 @@
       <optional>true</optional>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
+      <groupId>com.github.pjfanning</groupId>
       <artifactId>jersey-json</artifactId>
       <optional>true</optional>
       <exclusions>
         <exclusion>
-          <groupId>javax.xml.bind</groupId>
-          <artifactId>jaxb-api</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-core-asl</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-mapper-asl</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-databind</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-jaxrs</artifactId>
+          <groupId>com.fasterxml.jackson.jaxrs</groupId>
+          <artifactId>jackson-jaxrs-json-provider</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-xc</artifactId>
+          <groupId>javax.xml.bind</groupId>
+          <artifactId>jaxb-api</artifactId>
         </exclusion>
       </exclusions>
     </dependency>
@@ -467,6 +463,10 @@
           <groupId>javax.enterprise</groupId>
           <artifactId>cdi-api</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>ch.qos.cal10n</groupId>
+          <artifactId>cal10n-api</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <!-- skip org.apache.avro:avro-ipc because it doesn't look like hadoop-common actually uses it -->

hadoop-client-modules/hadoop-client/pom.xml

@@ -66,7 +66,7 @@
           <artifactId>jersey-core</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.sun.jersey</groupId>
+          <groupId>com.github.pjfanning</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
         <exclusion>
@@ -114,6 +114,18 @@
           <groupId>org.eclipse.jetty</groupId>
           <artifactId>jetty-server</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.jetbrains.kotlin</groupId>
+          <artifactId>kotlin-stdlib</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.jetbrains.kotlin</groupId>
+          <artifactId>kotlin-stdlib-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.squareup.okhttp3</groupId>
+          <artifactId>okhttp</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-core</artifactId>
@@ -167,7 +179,7 @@
           <artifactId>jersey-core</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.sun.jersey</groupId>
+          <groupId>com.github.pjfanning</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
         <exclusion>
@@ -218,7 +230,7 @@
           <artifactId>jersey-server</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.sun.jersey</groupId>
+          <groupId>com.github.pjfanning</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
         <exclusion>
@@ -275,7 +287,7 @@
           <artifactId>guice-servlet</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.sun.jersey</groupId>
+          <groupId>com.github.pjfanning</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
         <exclusion>

hadoop-common-project/hadoop-auth-examples/src/main/java/org/apache/hadoop/security/authentication/examples/RequestLoggerFilter.java

@@ -116,6 +116,7 @@ private List<String> getHeaderValues(String name, boolean reset) {
     public void addCookie(Cookie cookie) {
       super.addCookie(cookie);
       List<String> cookies = getHeaderValues("Set-Cookie", false);
+      cookies.addAll(getHeaderValues("set-cookie", false));
       cookies.add(cookie.getName() + "=" + cookie.getValue());
     }
 

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java

@@ -92,6 +92,9 @@ public synchronized Map<String, List<String>> get(URI uri,
     @Override
     public void put(URI uri, Map<String, List<String>> responseHeaders) {
       List<String> headers = responseHeaders.get("Set-Cookie");
+      if (headers == null) {
+        headers = responseHeaders.get("set-cookie");
+      }
       if (headers != null) {
         for (String header : headers) {
           List<HttpCookie> cookies;

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java

@@ -280,6 +280,9 @@ private boolean isNegotiate(HttpURLConnection conn) throws IOException {
     boolean negotiate = false;
     if (conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
       String authHeader = conn.getHeaderField(WWW_AUTHENTICATE);
+      if (authHeader == null) {
+        authHeader = conn.getHeaderField(WWW_AUTHENTICATE.toLowerCase());
+      }
       negotiate = authHeader != null && authHeader.trim().startsWith(NEGOTIATE);
     }
     return negotiate;
@@ -388,6 +391,9 @@ private byte[] readToken(HttpURLConnection conn)
     int status = conn.getResponseCode();
     if (status == HttpURLConnection.HTTP_OK || status == HttpURLConnection.HTTP_UNAUTHORIZED) {
       String authHeader = conn.getHeaderField(WWW_AUTHENTICATE);
+      if (authHeader == null) {
+        authHeader = conn.getHeaderField(WWW_AUTHENTICATE.toLowerCase());
+      }
       if (authHeader == null || !authHeader.trim().startsWith(NEGOTIATE)) {
         throw new AuthenticationException("Invalid SPNEGO sequence, '" + WWW_AUTHENTICATE +
                                           "' header incorrect: " + authHeader);

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java

@@ -616,7 +616,9 @@ && getMaxInactiveInterval() > 0) {
         // present.. reset to 403 if not found..
         if ((errCode == HttpServletResponse.SC_UNAUTHORIZED)
             && (!httpResponse.containsHeader(
-                KerberosAuthenticator.WWW_AUTHENTICATE))) {
+                KerberosAuthenticator.WWW_AUTHENTICATE)
+                && !httpResponse.containsHeader(
+                KerberosAuthenticator.WWW_AUTHENTICATE.toLowerCase()))) {
           errCode = HttpServletResponse.SC_FORBIDDEN;
         }
         // After Jetty 9.4.21, sendError() no longer allows a custom message.

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java

@@ -89,6 +89,44 @@ public void testExtractTokenFail() throws Exception {
     }
   }
 
+  @Test
+  public void testExtractTokenCookieHeader() throws Exception {
+    HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
+
+    Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_OK);
+
+    String tokenStr = "foo";
+    Map<String, List<String>> headers = new HashMap<>();
+    List<String> cookies = new ArrayList<>();
+    cookies.add(AuthenticatedURL.AUTH_COOKIE + "=" + tokenStr);
+    headers.put("Set-Cookie", cookies);
+    Mockito.when(conn.getHeaderFields()).thenReturn(headers);
+
+    AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+    AuthenticatedURL.extractToken(conn, token);
+
+    Assert.assertTrue(token.isSet());
+  }
+
+  @Test
+  public void testExtractTokenLowerCaseCookieHeader() throws Exception {
+    HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
+
+    Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_OK);
+
+    String tokenStr = "foo";
+    Map<String, List<String>> headers = new HashMap<>();
+    List<String> cookies = new ArrayList<>();
+    cookies.add(AuthenticatedURL.AUTH_COOKIE + "=" + tokenStr);
+    headers.put("set-cookie", cookies);
+    Mockito.when(conn.getHeaderFields()).thenReturn(headers);
+
+    AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+    AuthenticatedURL.extractToken(conn, token);
+
+    Assert.assertTrue(token.isSet());
+  }
+
   @Test
   public void testConnectionConfigurator() throws Exception {
     HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);