Update Hadoop to 3.4.2 and enable security (Kerberos) integration tests under JDK-24 and above

CHANGELOG.md

@@ -92,6 +92,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Exclude group com.microsoft.sqlserver from hadoop-minicluster ([#19889](https://github.com/opensearch-project/OpenSearch/pull/19889))
 - Bump `actions/github-script` from 7 to 8 ([#19946](https://github.com/opensearch-project/OpenSearch/pull/19946))
 - Bump `com.google.api:gax-httpjson` from 2.69.0 to 2.72.1 ([#19943](https://github.com/opensearch-project/OpenSearch/pull/19943))
+- Update Hadoop to 3.4.2 and enable security (Kerberos) integration tests under JDK-24 and above  ([#19952](https://github.com/opensearch-project/OpenSearch/pull/19952))
 
 ### Deprecated
 - Deprecated existing constructors in ThreadPoolStats.Stats in favor of the new Builder ([#19317](https://github.com/opensearch-project/OpenSearch/pull/19317))

gradle/libs.versions.toml

@@ -59,7 +59,7 @@ commonscollections4 = "4.5.0"
 aws               = "2.32.29"
 awscrt            = "0.35.0"
 reactivestreams   = "1.0.4"
-hadoop3           = "3.3.6"
+hadoop3           = "3.4.2"
 
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)

plugins/repository-hdfs/build.gradle

@@ -176,12 +176,6 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture',
 
 Set disabledIntegTestTaskNames = []
 
-// Temporarily disable xxxSecure tests for JDK-24 and above, the krb5 does not play well
-if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_24) {
-  disabledIntegTestTaskNames += ['integTestSecure', 'integTestSecureHa']
-  testingConventions.enabled = false
-}
-
 for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSecureHa']) {
   task "${integTestTaskName}"(type: RestIntegTestTask) {
     description = "Runs rest tests against an opensearch cluster with HDFS."
@@ -371,7 +365,6 @@ thirdPartyAudit {
     'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
     'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
     'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
-    'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
     'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64',
     'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$1',
     'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$Cell',
@@ -383,7 +376,9 @@ thirdPartyAudit {
     'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$1',
     'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$JvmMemoryAccessor',
     'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor',
-
+    'org.apache.hadoop.thirdparty.protobuf.MessageSchema',
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android32MemoryAccessor',
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor',
     'org.apache.hadoop.shaded.com.google.common.cache.Striped64',
     'org.apache.hadoop.shaded.com.google.common.cache.Striped64$1',
     'org.apache.hadoop.shaded.com.google.common.cache.Striped64$Cell',

plugins/repository-hdfs/licenses/hadoop-client-api-3.4.2.jar.sha1

@@ -0,0 +1 @@
+d49afafdccb52bddde866ea0f341e6b31edc97fe

plugins/repository-hdfs/licenses/hadoop-client-runtime-3.4.2.jar.sha1

@@ -0,0 +1 @@
+3e9508f154ac9f085f3f0400c696175dce771d2d

plugins/repository-hdfs/licenses/hadoop-hdfs-3.4.2.jar.sha1

@@ -0,0 +1 @@
+850d93a5a566ff19f3d77e67cb79c0de28d3ab78

server/src/main/resources/org/opensearch/bootstrap/security.policy

@@ -266,4 +266,7 @@ grant {
   permission java.security.SecurityPermission "insertProvider.BCJSSE";
 
   permission java.io.FilePermission "${java.home}/conf/security/policy/unlimited/*", "read";
+
+  // allow to access krb5.conf
+  permission java.io.FilePermission "${{java.security.krb5.conf}}", "read";
 };