diff --git a/CHANGELOG.md b/CHANGELOG.md index 51817010ccfcb..31ace45758c14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Exclude group com.microsoft.sqlserver from hadoop-minicluster ([#19889](https://github.com/opensearch-project/OpenSearch/pull/19889)) - Bump `actions/github-script` from 7 to 8 ([#19946](https://github.com/opensearch-project/OpenSearch/pull/19946)) - Bump `com.google.api:gax-httpjson` from 2.69.0 to 2.72.1 ([#19943](https://github.com/opensearch-project/OpenSearch/pull/19943)) +- Update Hadoop to 3.4.2 and enable security (Kerberos) integration tests under JDK-24 and above ([#19952](https://github.com/opensearch-project/OpenSearch/pull/19952)) ### Deprecated - Deprecated existing constructors in ThreadPoolStats.Stats in favor of the new Builder ([#19317](https://github.com/opensearch-project/OpenSearch/pull/19317)) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 49caa9a3d37a4..a9e85e9a2e74d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -59,7 +59,7 @@ commonscollections4 = "4.5.0" aws = "2.32.29" awscrt = "0.35.0" reactivestreams = "1.0.4" -hadoop3 = "3.3.6" +hadoop3 = "3.4.2" # when updating this version, you need to ensure compatibility with: # - plugins/ingest-attachment (transitive dependency, check the upstream POM) diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 9570a84fcc372..4f8784695c506 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -176,12 +176,6 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', Set disabledIntegTestTaskNames = [] -// Temporarily disable xxxSecure tests for JDK-24 and above, the krb5 does not play well -if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_24) { - disabledIntegTestTaskNames += ['integTestSecure', 'integTestSecureHa'] - testingConventions.enabled = false -} - for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSecureHa']) { task "${integTestTaskName}"(type: RestIntegTestTask) { description = "Runs rest tests against an opensearch cluster with HDFS." @@ -371,7 +365,6 @@ thirdPartyAudit { 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', - 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3', 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64', 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$1', 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$Cell', @@ -383,7 +376,9 @@ thirdPartyAudit { 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$1', 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$JvmMemoryAccessor', 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor', - + 'org.apache.hadoop.thirdparty.protobuf.MessageSchema', + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android32MemoryAccessor', + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor', 'org.apache.hadoop.shaded.com.google.common.cache.Striped64', 'org.apache.hadoop.shaded.com.google.common.cache.Striped64$1', 'org.apache.hadoop.shaded.com.google.common.cache.Striped64$Cell', diff --git a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 deleted file mode 100644 index d99793bc56522..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -12ac6f103a0ff29fce17a078c7c64d25320b6165 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-api-3.4.2.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-api-3.4.2.jar.sha1 new file mode 100644 index 0000000000000..c426a8f969c13 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-client-api-3.4.2.jar.sha1 @@ -0,0 +1 @@ +d49afafdccb52bddde866ea0f341e6b31edc97fe \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 deleted file mode 100644 index ea22d763b7bfa..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -81065531e63fccbe85fb04a3274709593fb00d3c \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.4.2.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.4.2.jar.sha1 new file mode 100644 index 0000000000000..39180a3bb4114 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.4.2.jar.sha1 @@ -0,0 +1 @@ +3e9508f154ac9f085f3f0400c696175dce771d2d \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 deleted file mode 100644 index fe60968056eb7..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ba40aca60f39599d5b1f1d32b35295bfde1f3c8b \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.4.2.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.4.2.jar.sha1 new file mode 100644 index 0000000000000..ac9494c186654 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.4.2.jar.sha1 @@ -0,0 +1 @@ +850d93a5a566ff19f3d77e67cb79c0de28d3ab78 \ No newline at end of file diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index d1fee5e550dac..6c81188335865 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -266,4 +266,7 @@ grant { permission java.security.SecurityPermission "insertProvider.BCJSSE"; permission java.io.FilePermission "${java.home}/conf/security/policy/unlimited/*", "read"; + + // allow to access krb5.conf + permission java.io.FilePermission "${{java.security.krb5.conf}}", "read"; };