HADOOP-18329 - Support for IBM Semeru JVM v>11.0.15.0 Vendor Name Changes

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java

@@ -42,7 +42,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
+import static org.apache.hadoop.util.PlatformName.USE_IBM_JAVA_PACKAGES;
 
 /**
  * The {@link KerberosAuthenticator} implements the Kerberos SPNEGO authentication sequence.
@@ -88,7 +88,7 @@ private static class KerberosConfiguration extends Configuration {
 
     /* Return the OS login module class name */
     private static String getOSLoginModuleName() {
-      if (IBM_JAVA) {
+      if (USE_IBM_JAVA_PACKAGES) {
         if (windows) {
           return is64Bit ? "com.ibm.security.auth.module.Win64LoginModule"
               : "com.ibm.security.auth.module.NTLoginModule";
@@ -117,14 +117,14 @@ private static String getOSLoginModuleName() {
 
     static {
       String ticketCache = System.getenv("KRB5CCNAME");
-      if (IBM_JAVA) {
+      if (USE_IBM_JAVA_PACKAGES) {
         USER_KERBEROS_OPTIONS.put("useDefaultCcache", "true");
       } else {
         USER_KERBEROS_OPTIONS.put("doNotPrompt", "true");
         USER_KERBEROS_OPTIONS.put("useTicketCache", "true");
       }
       if (ticketCache != null) {
-        if (IBM_JAVA) {
+        if (USE_IBM_JAVA_PACKAGES) {
           // The first value searched when "useDefaultCcache" is used.
           System.setProperty("KRB5CCNAME", ticketCache);
         } else {

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java

@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.security.authentication.util;
 
-import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
+import static org.apache.hadoop.util.PlatformName.USE_IBM_JAVA_PACKAGES;
 
 import java.io.File;
 import java.io.IOException;
@@ -50,7 +50,7 @@ public class KerberosUtil {
 
   /* Return the Kerberos login module name */
   public static String getKrb5LoginModuleName() {
-    return (IBM_JAVA)
+    return (USE_IBM_JAVA_PACKAGES)
       ? "com.ibm.security.auth.module.Krb5LoginModule"
       : "com.sun.security.auth.module.Krb5LoginModule";
   }
@@ -157,7 +157,7 @@ public static String getDomainRealm(String shortprinc) {
     Object principalName; //of type sun.security.krb5.PrincipalName or IBM equiv
     String realmString = null;
     try {
-      if (IBM_JAVA) {
+      if (USE_IBM_JAVA_PACKAGES) {
         classRef = Class.forName("com.ibm.security.krb5.PrincipalName");
       } else {
         classRef = Class.forName("sun.security.krb5.PrincipalName");

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/util/PlatformName.java

@@ -45,9 +45,34 @@ public class PlatformName {
 
   /**
    * A public static variable to indicate the current java vendor is
-   * IBM java or not.
+   * IBM and the type is Java Technology Edition which provides its
+   * own implementations of many security packages and Cipher suites.
+   * Note that these are not provided in Semeru runtimes:
+   * See https://developer.ibm.com/languages/java/semeru-runtimes/
    */
-  public static final boolean IBM_JAVA = JAVA_VENDOR_NAME.contains("IBM");
+  public static final boolean USE_IBM_JAVA_PACKAGES = shouldUseIbmPackages();
+
+  /*
+  * IBM_JAVA must be preserved due to the public nature of the property.
+  */
+  @Deprecated
+  public static final boolean IBM_JAVA = USE_IBM_JAVA_PACKAGES;
+
+  private static boolean shouldUseIbmPackages() {
+    if (JAVA_VENDOR_NAME.contains("IBM")) {
+      try {
+        /**
+         * This class is provided by all supported IBM JTE Runtimes,
+         * but ensures we do not make assumptions of existence of
+         * specialised security modules based on vendor alone.
+         */
+        Class.forName("com.ibm.security.auth.module.JAASLoginModule");
+        return true;
+      } catch(ClassNotFoundException ignored) {}
+    }
+
+    return false;
+  }
 
   public static void main(String[] args) {
     System.out.println(PLATFORM_NAME);

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/KerberosTestUtils.java

@@ -32,7 +32,7 @@
 import java.util.Set;
 import java.util.concurrent.Callable;
 
-import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
+import static org.apache.hadoop.util.PlatformName.USE_IBM_JAVA_PACKAGES;
 
 /**
  * Test helper class for Java Kerberos setup.
@@ -67,7 +67,7 @@ public KerberosConfiguration(String principal) {
     @Override
     public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
       Map<String, String> options = new HashMap<String, String>();
-      if (IBM_JAVA) {
+      if (USE_IBM_JAVA_PACKAGES) {
         options.put("useKeytab", KerberosTestUtils.getKeytabFile().startsWith("file://") ?   
                     KerberosTestUtils.getKeytabFile() : "file://" +  KerberosTestUtils.getKeytabFile());
         options.put("principal", principal);
@@ -86,7 +86,7 @@ public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
       } 
       String ticketCache = System.getenv("KRB5CCNAME");
       if (ticketCache != null) {
-        if (IBM_JAVA) {
+        if (USE_IBM_JAVA_PACKAGES) {
           // IBM JAVA only respect system property and not env variable
           // The first value searched when "useDefaultCcache" is used.
           System.setProperty("KRB5CCNAME", ticketCache);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java

@@ -25,7 +25,7 @@
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKENS;
 import static org.apache.hadoop.security.UGIExceptionMessages.*;
-import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
+import static org.apache.hadoop.util.PlatformName.USE_IBM_JAVA_PACKAGES;
 import static org.apache.hadoop.util.StringUtils.getTrimmedStringCollection;
 
 import org.apache.hadoop.classification.VisibleForTesting;
@@ -430,7 +430,7 @@ static Optional<ExecutorService> getKerberosLoginRenewalExecutor() {
   /* Return the OS login module class name */
   /* For IBM JDK, use the common OS login module class name for all platforms */
   private static String getOSLoginModuleName() {
-    if (IBM_JAVA) {
+    if (USE_IBM_JAVA_PACKAGES) {
       return "com.ibm.security.auth.module.JAASLoginModule";
     } else {
       return windows ? "com.sun.security.auth.module.NTLoginModule"
@@ -445,7 +445,7 @@ private static Class<? extends Principal> getOsPrincipalClass() {
     ClassLoader cl = ClassLoader.getSystemClassLoader();
     try {
       String principalClass = null;
-      if (IBM_JAVA) {
+      if (USE_IBM_JAVA_PACKAGES) {
         principalClass = "com.ibm.security.auth.UsernamePrincipal";
       } else {
         principalClass = windows ? "com.sun.security.auth.NTUserPrincipal"
@@ -2203,7 +2203,7 @@ private AppConfigurationEntry getKerberosEntry() {
       }
 
       // use keytab if given else fallback to ticket cache.
-      if (IBM_JAVA) {
+      if (USE_IBM_JAVA_PACKAGES) {
         if (params.containsKey(LoginParam.KEYTAB)) {
           final String keytab = params.get(LoginParam.KEYTAB);
           if (keytab != null) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java

@@ -25,7 +25,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.util.PlatformName.JAVA_VENDOR_NAME;
+import static org.apache.hadoop.util.PlatformName.USE_IBM_JAVA_PACKAGES;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.HttpsURLConnection;
@@ -102,11 +102,11 @@ public enum Mode { CLIENT, SERVER }
       "ssl.server.exclude.cipher.list";
 
   public static final String KEY_MANAGER_SSLCERTIFICATE =
-      JAVA_VENDOR_NAME.contains("IBM") ? "ibmX509" :
+      USE_IBM_JAVA_PACKAGES ? "ibmX509" :
           KeyManagerFactory.getDefaultAlgorithm();
 
   public static final String TRUST_MANAGER_SSLCERTIFICATE =
-      JAVA_VENDOR_NAME.contains("IBM") ? "ibmX509" :
+      USE_IBM_JAVA_PACKAGES ? "ibmX509" :
           TrustManagerFactory.getDefaultAlgorithm();
 
   public static final String KEYSTORES_FACTORY_CLASS_KEY =

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java

@@ -74,7 +74,7 @@
 import org.apache.hadoop.security.alias.LocalJavaKeyStoreProvider;
 import org.apache.hadoop.test.GenericTestUtils;
 
-import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
+import static org.apache.hadoop.util.PlatformName.USE_IBM_JAVA_PACKAGES;
 
 import org.apache.log4j.AppenderSkeleton;
 import org.apache.log4j.Logger;
@@ -99,7 +99,7 @@ public class TestConfiguration {
     "./test-config-multi-byte-saved-TestConfiguration.xml").getAbsolutePath();
   final static Random RAN = new Random();
   final static String XMLHEADER =
-            IBM_JAVA?"<?xml version=\"1.0\" encoding=\"UTF-8\"?><configuration>":
+            USE_IBM_JAVA_PACKAGES?"<?xml version=\"1.0\" encoding=\"UTF-8\"?><configuration>":
   "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><configuration>";
 
   /** Four apostrophes. */

hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java

@@ -38,8 +38,19 @@
 import java.util.Arrays;
 
 public class TestMiniKdc extends KerberosSecurityTestcase {
-  private static final boolean IBM_JAVA = System.getProperty("java.vendor")
-      .contains("IBM");
+  private static final boolean USE_IBM_JAVA_PACKAGES = shouldUseIbmPackages();
+
+  private static boolean shouldUseIbmPackages() {
+    if (System.getProperty("java.vendor").contains("IBM")) {
+      try {
+        Class.forName("com.ibm.security.auth.module.JAASLoginModule");
+        return true;
+      } catch(ClassNotFoundException ignored) {}
+    }
+
+    return false;
+  }
+
   @Test
   public void testMiniKdcStart() {
     MiniKdc kdc = getKdc();
@@ -98,7 +109,7 @@ public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
       Map<String, String> options = new HashMap<String, String>();
       options.put("principal", principal);
       options.put("refreshKrb5Config", "true");
-      if (IBM_JAVA) {
+      if (USE_IBM_JAVA_PACKAGES) {
         options.put("useKeytab", keytab);
         options.put("credsType", "both");
       } else {

hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java

@@ -53,7 +53,7 @@
 import static org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions.*;
 import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
 
-import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
+import static org.apache.hadoop.util.PlatformName.USE_IBM_JAVA_PACKAGES;
 
 /**
  * Implement the registry security ... a self contained service for
@@ -616,7 +616,7 @@ public static String getKerberosAuthModuleForJVM() {
    * Note the semicolon on the last entry
    */
   private static final String JAAS_ENTRY =
-      (IBM_JAVA ?
+      (USE_IBM_JAVA_PACKAGES ?
       "%s { %n"
       + " %s required%n"
       + " useKeytab=\"%s\"%n"

hadoop-common-project/hadoop-registry/src/test/java/org/apache/hadoop/registry/secure/KerberosConfiguration.java

@@ -25,7 +25,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
+import static org.apache.hadoop.util.PlatformName.USE_IBM_JAVA_PACKAGES;
 
 class KerberosConfiguration extends javax.security.auth.login.Configuration {
   private String principal;
@@ -54,7 +54,7 @@ public static javax.security.auth.login.Configuration createServerConfig(
   @Override
   public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
     Map<String, String> options = new HashMap<String, String>();
-    if (IBM_JAVA) {
+    if (USE_IBM_JAVA_PACKAGES) {
       options.put("useKeytab", keytab.startsWith("file://")
           ? keytab
           : "file://" + keytab);
@@ -74,7 +74,7 @@ public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
     }
     String ticketCache = System.getenv("KRB5CCNAME");
     if (ticketCache != null) {
-      if (IBM_JAVA) {
+      if (USE_IBM_JAVA_PACKAGES) {
         // IBM JAVA only respect system property and not env variable
         // The first value searched when "useDefaultCcache" is used.
         System.setProperty("KRB5CCNAME", ticketCache);

hadoop-common-project/hadoop-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java

@@ -47,7 +47,7 @@
 import static org.apache.hadoop.security.authentication.util.KerberosName.DEFAULT_MECHANISM;
 import static org.apache.hadoop.security.authentication.util.KerberosName.MECHANISM_HADOOP;
 import static org.apache.hadoop.security.authentication.util.KerberosName.MECHANISM_MIT;
-import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
+import static org.apache.hadoop.util.PlatformName.USE_IBM_JAVA_PACKAGES;
 
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -141,7 +141,7 @@ public void testKerberosAuth() throws Throwable {
     Object kerb5LoginObject = kerb5LoginConstr.newInstance();
     final Map<String, String> options = new HashMap<String, String>();
     options.put("debug", "true");
-    if (IBM_JAVA) {
+    if (USE_IBM_JAVA_PACKAGES) {
       options.put("useKeytab",
           keytab_alice.getAbsolutePath().startsWith("file://")
             ? keytab_alice.getAbsolutePath()