AMBARI-26238: Add Ambari Java Home configuration for JDK 17 in Ambari…

ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java

@@ -792,6 +792,15 @@ public class Configuration {
   public static final ConfigurationProperty<String> JAVA_HOME = new ConfigurationProperty<>(
       "java.home", null);
 
+  /**
+   * The location of the JDK on the Ambari Agent hosts.
+   */
+  @Markdown(
+          description = "The location of the JDK on the Ambari Agent hosts. This is only used by Ambari Server",
+          examples = { "/usr/jdk64/jdk1.8.0_112" })
+  public static final ConfigurationProperty<String> AMBARI_JAVA_HOME = new ConfigurationProperty<>(
+          "ambari.java.home", null);
+
   /**
    * The name of the JDK installation binary.
    */
@@ -4115,6 +4124,10 @@ public String getJavaHome() {
     return getProperty(JAVA_HOME);
   }
 
+  public String getAmbariJavaHome() {
+    return getProperty(AMBARI_JAVA_HOME);
+  }
+
   public String getJDKName() {
     return getProperty(JDK_NAME);
   }

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java

@@ -718,6 +718,11 @@ RequestStageContainer addStages(RequestStageContainer requestStages, Cluster clu
    */
   String getJavaHome();
 
+  /**
+   * Getter for the java home, stored in ambari.properties
+   */
+  String getAmbariJavaHome();
+
   /**
    * Getter for the jdk name, stored in ambari.properties
    */

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java

@@ -24,6 +24,7 @@
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_PASSWORD;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_URL;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_USERNAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_JAVA_HOME;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLUSTER_NAME;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_RETRY_ENABLED;
@@ -403,6 +404,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
 
   final private String jdkResourceUrl;
   final private String javaHome;
+  final private String ambariJavaHome;
   final private String jdkName;
   final private String jceName;
   final private String ojdbcUrl;
@@ -447,6 +449,7 @@ public AmbariManagementControllerImpl(ActionManager actionManager,
       }
       jdkResourceUrl = getAmbariServerURI(JDK_RESOURCE_LOCATION);
       javaHome = configs.getJavaHome();
+      ambariJavaHome = configs.getAmbariJavaHome();
       jdkName = configs.getJDKName();
       jceName = configs.getJCEName();
       ojdbcUrl = getAmbariServerURI(JDK_RESOURCE_LOCATION + "/" + configs.getOjdbcJarName());
@@ -459,6 +462,7 @@ public AmbariManagementControllerImpl(ActionManager actionManager,
 
       jdkResourceUrl = null;
       javaHome = null;
+      ambariJavaHome = null;
       jdkName = null;
       jceName = null;
       ojdbcUrl = null;
@@ -5091,6 +5095,11 @@ public String getJavaHome() {
     return javaHome;
   }
 
+  @Override
+  public String getAmbariJavaHome() {
+    return ambariJavaHome;
+  }
+
   @Override
   public String getJDKName() {
     return jdkName;
@@ -6019,6 +6028,7 @@ public TreeMap<String, String> getMetadataAmbariLevelParams() {
     TreeMap<String, String> clusterLevelParams = new TreeMap<>();
     clusterLevelParams.put(JDK_LOCATION, getJdkResourceUrl());
     clusterLevelParams.put(JAVA_HOME, getJavaHome());
+    clusterLevelParams.put(AMBARI_JAVA_HOME, getAmbariJavaHome());
     clusterLevelParams.put(JAVA_VERSION, String.valueOf(configs.getJavaVersion()));
     clusterLevelParams.put(JDK_NAME, getJDKName());
     clusterLevelParams.put(JCE_NAME, getJCEName());

ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java

@@ -579,7 +579,7 @@ private static String getRangedItem(Integer startOfRange, Integer endOfRange) {
    */
   public static void useAmbariJdkInCommandParams(Map<String, String> commandParams, Configuration configuration) {
     if (StringUtils.isNotEmpty(configuration.getJavaHome()) && !configuration.getJavaHome().equals(configuration.getStackJavaHome())) {
-      commandParams.put(AMBARI_JAVA_HOME, configuration.getJavaHome());
+      commandParams.put(AMBARI_JAVA_HOME, configuration.getAmbariJavaHome());
       commandParams.put(AMBARI_JAVA_VERSION, String.valueOf(configuration.getJavaVersion()));
       if (StringUtils.isNotEmpty(configuration.getJDKName())) { // if not custom jdk
         commandParams.put(AMBARI_JDK_NAME, configuration.getJDKName());

ambari-server/src/main/python/ambari-server.py

@@ -490,6 +490,8 @@ def init_setup_parser_options(parser):
 
   other_group.add_option('-j', '--java-home', default=None,
                          help="Use specified java_home.  Must be valid on all hosts")
+  other_group.add_option('--ambari-java-home',dest="ambari_java_home", default=None,
+                         help="Use specified java_home for ambari.  Must be valid on Ambari server hosts")
   other_group.add_option('--stack-java-home', dest="stack_java_home", default=None,
                     help="Use specified java_home for stack services.  Must be valid on all hosts")
   other_group.add_option('--skip-view-extraction', action="store_true", default=False, help="Skip extraction of system views", dest="skip_view_extraction")

ambari-server/src/main/python/ambari_server/serverConfiguration.py

@@ -24,7 +24,7 @@
 import re
 import shutil
 import stat
-import string
+from enum import Enum
 import sys
 import tempfile
 import getpass
@@ -94,6 +94,12 @@
 STACK_JCE_NAME_PROPERTY = "stack.jce.name"
 STACK_JAVA_VERSION = "stack.java.version"
 
+#ambari JDK
+AMBARI_JAVA_HOME_PROPERTY = "ambari.java.home"
+AMBARI_JDK_NAME_PROPERTY = "ambari.jdk.name"
+AMBARI_JCE_NAME_PROPERTY = "ambari.jce.name"
+AMBARI_JAVA_VERSION = "ambari.java.version"
+
 
 #TODO property used incorrectly in local case, it was meant to be dbms name, not postgres database name,
 # has workaround for now, as we don't need dbms name if persistence_type=local
@@ -178,6 +184,11 @@
 # JDK
 JDK_RELEASES="java.releases"
 
+class JavaHomeType(Enum):
+  GLOBAL = JAVA_HOME_PROPERTY
+  AMBARI = AMBARI_JAVA_HOME_PROPERTY
+
+
 if on_powerpc():
   JDK_RELEASES += ".ppc64le"
 
@@ -1423,13 +1434,12 @@ def __load_properties(properties, section_name):
     return (desc, url, dest_file, jcpol_url, jcpol_file, inst_dir, reg_exp)
   pass
 
-def get_JAVA_HOME():
+def get_JAVA_HOME(java_home_type=JavaHomeType.AMBARI):
   properties = get_ambari_properties()
   if properties == -1:
     print_error_msg("Error getting ambari properties")
     return None
-
-  java_home = properties[JAVA_HOME_PROPERTY]
+  java_home = properties[java_home_type.value]
 
   if (not 0 == len(java_home)) and (os.path.exists(java_home)):
     return java_home
@@ -1450,8 +1460,8 @@ def validate_jdk(jdk_path):
 #
 # Finds the available JDKs.
 #
-def find_jdk():
-  jdkPath = get_JAVA_HOME()
+def find_jdk(java_home_type=JavaHomeType.AMBARI):
+  jdkPath = get_JAVA_HOME(java_home_type)
   if jdkPath:
     if validate_jdk(jdkPath):
       return jdkPath
@@ -1471,8 +1481,8 @@ def find_jdk():
       print_error_msg ("JDK {0} is invalid".format(jdkPath))
   return
 
-def get_java_exe_path():
-  jdkPath = find_jdk()
+def get_java_exe_path(java_home_type=JavaHomeType.AMBARI):
+  jdkPath = find_jdk(java_home_type)
   if jdkPath:
     java_exe = os.path.join(jdkPath, configDefaults.JAVA_EXE_SUBPATH)
     return java_exe

ambari-server/src/main/python/ambari_server/serverSetup.py

@@ -42,7 +42,7 @@
   JAVA_HOME, JAVA_HOME_PROPERTY, JCE_NAME_PROPERTY, JDBC_RCA_URL_PROPERTY, JDBC_URL_PROPERTY, \
   JDK_NAME_PROPERTY, JDK_RELEASES, NR_USER_PROPERTY, OS_FAMILY, OS_FAMILY_PROPERTY, OS_TYPE, OS_TYPE_PROPERTY, OS_VERSION, \
   VIEWS_DIR_PROPERTY, JDBC_DATABASE_PROPERTY, JDK_DOWNLOAD_SUPPORTED_PROPERTY, JCE_DOWNLOAD_SUPPORTED_PROPERTY, SETUP_DONE_PROPERTIES, \
-  STACK_JAVA_HOME_PROPERTY, STACK_JDK_NAME_PROPERTY, STACK_JCE_NAME_PROPERTY, STACK_JAVA_VERSION, GPL_LICENSE_ACCEPTED_PROPERTY
+  STACK_JAVA_HOME_PROPERTY, STACK_JDK_NAME_PROPERTY, STACK_JCE_NAME_PROPERTY, STACK_JAVA_VERSION, GPL_LICENSE_ACCEPTED_PROPERTY, AMBARI_JAVA_HOME_PROPERTY
 from ambari_server.serverUtils import is_server_runing
 from ambari_server.setupSecurity import adjust_directory_permissions
 from ambari_server.userInput import get_YN_input, get_validated_string_input
@@ -414,6 +414,21 @@ def download_and_install_jdk(self, args, properties, ambariOnly = False):
     jcePolicyWarn = "JCE Policy files are required for configuring Kerberos security. If you plan to use Kerberos," \
                     "please make sure JCE Unlimited Strength Jurisdiction Policy Files are valid on all hosts."
 
+    if args.ambari_java_home:
+      print('start setting AMBARI_JAVA_HOME for Ambari...')
+      if not validate_jdk(args.ambari_java_home):
+        err = "Path to Ambari java home " + args.ambari_java_home + " or java binary file does not exist"
+        raise FatalException(1, err)
+
+      print_warning_msg("AMBARI_JAVA_HOME " + args.ambari_java_home + " must be valid on ALL hosts")
+      print_warning_msg(jcePolicyWarn)
+
+      properties.process_pair(AMBARI_JAVA_HOME_PROPERTY, args.ambari_java_home)
+      properties.removeOldProp(JDK_NAME_PROPERTY)
+      properties.removeOldProp(JCE_NAME_PROPERTY)
+      print('Setting AMBARI_JAVA_HOME for Ambari finished')
+
+
     if args.java_home:
       #java_home was specified among the command-line arguments. Use it as custom JDK location.
       if not validate_jdk(args.java_home):

ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/3.0.0/package/scripts/infra_solr.py

@@ -134,8 +134,8 @@ def disable_security(self, env):
       return
     zkmigrator = ZkMigrator(
       zk_host=params.zk_quorum,
-      java_exec=params.java_exec,
-      java_home=params.java64_home,
+      java_exec=params.ambari_java_exec,
+      java_home=params.ambari_java_home,
       jaas_file=params.infra_solr_jaas_file,
       user=params.infra_solr_user)
     zkmigrator.set_acls(params.infra_solr_znode, 'world:anyone:crdwa')

ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/3.0.0/package/scripts/params.py

@@ -71,7 +71,8 @@ def get_name_from_principal(principal):
 # shared configs
 java_home = config['ambariLevelParams']['java_home']
 ambari_java_home = default("/ambariLevelParams/ambari_java_home", None)
-java64_home = ambari_java_home if ambari_java_home is not None else java_home
+ambari_java_exec = f"{ambari_java_home}/bin/java"
+java64_home = java_home
 java_exec = format("{java64_home}/bin/java")
 zookeeper_hosts_list = config['clusterHostInfo']['zookeeper_server_hosts']
 zookeeper_hosts_list.sort()

ambari-server/src/main/resources/common-services/AMBARI_METRICS/3.0.0/configuration/ams-env.xml

@@ -185,13 +185,13 @@ export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}
 export AMS_HBASE_INIT_CHECK_ENABLED={{ams_hbase_init_check_enabled}}
 
 # AMS Collector options
-export AMS_COLLECTOR_OPTS="-Djava.library.path=/usr/lib/ams-hbase/lib/hadoop-native"
+export AMS_COLLECTOR_OPTS="-Djava.library.path=/usr/lib/ams-hbase/lib/hadoop-native --add-opens java.base/java.lang=ALL-UNNAMED"
 {% if security_enabled %}
 export AMS_COLLECTOR_OPTS="$AMS_COLLECTOR_OPTS -Dzookeeper.sasl.client.username={{zk_principal_user}} -Djava.security.auth.login.config={{ams_collector_jaas_config_file}}"
 {% endif %}
 
 # AMS Collector GC options
-export AMS_COLLECTOR_GC_OPTS="-XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{ams_collector_log_dir}}/collector-gc.log-`date +'%Y%m%d%H%M'`"
+export AMS_COLLECTOR_GC_OPTS=" -verbose:gc -XX:+PrintGCDetails  -Xloggc:{{ams_collector_log_dir}}/collector-gc.log-`date +'%Y%m%d%H%M'`"
 export AMS_COLLECTOR_OPTS="$AMS_COLLECTOR_OPTS $AMS_COLLECTOR_GC_OPTS"
 
 # Metrics collector host will be blacklisted for specified number of seconds if metric monitor failed to connect to it.

ambari-server/src/main/resources/common-services/AMBARI_METRICS/3.0.0/configuration/ams-hbase-env.xml

@@ -201,7 +201,7 @@ The timeout is introduced because there is a known bug when from time to time HB
 # Set environment variables here.
 
 # The java implementation to use. Java 1.6+ required.
-export JAVA_HOME={{java64_home}}
+export JAVA_HOME={{java_home}}
 
 # HBase Configuration directory
 export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}

ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HIVE/package/scripts/hive_service.py

@@ -150,7 +150,7 @@ def validate_connection(target_path_to_jdbc, hive_lib_path):
       Logger.error(error_message)
 
   db_connection_check_command = format(
-    "{java64_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")
+    "{ambari_java_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")
 
   try:
     Execute(db_connection_check_command,

ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/HIVE/package/scripts/params.py

@@ -376,7 +376,8 @@
 hive_metastore_heapsize = config['configurations']['hive-env']['hive.metastore.heapsize']
 
 java64_home = config['ambariLevelParams']['java_home']
-java_exec = format("{java64_home}/bin/java")
+ambari_java_home = config['ambariLevelParams']['ambari_java_home']
+ambari_java_exec = format("{ambari_java_home}/bin/java")
 java_version = expect("/ambariLevelParams/java_version", int)
 
 ##### MYSQL

ambari-server/src/main/resources/stacks/BIGTOP/3.3.0/services/RANGER/package/scripts/params.py

@@ -117,6 +117,8 @@
 usersync_start = format('{usersync_services_file} start')
 
 java_home = config['ambariLevelParams']['java_home']
+ambari_java_home = config['ambariLevelParams']['ambari_java_home']
+ambari_java_exec = format("{ambari_java_home}/bin/java")
 unix_user  = config['configurations']['ranger-env']['ranger_user']
 unix_group = config['configurations']['ranger-env']['ranger_group']
 ranger_pid_dir = default("/configurations/ranger-env/ranger_pid_dir", "/var/run/ranger")

ambari-server/src/main/resources/stacks/BIGTOP/3.3.0/services/RANGER/package/scripts/setup_ranger_xml.py

@@ -86,7 +86,7 @@ def setup_ranger_admin(upgrade_type=None):
   cp = cp + os.pathsep + format("{ranger_home}/ews/lib/*")
 
   db_connection_check_command = format(
-    "{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_jdbc_connection_url}' {ranger_db_user} {ranger_db_password!p} {ranger_jdbc_driver}")
+    "{ambari_java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_jdbc_connection_url}' {ranger_db_user} {ranger_db_password!p} {ranger_jdbc_driver}")
 
   env_dict = {}
   if params.db_flavor.lower() == 'sqla':

ambari-server/src/main/resources/stacks/BIGTOP/3.3.0/services/RANGER_KMS/package/scripts/kms.py

@@ -204,7 +204,7 @@ def kms(upgrade_type=None):
       cp = cp + os.pathsep + path_to_jdbc
 
     db_connection_check_command = format(
-      "{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_kms_jdbc_connection_url}' {db_user} {db_password!p} {ranger_kms_jdbc_driver}")
+      "{ambari_java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_kms_jdbc_connection_url}' {db_user} {db_password!p} {ranger_kms_jdbc_driver}")
 
     env_dict = {}
     if params.db_flavor.lower() == 'sqla':

ambari-server/src/main/resources/stacks/BIGTOP/3.3.0/services/RANGER_KMS/package/scripts/params.py

@@ -67,6 +67,8 @@
 kms_lib_path = format('{kms_home}/ews/lib/')
 kms_log_dir = default("/configurations/kms-env/kms_log_dir", "/var/log/ranger/kms")
 java_home = config['ambariLevelParams']['java_home']
+ambari_java_home = config['ambariLevelParams']['ambari_java_home']
+ambari_java_exec = format("{ambari_java_home}/bin/java")
 kms_user  = default("/configurations/kms-env/kms_user", "kms")
 kms_group = default("/configurations/kms-env/kms_group", "kms")
 

ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java

@@ -597,10 +597,11 @@ public void testUseAmbariJdkWithCustomJavaHome() {
     Map<String, String> commandParams = new HashMap<>();
     Configuration configuration = new Configuration();
     configuration.setProperty("java.home", "myJavaHome");
+    configuration.setProperty("ambari.java.home", "ambari_java_home");
     // WHEN
     StageUtils.useAmbariJdkInCommandParams(commandParams, configuration);
     // THEN
-    assertEquals("myJavaHome", commandParams.get("ambari_java_home"));
+    assertEquals("ambari_java_home", commandParams.get("ambari_java_home"));
     assertEquals(2, commandParams.size());
   }
 
@@ -612,10 +613,11 @@ public void testUseAmbariJdk() {
     configuration.setProperty("java.home", "myJavaHome");
     configuration.setProperty("jdk.name", "myJdkName");
     configuration.setProperty("jce.name", "myJceName");
+    configuration.setProperty("ambari.java.home", "ambari_java_home");
     // WHEN
     StageUtils.useAmbariJdkInCommandParams(commandParams, configuration);
     // THEN
-    assertEquals("myJavaHome", commandParams.get("ambari_java_home"));
+    assertEquals("ambari_java_home", commandParams.get("ambari_java_home"));
     assertEquals("myJdkName", commandParams.get("ambari_jdk_name"));
     assertEquals("myJceName", commandParams.get("ambari_jce_name"));
     assertEquals(4, commandParams.size());

ambari-server/src/test/python/TestAmbariServer.py

@@ -2916,6 +2916,7 @@ def _init_test_jdk_mocks():
 
     # Successful JDK download
     args.java_home = None
+    args.ambari_java_home = None
     validate_jdk_mock.return_value = False
     path_existsMock.reset_mock()
     path_existsMock.side_effect = [False, False, False]