AMBARI-24745. Enable encryption of sensitive data in Ambari DB using …

ambari-server/src/main/python/ambari-server.py

@@ -54,7 +54,7 @@
 from ambari_server.setupHttps import setup_https, setup_truststore
 from ambari_server.setupMpacks import install_mpack, uninstall_mpack, upgrade_mpack, STACK_DEFINITIONS_RESOURCE_NAME, \
   SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME
-from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas, setup_pam, \
+from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_sensitive_data_encryption, setup_ambari_krb5_jaas, setup_pam, \
   migrate_ldap_pam, LDAP_TYPES
 from ambari_server.setupSso import setup_sso
 from ambari_server.setupTrustedProxy import setup_trusted_proxy
@@ -253,7 +253,7 @@ def refresh_stack_hash_action():
 def create_setup_security_actions(args):
   action_list = [
       ['setup-https', 'Enable HTTPS for Ambari server.', UserActionRestart(setup_https, args)],
-      ['encrypt-passwords', 'Encrypt passwords stored in ambari.properties file.', UserAction(setup_master_key, args)],
+      ['encrypt-passwords', 'Encrypt passwords managed by Ambari.', UserAction(setup_sensitive_data_encryption, args)],
       ['setup-kerberos-jaas', 'Setup Ambari kerberos JAAS configuration.', UserAction(setup_ambari_krb5_jaas, args)],
       ['setup-truststore', 'Setup truststore.', UserActionRestart(setup_truststore, args)],
       ['import-certificate', 'Import certificate to truststore.', UserActionRestart(setup_truststore, True, args)],
@@ -264,7 +264,7 @@ def create_setup_security_actions(args):
 def create_setup_security_actions(args):
   action_list = [
       ['setup-https', 'Enable HTTPS for Ambari server.', UserActionRestart(setup_https, args)],
-      ['encrypt-passwords', 'Encrypt passwords stored in ambari.properties file.', UserAction(setup_master_key, args)],
+      ['encrypt-passwords', 'Encrypt passwords managed by Ambari.', UserAction(setup_sensitive_data_encryption, args)],
       ['setup-kerberos-jaas', 'Setup Ambari kerberos JAAS configuration.', UserAction(setup_ambari_krb5_jaas, args)],
       ['setup-truststore', 'Setup truststore.', UserActionRestart(setup_truststore, args)],
       ['import-certificate', 'Import certificate to truststore.', UserActionRestart(setup_truststore, args, True)],

ambari-server/src/main/python/ambari_server/serverConfiguration.py

@@ -608,6 +608,7 @@ def __init__(self):
 SECURITY_KEY_ENV_VAR_NAME = "AMBARI_SECURITY_MASTER_KEY"
 SECURITY_MASTER_KEY_FILENAME = "master"
 SECURITY_IS_ENCRYPTION_ENABLED = "security.passwords.encryption.enabled"
+SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED = "security.server.encrypt_sensitive_data"
 SECURITY_KERBEROS_JASS_FILENAME = "krb5JAASLogin.conf"
 
 SECURITY_PROVIDER_GET_CMD = "{0} -cp {1} " + \
@@ -625,6 +626,11 @@ def __init__(self):
                             ".MasterKeyServiceImpl {2} {3} {4} " + \
                             "> " + configDefaults.SERVER_OUT_FILE + " 2>&1"
 
+SECURITY_SENSITIVE_DATA_ENCRYPTON_CMD = "{0} -cp {1} " + \
+                            "org.apache.ambari.server.security.encryption.SensitiveDataEncryption" + \
+                            " {2} " + \
+                            "> " + configDefaults.SERVER_OUT_FILE + " 2>&1"
+
 
 
 def read_ambari_user():
@@ -1045,7 +1051,7 @@ def get_original_master_key(properties, options = None):
           masterKey = options.master_key
         if not masterKey:
           masterKey = get_validated_string_input('Enter current Master Key: ',
-                                                 "", ".*", "", True, False)
+                                                 "", ".*", "", True, True)
           if options is not None:
             options.master_key = masterKey
       except KeyboardInterrupt:

ambari-server/src/main/python/ambari_server/setupSecurity.py

@@ -47,8 +47,8 @@
   BLIND_PASSWORD, BOOTSTRAP_DIR_PROPERTY, JDBC_PASSWORD_FILENAME, JDBC_PASSWORD_PROPERTY, \
   JDBC_RCA_PASSWORD_ALIAS, JDBC_RCA_PASSWORD_FILE_PROPERTY, JDBC_USE_INTEGRATED_AUTH_PROPERTY, \
   LDAP_MGR_PASSWORD_ALIAS, LDAP_MGR_PASSWORD_PROPERTY, CLIENT_SECURITY, \
-  SECURITY_IS_ENCRYPTION_ENABLED, SECURITY_KEY_ENV_VAR_NAME, SECURITY_KERBEROS_JASS_FILENAME, \
-  SECURITY_PROVIDER_KEY_CMD, SECURITY_MASTER_KEY_FILENAME, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
+  SECURITY_IS_ENCRYPTION_ENABLED, SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED, SECURITY_KEY_ENV_VAR_NAME, SECURITY_KERBEROS_JASS_FILENAME, \
+  SECURITY_PROVIDER_KEY_CMD, SECURITY_SENSITIVE_DATA_ENCRYPTON_CMD, SECURITY_MASTER_KEY_FILENAME, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
   SSL_TRUSTSTORE_PASSWORD_PROPERTY, SSL_TRUSTSTORE_PATH_PROPERTY, SSL_TRUSTSTORE_TYPE_PROPERTY, \
   JDK_NAME_PROPERTY, JCE_NAME_PROPERTY, JAVA_HOME_PROPERTY, \
   get_resources_location, SECURITY_MASTER_KEY_LOCATION, SETUP_OR_UPGRADE_MSG, \
@@ -487,9 +487,24 @@ def sync_ldap(options):
   sys.stdout.write('\n')
   sys.stdout.flush()
 
-def setup_master_key(options):
+def sensitive_data_encryption(options, direction, masterKey=None):
+  environ = os.environ.copy()
+  if masterKey:
+    environ[SECURITY_KEY_ENV_VAR_NAME] = masterKey
+  jdk_path = find_jdk()
+  if jdk_path is None:
+    print_error_msg("No JDK found, please run the \"setup\" "
+                    "command to install a JDK automatically or install any "
+                    "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
+    return 1
+  serverClassPath = ServerClassPath(get_ambari_properties(), options)
+  command = SECURITY_SENSITIVE_DATA_ENCRYPTON_CMD.format(get_java_exe_path(), serverClassPath.get_full_ambari_classpath_escaped_for_shell(), direction)
+  (retcode, stdout, stderr) = run_os_command(command, environ)
+  pass
+
+def setup_sensitive_data_encryption(options):
   if not is_root():
-    warn = 'ambari-server setup-https is run as ' \
+    warn = 'ambari-server encrypt-passwords is run as ' \
           'non-root user, some sudo privileges might be required'
     print warn
 
@@ -516,76 +531,67 @@ def setup_master_key(options):
 
   ts_password = properties.get_property(SSL_TRUSTSTORE_PASSWORD_PROPERTY)
   resetKey = False
+  decrypt = False
   masterKey = None
 
   if isSecure:
     print "Password encryption is enabled."
-    resetKey = True if options.security_option is not None else get_YN_input("Do you want to reset Master Key? [y/n] (n): ", False)
-
-  # For encrypting of only unencrypted passwords without resetting the key ask
-  # for master key if not persisted.
-  if isSecure and not isPersisted and not resetKey:
-    print "Master Key not persisted."
-    masterKey = get_original_master_key(properties, options)
-  pass
+    decrypt = get_YN_input("Do you want to decrypt passwords managed by Ambari? [y/n] (n): ", False)
+    if not decrypt:
+      resetKey = get_YN_input("Do you want to reset Master Key? [y/n] (n): ", False)
 
   # Make sure both passwords are clear-text if master key is lost
-  if resetKey:
-    if not isPersisted:
+  if resetKey or decrypt:
+    if isPersisted:
+      sensitive_data_encryption(options, "decryption")
+    else:
       print "Master Key not persisted."
       masterKey = get_original_master_key(properties, options)
       # Unable get the right master key or skipped question <enter>
       if not masterKey:
-        print "To disable encryption, do the following:"
-        print "- Edit " + find_properties_file() + \
-              " and set " + SECURITY_IS_ENCRYPTION_ENABLED + " = " + "false."
-        err = "{0} is already encrypted. Please call {1} to store unencrypted" \
-              " password and call 'encrypt-passwords' again."
-        if db_sql_auth and db_password and is_alias_string(db_password):
-          print err.format('- Database password', "'" + SETUP_ACTION + "'")
-        if ts_password and is_alias_string(ts_password):
-          print err.format('TrustStore password', "'" + LDAP_SETUP_ACTION + "'")
-
+        # todo fix unreachable code
+        printManualDecryptionWarning(db_password, db_sql_auth, ts_password)
         return 1
-      pass
-    pass
-  pass
+      sensitive_data_encryption(options, "decryption", masterKey)
 
   # Read back any encrypted passwords
-  if db_sql_auth  and db_password and is_alias_string(db_password):
-    db_password = read_passwd_for_alias(JDBC_RCA_PASSWORD_ALIAS, masterKey)
-  if ts_password and is_alias_string(ts_password):
-    ts_password = read_passwd_for_alias(SSL_TRUSTSTORE_PASSWORD_ALIAS, masterKey)
-  # Read master key, if non-secure or reset is true
-  if resetKey or not isSecure:
-    masterKey = read_master_key(resetKey, options)
-    persist = get_YN_input("Do you want to persist master key. If you choose " \
-                           "not to persist, you need to provide the Master " \
-                           "Key while starting the ambari server as an env " \
-                           "variable named " + SECURITY_KEY_ENV_VAR_NAME + \
-                           " or the start will prompt for the master key."
-                           " Persist [y/n] (y)? ", True, options.master_key_persist)
-    if persist:
-      save_master_key(options, masterKey, get_master_key_location(properties) + os.sep +
-                      SECURITY_MASTER_KEY_FILENAME, persist)
-    elif not persist and masterKeyFile:
-      try:
-        os.remove(masterKeyFile)
-        print_info_msg("Deleting master key file at location: " + str(
-          masterKeyFile))
-      except Exception, e:
-        print 'ERROR: Could not remove master key file. %s' % e
-    # Blow up the credential store made with previous key, if any
-    store_file = get_credential_store_location(properties)
-    if os.path.exists(store_file):
-      try:
-        os.remove(store_file)
-      except:
-        print_warning_msg("Failed to remove credential store file.")
-      pass
-    pass
-  pass
+  db_password, ts_password = deryptPasswordsConfigs(db_password, db_sql_auth, masterKey, ts_password)
+  save_decrypted_ambari_properties(db_password, properties, ts_password)
+
 
+  if not decrypt:
+    if resetKey or not isSecure:
+      # Read master key and encrypt sensitive data, if non-secure or reset is true
+      masterKey, isPersisted = setup_master_key(masterKeyFile, options, properties, resetKey)
+    else:
+      if not isPersisted:
+        # For encrypting of only unencrypted passwords without resetting the key ask
+        # for master key if not persisted.
+        print "Master Key not persisted."
+        masterKey = get_original_master_key(properties, options)
+    encrypt_sensitive_data(db_password, masterKey, options, isPersisted, properties, ts_password)
+
+  # Since files for store and master are created we need to ensure correct
+  # permissions
+  ambari_user = read_ambari_user()
+  if ambari_user:
+    adjust_directory_permissions(ambari_user)
+  return 0
+
+
+def save_decrypted_ambari_properties(db_password, properties, ts_password):
+  propertyMap = {SECURITY_IS_ENCRYPTION_ENABLED: 'false'}
+  propertyMap[SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED] = 'false'
+  if db_password:
+    propertyMap[JDBC_PASSWORD_PROPERTY] = db_password
+    if properties.get_property(JDBC_RCA_PASSWORD_FILE_PROPERTY):
+      propertyMap[JDBC_RCA_PASSWORD_FILE_PROPERTY] = db_password
+  if ts_password:
+    propertyMap[SSL_TRUSTSTORE_PASSWORD_PROPERTY] = ts_password
+  update_properties_2(properties, propertyMap)
+
+
+def encrypt_sensitive_data(db_password, masterKey, options, persist, properties, ts_password):
   propertyMap = {SECURITY_IS_ENCRYPTION_ENABLED: 'true'}
   # Encrypt only un-encrypted passwords
   if db_password and not is_alias_string(db_password):
@@ -597,25 +603,71 @@ def setup_master_key(options):
       remove_password_file(JDBC_PASSWORD_FILENAME)
       if properties.get_property(JDBC_RCA_PASSWORD_FILE_PROPERTY):
         propertyMap[JDBC_RCA_PASSWORD_FILE_PROPERTY] = get_alias_string(JDBC_RCA_PASSWORD_ALIAS)
-  pass
 
   if ts_password and not is_alias_string(ts_password):
     retCode = save_passwd_for_alias(SSL_TRUSTSTORE_PASSWORD_ALIAS, ts_password, masterKey)
     if retCode != 0:
       print 'Failed to save secure TrustStore password.'
     else:
       propertyMap[SSL_TRUSTSTORE_PASSWORD_PROPERTY] = get_alias_string(SSL_TRUSTSTORE_PASSWORD_ALIAS)
-  pass
 
+  propertyMap[SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED] = 'true'
+  if persist:
+    sensitive_data_encryption(options, "encryption")
+  else:
+    sensitive_data_encryption(options, "encryption", masterKey)
   update_properties_2(properties, propertyMap)
 
-  # Since files for store and master are created we need to ensure correct
-  # permissions
-  ambari_user = read_ambari_user()
-  if ambari_user:
-    adjust_directory_permissions(ambari_user)
 
-  return 0
+def setup_master_key(masterKeyFile, options, properties, resetKey):
+  masterKey = read_master_key(resetKey, options)
+  persist = get_YN_input("Do you want to persist master key. If you choose " \
+                         "not to persist, you need to provide the Master " \
+                         "Key while starting the ambari server as an env " \
+                         "variable named " + SECURITY_KEY_ENV_VAR_NAME + \
+                         " or the start will prompt for the master key."
+                         " Persist [y/n] (y)? ", True, options.master_key_persist)
+  if persist:
+    save_master_key(options, masterKey, get_master_key_location(properties) + os.sep +
+                    SECURITY_MASTER_KEY_FILENAME, persist)
+  elif not persist and masterKeyFile:
+    try:
+      os.remove(masterKeyFile)
+      print_info_msg("Deleting master key file at location: " + str(
+        masterKeyFile))
+    except Exception, e:
+      print 'ERROR: Could not remove master key file. %s' % e
+  # Blow up the credential store made with previous key, if any
+  store_file = get_credential_store_location(properties)
+  if os.path.exists(store_file):
+    try:
+      os.remove(store_file)
+    except:
+      print_warning_msg("Failed to remove credential store file.")
+  return masterKey, persist
+
+
+def deryptPasswordsConfigs(db_password, db_sql_auth, masterKey, ts_password):
+  if db_sql_auth and db_password and is_alias_string(db_password):
+    db_password = read_passwd_for_alias(JDBC_RCA_PASSWORD_ALIAS, masterKey)
+  if ts_password and is_alias_string(ts_password):
+    ts_password = read_passwd_for_alias(SSL_TRUSTSTORE_PASSWORD_ALIAS, masterKey)
+  return db_password, ts_password
+
+
+def printManualDecryptionWarning(db_password, db_sql_auth, ts_password):
+  print "To disable encryption, do the following:"
+  print "- Edit " + find_properties_file() + \
+        " and set " + SECURITY_IS_ENCRYPTION_ENABLED + " = " + "false." + \
+        " and set " + SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED + " = " + "false." + \
+        " and set all passwords and sensitive data in service configs to right value."
+  err = "{0} is already encrypted. Please call {1} to store unencrypted" \
+        " password and call 'encrypt-passwords' again."
+  if db_sql_auth and db_password and is_alias_string(db_password):
+    print err.format('- Database password', "'" + SETUP_ACTION + "'")
+  if ts_password and is_alias_string(ts_password):
+    print err.format('TrustStore password', "'" + LDAP_SETUP_ACTION + "'")
+
 
 def setup_ambari_krb5_jaas(options):
   jaas_conf_file = search_file(SECURITY_KERBEROS_JASS_FILENAME, get_conf_dir())