AMBARI-24742. Encrypting/decrypting PASSWORD type properties when inserting them into the DB/using them

ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java

@@ -2572,6 +2572,9 @@ public class Configuration {
   @Markdown(description = "Whether security password encryption is enabled or not. In case it is we store passwords in their own file(s); otherwise we store passwords in the Ambari credential store.")
   public static final ConfigurationProperty<Boolean> SECURITY_PASSWORD_ENCRYPTON_ENABLED = new ConfigurationProperty<>("security.passwords.encryption.enabled", false);
 
+  @Markdown(description="Whether to encrypt sensitive data (at rest) on service level configuration.")
+  public static final ConfigurationProperty<Boolean> SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED = new ConfigurationProperty<>("security.server.encrypt_sensitive_data", false);
+
   /**
    * The maximum number of authentication attempts permitted to a local user. Once the number of failures reaches this limit the user will be locked out. 0 indicates unlimited failures
    */
@@ -5518,6 +5521,14 @@ public boolean isSecurityPasswordEncryptionEnabled() {
     return Boolean.parseBoolean(getProperty(SECURITY_PASSWORD_ENCRYPTON_ENABLED));
   }
 
+  public boolean isSensitiveDataEncryptionEnabled() {
+    return Boolean.parseBoolean(getProperty(SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED));
+  }
+
+  public boolean shouldEncryptSensitiveData() {
+    return isSecurityPasswordEncryptionEnabled() && isSensitiveDataEncryptionEnabled();
+  }
+
   /**
    * @return default value of number of tasks to run in parallel during upgrades
    */

ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java

@@ -115,9 +115,11 @@
 import org.apache.ambari.server.security.authorization.internal.InternalAuthenticationInterceptor;
 import org.apache.ambari.server.security.authorization.internal.RunWithInternalSecurityContext;
 import org.apache.ambari.server.security.encryption.AESEncryptionService;
+import org.apache.ambari.server.security.encryption.ConfigPropertiesEncryptor;
 import org.apache.ambari.server.security.encryption.CredentialStoreService;
 import org.apache.ambari.server.security.encryption.CredentialStoreServiceImpl;
 import org.apache.ambari.server.security.encryption.EncryptionService;
+import org.apache.ambari.server.security.encryption.Encryptor;
 import org.apache.ambari.server.serveraction.kerberos.KerberosOperationHandlerFactory;
 import org.apache.ambari.server.serveraction.users.CollectionPersisterService;
 import org.apache.ambari.server.serveraction.users.CollectionPersisterServiceFactory;
@@ -179,6 +181,7 @@
 import com.google.inject.AbstractModule;
 import com.google.inject.Scopes;
 import com.google.inject.Singleton;
+import com.google.inject.TypeLiteral;
 import com.google.inject.assistedinject.FactoryModuleBuilder;
 import com.google.inject.name.Names;
 import com.google.inject.persist.PersistModule;
@@ -332,6 +335,8 @@ protected void configure() {
 
     bind(CredentialStoreService.class).to(CredentialStoreServiceImpl.class);
     bind(EncryptionService.class).to(AESEncryptionService.class);
+    //to support different Encryptor implementation we have to annotate them by their name and use them as @Named injects
+    bind(new TypeLiteral<Encryptor<Config>>() {}).annotatedWith(Names.named("ConfigPropertiesEncryptor")).to(ConfigPropertiesEncryptor.class);
 
     bind(Configuration.class).toInstance(configuration);
     bind(OsFamily.class).toInstance(os_family);

ambari-server/src/main/java/org/apache/ambari/server/security/encryption/AESEncryptionService.java

@@ -24,6 +24,8 @@
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.binary.Hex;
 
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import com.google.inject.Singleton;
 
 @Singleton
@@ -32,6 +34,8 @@ public class AESEncryptionService implements EncryptionService {
   private static final String ENCODED_TEXT_FIELD_DELIMITER = "::";
   private static final String UTF_8_CHARSET = StandardCharsets.UTF_8.name();
 
+  private final Cache<String, AESEncryptor> aesEncryptorCache = CacheBuilder.newBuilder().build();
+
   private MasterKeyService environmentMasterKeyService;
 
   @Override
@@ -51,11 +55,20 @@ public String encrypt(String toBeEncrypted, String key) throws Exception {
 
   @Override
   public String encrypt(String toBeEncrypted, String key, TextEncoding textEncoding) throws Exception {
-    final AESEncryptor aes = new AESEncryptor(key);
-    final EncryptionResult encryptionResult = aes.encrypt(toBeEncrypted);
+    final EncryptionResult encryptionResult = getAesEncryptor(key).encrypt(toBeEncrypted);
     return TextEncoding.BASE_64 == textEncoding ? encodeEncryptionResultBase64(encryptionResult) : encodeEncryptionResultBinHex(encryptionResult);
   }
 
+  private AESEncryptor getAesEncryptor(String key) {
+    AESEncryptor aesEncryptor = aesEncryptorCache.getIfPresent(key);
+    if (aesEncryptor == null) {
+      aesEncryptor = new AESEncryptor(key);
+      aesEncryptorCache.put(key, aesEncryptor);
+    }
+
+    return aesEncryptor;
+  }
+
   private final String getAmbariMasterKey() {
     initEnvironmentMasterKeyService();
     return String.valueOf(environmentMasterKeyService.getMasterSecret());
@@ -100,7 +113,7 @@ public String decrypt(String toBeDecrypted, String key, TextEncoding textEncodin
     final byte[] decodedValue = TextEncoding.BASE_64 == textEncoding ? Base64.decodeBase64(toBeDecrypted) : Hex.decodeHex(toBeDecrypted.toCharArray());
     final String decodedText = new String(decodedValue, UTF_8_CHARSET);
     final String[] decodedParts = decodedText.split(ENCODED_TEXT_FIELD_DELIMITER);
-    final AESEncryptor aes = new AESEncryptor(key);
+    final AESEncryptor aes = getAesEncryptor(key);
     if (TextEncoding.BASE_64 == textEncoding) {
       return new String(aes.decrypt(Base64.decodeBase64(decodedParts[0]), Base64.decodeBase64(decodedParts[1]), Base64.decodeBase64(decodedParts[2])), UTF_8_CHARSET);
     } else {

ambari-server/src/main/java/org/apache/ambari/server/security/encryption/ConfigPropertiesEncryptor.java

@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.security.encryption;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.AmbariRuntimeException;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.PropertyInfo.PropertyType;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.utils.TextEncoding;
+import org.apache.commons.collections.CollectionUtils;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+/**
+ * {@link Encryptor} implementation for encrypting/decrypting PASSWORD type
+ * properties in {@link Config}'s properties
+ */
+
+@Singleton
+public class ConfigPropertiesEncryptor implements Encryptor<Config> {
+
+  private static final String ENCRYPTED_PROPERTY_PREFIX = "${enc=aes256_hex, value=";
+  private static final String ENCRYPTED_PROPERTY_SCHEME = ENCRYPTED_PROPERTY_PREFIX + "%s}";
+
+  private final EncryptionService encryptionService;
+  private final Map<Long, Map<StackId, Map<String, Set<String>>>> clusterPasswordProperties = new ConcurrentHashMap<>(); //Map<clusterId, <Map<stackId, Map<configType, Set<passwordPropertyKeys>>>>;
+
+  @Inject
+  public ConfigPropertiesEncryptor(EncryptionService encryptionService) {
+    this.encryptionService = encryptionService;
+  }
+
+  @Override
+  public void encryptSensitiveData(Config config) {
+    try {
+      final Map<String, String> configProperties = config.getProperties();
+      if (configProperties != null) {
+        final Set<String> passwordProperties = getPasswordProperties(config.getCluster(), config.getType());
+        if (CollectionUtils.isNotEmpty(passwordProperties)) {
+          final Map<String, String> encryptedProperties = new HashMap<>(configProperties);
+          for (Map.Entry<String, String> property : configProperties.entrySet()) {
+            if (passwordProperties.contains(property.getKey()) && !isEncryptedPassword(property.getValue())) {
+              encryptedProperties.put(property.getKey(), encryptAndDecoratePropertyValue(property.getValue()));
+            }
+          }
+          config.setProperties(encryptedProperties);
+        }
+      }
+    } catch (Exception e) {
+      throw new AmbariRuntimeException("Error while encrypting sensitive data", e);
+    }
+  }
+
+  private boolean isEncryptedPassword(String password) {
+    return password != null && password.startsWith(ENCRYPTED_PROPERTY_PREFIX); // assuming previous encryption by this class
+  }
+
+  private Set<String> getPasswordProperties(Cluster cluster, String configType) throws AmbariException {
+    //in case of normal configuration change on the UI - or via the API - the current and desired stacks are equal
+    //in case of an upgrade they are different; in this case we want to get password properties from the desired stack
+    if (cluster.getCurrentStackVersion().equals(cluster.getDesiredStackVersion())) {
+      return getPasswordProperties(cluster, cluster.getCurrentStackVersion(), configType);
+    } else {
+      return getPasswordProperties(cluster, cluster.getDesiredStackVersion(), configType);
+    }
+  }
+
+  private Set<String> getPasswordProperties(Cluster cluster, StackId stackId, String configType) {
+    final long clusterId = cluster.getClusterId();
+    clusterPasswordProperties.computeIfAbsent(clusterId, v -> new ConcurrentHashMap<>()).computeIfAbsent(stackId, v -> new ConcurrentHashMap<>())
+        .computeIfAbsent(configType, v -> cluster.getConfigPropertiesTypes(configType, stackId).getOrDefault(PropertyType.PASSWORD, new HashSet<>()));
+    return clusterPasswordProperties.get(clusterId).get(stackId).getOrDefault(configType, new HashSet<>());
+  }
+
+  private String encryptAndDecoratePropertyValue(String propertyValue) throws Exception {
+    final String encrypted = encryptionService.encrypt(propertyValue, TextEncoding.BIN_HEX);
+    return String.format(ENCRYPTED_PROPERTY_SCHEME, encrypted);
+  }
+
+  @Override
+  public void decryptSensitiveData(Config config) {
+    final Map<String, String> configProperties = config.getProperties();
+    if (configProperties != null) {
+      final Map<String, String> decryptedProperties = new HashMap<>(configProperties);
+      for (Map.Entry<String, String> property : configProperties.entrySet()) {
+        if (isEncryptedPassword(property.getValue())) {
+          decryptedProperties.put(property.getKey(), decryptProperty(property.getValue()));
+        }
+      }
+      config.setProperties(decryptedProperties);
+    }
+  }
+
+  private String decryptProperty(String property) {
+    try {
+      // sample value: ${enc=aes256_hex, value=5248...303d}
+      final String encrypted = property.substring(ENCRYPTED_PROPERTY_PREFIX.length(), property.indexOf('}'));
+      return encryptionService.decrypt(encrypted, TextEncoding.BIN_HEX);
+    } catch (Exception e) {
+      throw new AmbariRuntimeException("Error while decrypting property", e);
+    }
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/security/encryption/Encryptor.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.security.encryption;
+
+/**
+ * Defines a generic contract on encrypting/decrypting sensitive data
+ */
+public interface Encryptor<T> {
+
+  /**
+   * Encrypts the given encryptible object
+   * 
+   * @param encryptible
+   *          to be encrypted
+   * @return the encrypted value
+   */
+  void encryptSensitiveData(T encryptible);
+
+  /**
+   * Decrypts the given decryptible object
+   * 
+   * @param decryptible
+   *          to be decrypted
+   * @return the decrypted value
+   */
+  void decryptSensitiveData(T decryptible);
+
+}

ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java

@@ -268,12 +268,20 @@ List<Host> transitionHostsToInstalling(RepositoryVersionEntity repoVersionEntity
   Map<String, Config> getConfigsByType(String configType);
 
   /**
-   * Gets all properties types that mach the specified type.
+   * Gets all properties types that matches the specified type for the current stack.
    * @param configType the config type to return
    * @return properties types for given config type
    */
   Map<PropertyInfo.PropertyType, Set<String>> getConfigPropertiesTypes(String configType);
 
+  /**
+   * Gets all properties types that matches the specified type for the given stack.
+   * @param configType the config type to return
+   * @param stackId the stack to scan properties for
+   * @return properties types for given config type
+   */
+  Map<PropertyInfo.PropertyType, Set<String>> getConfigPropertiesTypes(String configType, StackId stackId);
+
   /**
    * Gets the specific config that matches the specified type and tag.  This not
    * necessarily a DESIRED configuration that applies to a cluster.

ambari-server/src/main/java/org/apache/ambari/server/state/Config.java

@@ -97,4 +97,9 @@ public interface Config {
    * Persist the configuration.
    */
   void save();
+
+  /**
+   * @return the cluster where this config belongs to
+   */
+  Cluster getCluster();
 }