Add deprecation warnings for ssl config fallback (#36847)

SSL configuration fallback has long been present in security but is a
source of confusion due to the behavior. Ultimately, we plan to remove
support for fallback in the next major version so this commit provides
deprecation warnings for the current line of stable releases.

Author: jaymode

docs/reference/settings/security-settings.asciidoc

@@ -1213,6 +1213,10 @@ through the list of URLs will continue until a successful connection is made.
 [[ssl-tls-settings]]
 ==== Default TLS/SSL settings
 
+deprecated[6.7.0, fallback to the default TLS/SSL settings is deprecated as it
+leads to ambiguity in what configuration is used. Fully configure each components'
+TLS/SSL settings as the default settings will be removed in 7.0]
+
 You can configure the following TLS/SSL settings in
 `elasticsearch.yml`. For more information, see
 {stack-ov}/encrypting-communications.html[Encrypting communications]. These

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/DefaultJDKTrustConfig.java

@@ -25,6 +25,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * This class represents a trust configuration that corresponds to the default trusted certificates of the JDK
@@ -69,16 +70,6 @@ public String toString() {
         return "JDK trusted certs";
     }
 
-    @Override
-    public boolean equals(Object o) {
-        return o == this;
-    }
-
-    @Override
-    public int hashCode() {
-        return System.identityHashCode(this);
-    }
-
     /**
      * Merges the default trust configuration with the provided {@link TrustConfig}
      * @param trustConfig the trust configuration to merge with
@@ -109,4 +100,17 @@ private KeyStore getSystemTrustStore() throws KeyStoreException, CertificateExce
         }
         return null;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DefaultJDKTrustConfig that = (DefaultJDKTrustConfig) o;
+        return Objects.equals(trustStorePassword, that.trustStorePassword);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(trustStorePassword);
+    }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfiguration.java

@@ -256,7 +256,9 @@ private static SecureString getDefaultTrustStorePassword(Settings settings) {
                         return trustStorePassword;
                     }
                 }
-                return systemTrustStorePassword;
+                // since we are in a try with resources block, we need to clone the value so it doesn't get
+                // cleared!
+                return systemTrustStorePassword.clone();
             }
         }
         return null;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettings.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.xpack.core.XPackSettings;
 
 import javax.net.ssl.TrustManagerFactory;
 
@@ -57,12 +58,12 @@ public class SSLConfigurationSettings {
     private static final String PKCS12_KEYSTORE_TYPE = "PKCS12";
 
     private static final Function<String, Setting<List<String>>> CIPHERS_SETTING_TEMPLATE = key -> Setting.listSetting(key, Collections
-            .emptyList(), Function.identity(), Property.NodeScope, Property.Filtered);
+            .emptyList(), Function.identity(), propertiesFromKey(key));
     public static final Setting<List<String>> CIPHERS_SETTING_PROFILES =  Setting.affixKeySetting("transport.profiles.",
             "xpack.security.ssl.cipher_suites", CIPHERS_SETTING_TEMPLATE);
 
     private static final Function<String,Setting<List<String>>> SUPPORTED_PROTOCOLS_TEMPLATE = key ->  Setting.listSetting(key,
-            Collections.emptyList(), Function.identity(), Property.NodeScope, Property.Filtered);
+            Collections.emptyList(), Function.identity(), propertiesFromKey(key));
     public static final Setting<List<String>> SUPPORTED_PROTOCOLS_PROFILES = Setting.affixKeySetting("transport.profiles.",
             "xpack.security.ssl.supported_protocols", SUPPORTED_PROTOCOLS_TEMPLATE) ;
 
@@ -82,7 +83,7 @@ public class SSLConfigurationSettings {
             "xpack.security.ssl.keystore.secure_key_password", X509KeyPairSettings.KEYSTORE_KEY_PASSWORD_TEMPLATE);
 
     private static final Function<String, Setting<Optional<String>>> TRUST_STORE_PATH_TEMPLATE = key -> new Setting<>(key, s -> null,
-            Optional::ofNullable, Property.NodeScope, Property.Filtered);
+            Optional::ofNullable, propertiesFromKey(key));
     public static final Setting<Optional<String>> TRUST_STORE_PATH_PROFILES = Setting.affixKeySetting("transport.profiles.",
             "xpack.security.ssl.truststore.path", TRUST_STORE_PATH_TEMPLATE);
 
@@ -96,7 +97,8 @@ public class SSLConfigurationSettings {
 
     private static final Function<String, Setting<SecureString>> TRUSTSTORE_PASSWORD_TEMPLATE  =  key ->
             SecureSetting.secureString(key, LEGACY_TRUSTSTORE_PASSWORD_TEMPLATE.apply(key.replace("truststore.secure_password",
-                    "truststore.password")));
+                    "truststore.password")),
+                key.startsWith(XPackSettings.GLOBAL_SSL_PREFIX) ? new Property[] { Property.Deprecated } : new Property[0]);
     public static final Setting<SecureString> TRUSTSTORE_PASSWORD_PROFILES = Setting.affixKeySetting("transport.profiles.",
             "xpack.security.ssl.truststore.secure_password", TRUSTSTORE_PASSWORD_TEMPLATE);
 
@@ -105,7 +107,7 @@ public class SSLConfigurationSettings {
 
     private static final Function<String, Setting<String>> TRUST_STORE_ALGORITHM_TEMPLATE  =  key ->
             new Setting<>(key, s -> TrustManagerFactory.getDefaultAlgorithm(),
-                    Function.identity(), Property.NodeScope, Property.Filtered);
+                    Function.identity(), propertiesFromKey(key));
     public static final Setting<String> TRUST_STORE_ALGORITHM_PROFILES = Setting.affixKeySetting("transport.profiles.",
             "xpack.security.ssl.truststore.algorithm", TRUST_STORE_ALGORITHM_TEMPLATE);
 
@@ -118,7 +120,7 @@ public class SSLConfigurationSettings {
             "xpack.security.ssl.truststore.type", TRUST_STORE_TYPE_TEMPLATE);
 
     private static final Function<String, Setting<Optional<String>>> TRUST_RESTRICTIONS_TEMPLATE = key -> new Setting<>(key, s -> null,
-            Optional::ofNullable, Property.NodeScope, Property.Filtered);
+            Optional::ofNullable, propertiesFromKey(key));
     public static final Setting<Optional<String>> TRUST_RESTRICTIONS_PROFILES = Setting.affixKeySetting("transport.profiles.",
             "xpack.security.ssl.trust_restrictions", TRUST_RESTRICTIONS_TEMPLATE);
 
@@ -132,19 +134,19 @@ public class SSLConfigurationSettings {
             "xpack.security.ssl.certificate", X509KeyPairSettings.CERT_TEMPLATE);
 
     private static final Function<String, Setting<List<String>>> CAPATH_SETTING_TEMPLATE = key -> Setting.listSetting(key, Collections
-            .emptyList(), Function.identity(), Property.NodeScope, Property.Filtered);
+            .emptyList(), Function.identity(), propertiesFromKey(key));
     public static final Setting<List<String>> CAPATH_SETTING_PROFILES =  Setting.affixKeySetting("transport.profiles.",
             "xpack.security.ssl.certificate_authorities", CAPATH_SETTING_TEMPLATE);
 
     private static final Function<String, Setting<Optional<SSLClientAuth>>> CLIENT_AUTH_SETTING_TEMPLATE =
             key -> new Setting<>(key, (String) null, s -> s == null ? Optional.empty() : Optional.of(SSLClientAuth.parse(s)),
-                    Property.NodeScope, Property.Filtered);
+                propertiesFromKey(key));
     public static final Setting<Optional<SSLClientAuth>> CLIENT_AUTH_SETTING_PROFILES =  Setting.affixKeySetting("transport.profiles.",
             "xpack.security.ssl.client_authentication", CLIENT_AUTH_SETTING_TEMPLATE);
 
     private static final Function<String, Setting<Optional<VerificationMode>>> VERIFICATION_MODE_SETTING_TEMPLATE =
             key -> new Setting<>(key, (String) null, s -> s == null ? Optional.empty() : Optional.of(VerificationMode.parse(s)),
-                    Property.NodeScope, Property.Filtered);
+                propertiesFromKey(key));
     public static final Setting<Optional<VerificationMode>> VERIFICATION_MODE_SETTING_PROFILES = Setting.affixKeySetting(
             "transport.profiles.", "xpack.security.ssl.verification_mode", VERIFICATION_MODE_SETTING_TEMPLATE);
 
@@ -190,6 +192,14 @@ private static String inferKeyStoreType(String path) {
         }
     }
 
+    static Property[] propertiesFromKey(String key) {
+        if (key.startsWith(XPackSettings.GLOBAL_SSL_PREFIX)) {
+            return new Property[] { Property.NodeScope, Property.Filtered, Property.Deprecated };
+        } else {
+            return new Property[] { Property.NodeScope, Property.Filtered };
+        }
+    }
+
     public List<Setting<?>> getAllSettings() {
         return allSettings;
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java

@@ -7,16 +7,21 @@
 
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.common.socket.SocketAccess;
 import org.elasticsearch.xpack.core.security.SecurityField;
+import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings;
 import org.elasticsearch.xpack.core.ssl.cert.CertificateInfo;
 
 import javax.net.ssl.HostnameVerifier;
@@ -61,7 +66,8 @@
  */
 public class SSLService extends AbstractComponent {
 
-    private final Settings settings;
+    private static final Logger logger = LogManager.getLogger(SSLService.class);
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
     /**
      * This is a mapping from "context name" (in general use, the name of a setting key)
@@ -83,6 +89,7 @@ public class SSLService extends AbstractComponent {
     private final SSLConfiguration globalSSLConfiguration;
     private final SetOnce<SSLConfiguration> transportSSLConfiguration = new SetOnce<>();
     private final Environment env;
+    private final Settings settings;
 
     /**
      * Create a new SSLService that parses the settings for the ssl contexts that need to be created, creates them, and then caches them
@@ -119,6 +126,13 @@ Map<SSLConfiguration, SSLContextHolder> loadSSLConfigurations() {
                 return Collections.emptyMap();
             }
 
+            @Override
+            SSLConfiguration sslConfiguration(Settings settings) {
+                SSLConfiguration sslConfiguration = super.sslConfiguration(settings);
+                SSLService.this.checkSSLConfigurationForFallback("monitoring.exporters", settings, sslConfiguration);
+                return sslConfiguration;
+            }
+
             /**
              * Returns the existing {@link SSLContextHolder} for the configuration
              * @throws IllegalArgumentException if not found
@@ -405,22 +419,35 @@ Map<SSLConfiguration, SSLContextHolder> loadSSLConfigurations() {
 
         sslSettingsMap.forEach((key, sslSettings) -> {
             if (sslSettings.isEmpty()) {
+                if (shouldCheckForFallbackDeprecation(key)) {
+                    checkSSLConfigurationForFallback(key, sslSettings, new SSLConfiguration(sslSettings, globalSSLConfiguration));
+                }
                 storeSslConfiguration(key, globalSSLConfiguration);
             } else {
                 final SSLConfiguration configuration = new SSLConfiguration(sslSettings, globalSSLConfiguration);
+                if (shouldCheckForFallbackDeprecation(key)) {
+                    checkSSLConfigurationForFallback(key, sslSettings, configuration);
+                }
                 storeSslConfiguration(key, configuration);
                 sslContextHolders.computeIfAbsent(configuration, this::createSslContext);
             }
         });
 
         final Settings transportSSLSettings = settings.getByPrefix(XPackSettings.TRANSPORT_SSL_PREFIX);
         final SSLConfiguration transportSSLConfiguration = new SSLConfiguration(transportSSLSettings, globalSSLConfiguration);
+        final boolean transportSSLEnabled = XPackSettings.TRANSPORT_SSL_ENABLED.get(settings);
+        if (transportSSLEnabled) {
+            checkSSLConfigurationForFallback(XPackSettings.TRANSPORT_SSL_PREFIX, transportSSLSettings, transportSSLConfiguration);
+        }
         this.transportSSLConfiguration.set(transportSSLConfiguration);
         storeSslConfiguration(XPackSettings.TRANSPORT_SSL_PREFIX, transportSSLConfiguration);
         Map<String, Settings> profileSettings = getTransportProfileSSLSettings(settings);
         sslContextHolders.computeIfAbsent(transportSSLConfiguration, this::createSslContext);
         profileSettings.forEach((key, profileSetting) -> {
             final SSLConfiguration configuration = new SSLConfiguration(profileSetting, transportSSLConfiguration);
+            if (transportSSLEnabled && key.equals("transport.profiles.default.xpack.security.ssl") == false) {
+                checkSSLConfigurationForFallback(key, profileSetting, configuration);
+            }
             storeSslConfiguration(key, configuration);
             sslContextHolders.computeIfAbsent(configuration, this::createSslContext);
         });
@@ -435,6 +462,57 @@ private void storeSslConfiguration(String key, SSLConfiguration configuration) {
         sslConfigurations.put(key, configuration);
     }
 
+    private boolean shouldCheckForFallbackDeprecation(String name) {
+        if (name.startsWith("xpack.security.authc.realms.")) {
+            // try to see if this is actually using TLS
+            Settings realm = settings.getByPrefix(name.substring(0, name.indexOf(".ssl")));
+            String type = realm.get("type");
+            // only check the types we know use ssl. custom realms may but we don't want to cause confusion
+            if (LdapRealmSettings.LDAP_TYPE.equals(type) || LdapRealmSettings.AD_TYPE.equals(type)) {
+                List<String> urls = realm.getAsList("url");
+                return urls.isEmpty() == false && urls.stream().anyMatch(s -> s.startsWith("ldaps://"));
+            } else if (SamlRealmSettings.TYPE.equals(type)) {
+                final String idpMetadataPath = SamlRealmSettings.IDP_METADATA_PATH.get(realm);
+                return Strings.hasText(idpMetadataPath) && idpMetadataPath.startsWith("https://");
+            }
+        } else if (name.startsWith("xpack.monitoring.exporters.")) {
+            Settings exporterSettings = settings.getByPrefix(name.substring(0, name.indexOf(".ssl")));
+            List<String> hosts = exporterSettings.getAsList("host");
+            return hosts.stream().anyMatch(s -> s.startsWith("https"));
+        } else if (name.equals(XPackSettings.HTTP_SSL_PREFIX) && XPackSettings.HTTP_SSL_ENABLED.get(settings)) {
+            return true;
+        } else if (name.equals("xpack.http.ssl") && XPackSettings.WATCHER_ENABLED.get(settings)) {
+            return true;
+        }
+        return false;
+    }
+
+    private void checkSSLConfigurationForFallback(String name, Settings settings, SSLConfiguration config) {
+        final SSLConfiguration noFallBackConfig = new SSLConfiguration(settings);
+        if (config.equals(noFallBackConfig) == false) {
+            List<String> fallbackReliers = new ArrayList<>();
+            if (config.keyConfig().equals(noFallBackConfig.keyConfig()) == false) {
+                fallbackReliers.add("key configuration");
+            }
+            if (config.trustConfig().equals(noFallBackConfig.trustConfig()) == false) {
+                fallbackReliers.add("trust configuration");
+            }
+            if (config.cipherSuites().equals(noFallBackConfig.cipherSuites()) == false) {
+                fallbackReliers.add("enabled cipher suites");
+            }
+            if (config.sslClientAuth() != noFallBackConfig.sslClientAuth()) {
+                fallbackReliers.add("client authentication");
+            }
+            if (config.supportedProtocols().equals(noFallBackConfig.supportedProtocols()) == false) {
+                fallbackReliers.add("supported protocols");
+            }
+            if (config.verificationMode() != noFallBackConfig.verificationMode()) {
+                fallbackReliers.add("certificate verification mode");
+            }
+            deprecationLogger.deprecated("SSL configuration [{}] relies upon fallback to another configuration for {}, which is " +
+                "deprecated.", name, fallbackReliers);
+        }
+    }
 
     /**
      * Returns information about each certificate that is referenced by any SSL configuration.