Secure password for monitoring HTTP exporter

docs/reference/settings/monitoring-settings.asciidoc

@@ -29,11 +29,11 @@ For more information, see <<monitor-elasticsearch-cluster>>.
 ==== General Monitoring Settings
 
 `xpack.monitoring.enabled`::
-Set to `true` (default) to enable {es} {monitoring} for {es} on the node. 
+Set to `true` (default) to enable {es} {monitoring} for {es} on the node.
 +
 --
-NOTE: To enable data collection, you must also set `xpack.monitoring.collection.enabled` 
-to `true`. Its default value is `false`. 
+NOTE: To enable data collection, you must also set `xpack.monitoring.collection.enabled`
+to `true`. Its default value is `false`.
 --
 
 [float]
@@ -51,7 +51,7 @@ this setting is `false` (default), {es} monitoring data is not collected and
 all monitoring data from other sources such as {kib}, Beats, and Logstash is
 ignored.
 
-`xpack.monitoring.collection.interval` (<<cluster-update-settings,Dynamic>>):: 
+`xpack.monitoring.collection.interval` (<<cluster-update-settings,Dynamic>>)::
 
 Setting to `-1` to disable data collection is no longer supported beginning with
 7.0.0. deprecated[6.3.0, Use `xpack.monitoring.collection.enabled` set to `false` instead.]
@@ -200,9 +200,14 @@ xpack.monitoring.exporters:
 
 The username is required if a `auth.password` is supplied.
 
+`secure_auth.password` ({ref}/secure-settings.html[Secure], {ref}/secure-settings.html#reloadable-secure-settings[reloadable])::
+
+The password for the `auth.username`. May not be specified with `auth.password`.
+
 `auth.password`::
 
-The password for the `auth.username`.
+The password for the `auth.username`. May not be specified with `secure_auth.password`. This setting is deprecated[7.6.0, Use
+`secure_auth.password` instead.].
 
 `connection.timeout`::
 

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.script.ScriptService;
@@ -68,7 +69,7 @@
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.settings.Setting.boolSetting;
 
-public class Monitoring extends Plugin implements ActionPlugin {
+public class Monitoring extends Plugin implements ActionPlugin, ReloadablePlugin {
 
     /**
      * The ability to automatically cleanup ".watcher_history*" indices while also cleaning up Monitoring indices.
@@ -80,6 +81,8 @@ public class Monitoring extends Plugin implements ActionPlugin {
     protected final Settings settings;
     private final boolean enabled;
 
+    private Exporters exporters;
+
     public Monitoring(Settings settings) {
         this.settings = settings;
         this.enabled = XPackSettings.MONITORING_ENABLED.get(settings);
@@ -110,8 +113,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         Map<String, Exporter.Factory> exporterFactories = new HashMap<>();
         exporterFactories.put(HttpExporter.TYPE, config -> new HttpExporter(config, dynamicSSLService, threadPool.getThreadContext()));
         exporterFactories.put(LocalExporter.TYPE, config -> new LocalExporter(config, client, cleanerService));
-        final Exporters exporters = new Exporters(settings, exporterFactories, clusterService, getLicenseState(),
-            threadPool.getThreadContext());
+        exporters = new Exporters(settings, exporterFactories, clusterService, getLicenseState(), threadPool.getThreadContext());
 
         Set<Collector> collectors = new HashSet<>();
         collectors.add(new IndexStatsCollector(clusterService, getLicenseState(), client));
@@ -178,4 +180,13 @@ public List<String> getSettingsFilter() {
         final String exportersKey = "xpack.monitoring.exporters.";
         return List.of(exportersKey + "*.auth.*", exportersKey + "*.ssl.*");
     }
+
+    @Override
+    public void reload(Settings settings) throws Exception {
+        final List<String> changedExporters = HttpExporter.loadSettings(settings);
+        for (String changedExporter : changedExporters) {
+            final Settings settingsForChangedExporter = settings.filter(x -> x.startsWith("xpack.monitoring.exporters." + changedExporter));
+            exporters.setExportersSetting(settingsForChangedExporter);
+        }
+    }
 }

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringService.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.Collector;
 import org.elasticsearch.xpack.monitoring.exporter.Exporters;
+import org.elasticsearch.xpack.monitoring.exporter.http.HttpExporter;
 
 import java.io.Closeable;
 import java.util.ArrayList;
@@ -104,6 +105,8 @@ public class MonitoringService extends AbstractLifecycleComponent {
             .addSettingsUpdateConsumer(ELASTICSEARCH_COLLECTION_ENABLED, this::setElasticsearchCollectionEnabled);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(ENABLED, this::setMonitoringActive);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(INTERVAL, this::setInterval);
+
+        HttpExporter.loadSettings(settings);
     }
 
     void setElasticsearchCollectionEnabled(final boolean enabled) {

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java

@@ -35,6 +35,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 
@@ -58,15 +59,17 @@ public Exporters(Settings settings, Map<String, Exporter.Factory> factories,
         this.clusterService = Objects.requireNonNull(clusterService);
         this.licenseState = Objects.requireNonNull(licenseState);
 
-        clusterService.getClusterSettings().addSettingsUpdateConsumer(this::setExportersSetting, getSettings());
+        final List<Setting.AffixSetting<?>> dynamicSettings =
+            getSettings().stream().filter(Setting::isDynamic).collect(Collectors.toList());
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(this::setExportersSetting, dynamicSettings);
         HttpExporter.registerSettingValidators(clusterService);
-        // this ensures, that logging is happening by adding an empty consumer per affix setting
-        for (Setting.AffixSetting<?> affixSetting : getSettings()) {
+        // this ensures that logging is happening by adding an empty consumer per affix setting
+        for (Setting.AffixSetting<?> affixSetting : dynamicSettings) {
             clusterService.getClusterSettings().addAffixUpdateConsumer(affixSetting, (s, o) -> {}, (s, o) -> {});
         }
     }
 
-    private void setExportersSetting(Settings exportersSetting) {
+    public void setExportersSetting(Settings exportersSetting) {
         if (this.lifecycle.started()) {
             Map<String, Exporter> updated = initExporters(exportersSetting);
             closeExporters(logger, this.exporters.getAndSet(updated));

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java

@@ -25,6 +25,8 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -47,6 +49,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -206,17 +209,9 @@ public void validate(final String username, final Map<Setting<?>, Object> settin
                                 final String namespace =
                                     HttpExporter.AUTH_USERNAME_SETTING.getNamespace(
                                         HttpExporter.AUTH_USERNAME_SETTING.getConcreteSetting(key));
-                                final String password =
-                                    (String) settings.get(AUTH_PASSWORD_SETTING.getConcreteSettingForNamespace(namespace));
 
                                 // password must be specified along with username for any auth
                                 if (Strings.isNullOrEmpty(username) == false) {
-                                    if (Strings.isNullOrEmpty(password)) {
-                                        throw new SettingsException(
-                                            "[" + AUTH_USERNAME_SETTING.getConcreteSettingForNamespace(namespace).getKey() + "] is set " +
-                                            "but [" + AUTH_PASSWORD_SETTING.getConcreteSettingForNamespace(namespace).getKey() + "] is " +
-                                            "missing");
-                                    }
                                     final String type =
                                         (String) settings.get(Exporter.TYPE_SETTING.getConcreteSettingForNamespace(namespace));
                                     if ("http".equals(type) == false) {
@@ -232,8 +227,7 @@ public Iterator<Setting<?>> settings() {
                                         HttpExporter.AUTH_USERNAME_SETTING.getConcreteSetting(key));
 
                                 final List<Setting<?>> settings = List.of(
-                                    Exporter.TYPE_SETTING.getConcreteSettingForNamespace(namespace),
-                                    HttpExporter.AUTH_PASSWORD_SETTING.getConcreteSettingForNamespace(namespace));
+                                    Exporter.TYPE_SETTING.getConcreteSettingForNamespace(namespace));
                                 return settings.iterator();
                             }
 
@@ -285,8 +279,18 @@ public Iterator<Setting<?>> settings() {
                         },
                         Property.Dynamic,
                         Property.NodeScope,
-                        Property.Filtered),
+                        Property.Filtered,
+                        Property.Deprecated),
                     TYPE_DEPENDENCY);
+    /**
+     * Secure password for basic auth.
+     */
+    public static final Setting.AffixSetting<SecureString> SECURE_AUTH_PASSWORD_SETTING =
+        Setting.affixKeySetting(
+            "xpack.monitoring.exporters.",
+            "secure_auth_password",
+            key -> SecureSetting.secureString(key, null),
+            TYPE_DEPENDENCY);
     /**
      * The SSL settings.
      *
@@ -401,6 +405,7 @@ public Iterator<Setting<?>> settings() {
      */
     private final AtomicBoolean clusterAlertsAllowed = new AtomicBoolean(false);
 
+    private static final Map<String, String> SECURE_AUTH_PASSWORDS = new HashMap<>();
     private final ThreadContext threadContext;
     private final DateFormatter dateTimeFormatter;
 
@@ -689,6 +694,29 @@ private static void configureTimeouts(final RestClientBuilder builder, final Con
         builder.setRequestConfigCallback(new TimeoutRequestConfigCallback(connectTimeout, socketTimeout));
     }
 
+
+    /**
+     * Caches secure settings for use when dynamically configuring HTTP exporters
+     * @param settings settings used for configuring HTTP exporter
+     * @return names of HTTP exporters whose secure settings changed, if any
+     */
+    public static List<String> loadSettings(Settings settings) {
+        final List<String> changedExporters = new ArrayList<>();
+        for (final String namespace : SECURE_AUTH_PASSWORD_SETTING.getNamespaces(settings)) {
+            final Setting<?> s = SECURE_AUTH_PASSWORD_SETTING.getConcreteSettingForNamespace(namespace);
+            final String password = s.get(settings).toString();
+            String existingPassword;
+            synchronized (SECURE_AUTH_PASSWORDS) {
+                existingPassword = SECURE_AUTH_PASSWORDS.get(namespace);
+                SECURE_AUTH_PASSWORDS.put(namespace, password);
+            }
+            if (password.equals(existingPassword) == false) {
+                changedExporters.add(namespace);
+            }
+        }
+        return changedExporters;
+    }
+
     /**
      * Creates the optional {@link CredentialsProvider} with the username/password to use with <em>all</em> requests for user
      * authentication.
@@ -700,7 +728,13 @@ private static void configureTimeouts(final RestClientBuilder builder, final Con
     @Nullable
     private static CredentialsProvider createCredentialsProvider(final Config config) {
         final String username = AUTH_USERNAME_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings());
-        final String password = AUTH_PASSWORD_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings());
+
+        String password;
+        synchronized (SECURE_AUTH_PASSWORDS) {
+            password = SECURE_AUTH_PASSWORDS.containsKey(config.name())
+                ? SECURE_AUTH_PASSWORDS.get(config.name())
+                : AUTH_PASSWORD_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings());
+        }
 
         final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
         credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
@@ -865,9 +899,19 @@ public void doClose() {
         }
     }
 
-    public static List<Setting.AffixSetting<?>> getSettings() {
+    public static List<Setting.AffixSetting<?>> getDynamicSettings() {
         return Arrays.asList(HOST_SETTING, TEMPLATE_CREATE_LEGACY_VERSIONS_SETTING, AUTH_PASSWORD_SETTING, AUTH_USERNAME_SETTING,
                 BULK_TIMEOUT_SETTING, CONNECTION_READ_TIMEOUT_SETTING, CONNECTION_TIMEOUT_SETTING, PIPELINE_CHECK_TIMEOUT_SETTING,
                 PROXY_BASE_PATH_SETTING, SNIFF_ENABLED_SETTING, TEMPLATE_CHECK_TIMEOUT_SETTING, SSL_SETTING, HEADERS_SETTING);
     }
+
+    public static List<Setting.AffixSetting<?>> getSecureSettings() {
+        return List.of(SECURE_AUTH_PASSWORD_SETTING);
+    }
+
+    public static List<Setting.AffixSetting<?>> getSettings() {
+        List<Setting.AffixSetting<?>> allSettings = new ArrayList<>(getDynamicSettings());
+        allSettings.addAll(getSecureSettings());
+        return allSettings;
+    }
 }

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterTests.java

@@ -233,6 +233,8 @@ public void testExporterWithPasswordButNoUsername() {
             IllegalArgumentException.class,
             () -> HttpExporter.AUTH_PASSWORD_SETTING.getConcreteSetting(prefix + ".auth.password").get(settings));
         assertThat(e, hasToString(containsString(expected)));
+        assertWarnings("[xpack.monitoring.exporters._http.auth.password] setting was deprecated in Elasticsearch and will be removed " +
+            "in a future release! See the breaking changes documentation for the next major version.");
     }
 
     public void testExporterWithUsernameButNoPassword() {
@@ -331,7 +333,8 @@ public void testCreateRestClient() throws IOException {
                 .put("xpack.monitoring.exporters._http.host", "http://localhost:9200");
 
         // use basic auth
-        if (randomBoolean()) {
+        final boolean useBasicAuth = randomBoolean();
+        if (useBasicAuth) {
             builder.put("xpack.monitoring.exporters._http.auth.username", "_user")
                    .put("xpack.monitoring.exporters._http.auth.password", "_pass");
         }
@@ -346,6 +349,10 @@ public void testCreateRestClient() throws IOException {
 
         // doesn't explode
         HttpExporter.createRestClient(config, sslService, listener).close();
+        if (useBasicAuth) {
+            assertWarnings("[xpack.monitoring.exporters._http.auth.password] setting was deprecated in Elasticsearch and will be " +
+                "removed in a future release! See the breaking changes documentation for the next major version.");
+        }
     }
 
     public void testCreateSnifferDisabledByDefault() {

x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java

@@ -160,7 +160,7 @@ public void enableExporter() throws Exception {
             .put("xpack.monitoring.exporters._http.ssl.certificate_authorities", "testnode.crt")
             .build();
         ClusterUpdateSettingsResponse response = newHighLevelClient().cluster().putSettings(
-            new ClusterUpdateSettingsRequest().transientSettings(exporterSettings), RequestOptions.DEFAULT);
+            new ClusterUpdateSettingsRequest().transientSettings(exporterSettings), getRequestOptions());
         assertTrue(response.isAcknowledged());
     }
 
@@ -177,10 +177,17 @@ public void disableExporter() throws IOException {
             .putNull("xpack.monitoring.exporters._http.ssl.certificate_authorities")
             .build();
         ClusterUpdateSettingsResponse response = newHighLevelClient().cluster().putSettings(
-            new ClusterUpdateSettingsRequest().transientSettings(exporterSettings), RequestOptions.DEFAULT);
+            new ClusterUpdateSettingsRequest().transientSettings(exporterSettings), getRequestOptions());
         assertTrue(response.isAcknowledged());
     }
 
+    private RequestOptions getRequestOptions() {
+        String deprecationWarning = "[xpack.monitoring.exporters._http.auth.password] setting was deprecated in Elasticsearch and will " +
+            "be removed in a future release! See the breaking changes documentation for the next major version.";
+        return RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> warnings.size() != 1 ||
+            warnings.get(0).equals(deprecationWarning) == false).build();
+    }
+
     private boolean getMonitoringUsageExportersDefined() throws Exception {
         RestHighLevelClient client = newHighLevelClient();
         final XPackUsageResponse usageResponse = client.xpack().usage(new XPackUsageRequest(), RequestOptions.DEFAULT);