From 8bf5a963c090fdca4ea6bfe755a335d321f02ac7 Mon Sep 17 00:00:00 2001 From: felicity3786 Date: Thu, 11 Dec 2025 10:45:16 -0800 Subject: [PATCH 1/3] Allow health check monitor to use mtls on backend connection --- .../ClusterStatsJmxMonitor.java | 10 ++++--- .../ClusterStatsMetricsMonitor.java | 19 ++++++++++---- .../ha/config/BackendStateConfiguration.java | 11 ++++++++ .../ha/module/HaGatewayProviderModule.java | 26 +++++++++++++++++++ 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/clustermonitor/ClusterStatsJmxMonitor.java b/gateway-ha/src/main/java/io/trino/gateway/ha/clustermonitor/ClusterStatsJmxMonitor.java index 35443ea29..59ca1d8e1 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/clustermonitor/ClusterStatsJmxMonitor.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/clustermonitor/ClusterStatsJmxMonitor.java @@ -47,6 +47,7 @@ public class ClusterStatsJmxMonitor private final String username; private final String password; private final boolean xForwardedProtoHeader; + private final boolean monitorMtlsEnabled; public ClusterStatsJmxMonitor(HttpClient client, BackendStateConfiguration backendStateConfiguration) { @@ -54,6 +55,7 @@ public ClusterStatsJmxMonitor(HttpClient client, BackendStateConfiguration backe this.username = backendStateConfiguration.getUsername(); this.password = backendStateConfiguration.getPassword(); this.xForwardedProtoHeader = backendStateConfiguration.getXForwardedProtoHeader(); + this.monitorMtlsEnabled = backendStateConfiguration.isMonitorMtlsEnabled(); } private static void updateClusterStatsFromDiscoveryNodeManagerResponse(JmxResponse response, ClusterStats.Builder clusterStats) @@ -132,8 +134,10 @@ private Optional queryJmx(ProxyBackendConfiguration backend, String .setUri(uriBuilderFrom(URI.create(jmxUrl)) .appendPath(JMX_PATH) .appendPath(mbeanName) - .build()) - .addHeader("X-Trino-User", username); + .build()); + if (!monitorMtlsEnabled) { + requestBuilder.addHeader("X-Trino-User", username); + } if (xForwardedProtoHeader) { requestBuilder.addHeader(X_FORWARDED_PROTO, "https"); } @@ -141,7 +145,7 @@ private Optional queryJmx(ProxyBackendConfiguration backend, String boolean isHttps = preparedRequest.getUri().getScheme().equalsIgnoreCase("https"); - if (isHttps) { + if (isHttps && !monitorMtlsEnabled) { HttpRequestFilter filter = new BasicAuthRequestFilter(username, password); preparedRequest = filter.filterRequest(preparedRequest); } diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/clustermonitor/ClusterStatsMetricsMonitor.java b/gateway-ha/src/main/java/io/trino/gateway/ha/clustermonitor/ClusterStatsMetricsMonitor.java index 3064b78b0..ce916a09b 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/clustermonitor/ClusterStatsMetricsMonitor.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/clustermonitor/ClusterStatsMetricsMonitor.java @@ -53,6 +53,7 @@ public class ClusterStatsMetricsMonitor private final int retries; private final MetricsResponseHandler metricsResponseHandler; private final Header identityHeader; + private final boolean monitorMtlsEnabled; private final String metricsEndpoint; private final String runningQueriesMetricName; private final String queuedQueriesMetricName; @@ -65,12 +66,18 @@ public ClusterStatsMetricsMonitor(HttpClient httpClient, BackendStateConfigurati { this.httpClient = requireNonNull(httpClient, "httpClient is null"); retries = monitorConfiguration.getRetries(); - if (!isNullOrEmpty(backendStateConfiguration.getPassword())) { - identityHeader = new Header("Authorization", - new BasicCredentials(backendStateConfiguration.getUsername(), backendStateConfiguration.getPassword()).getBasicAuthHeader()); + monitorMtlsEnabled = backendStateConfiguration.isMonitorMtlsEnabled(); + if (!monitorMtlsEnabled) { + if (!isNullOrEmpty(backendStateConfiguration.getPassword())) { + identityHeader = new Header("Authorization", + new BasicCredentials(backendStateConfiguration.getUsername(), backendStateConfiguration.getPassword()).getBasicAuthHeader()); + } + else { + identityHeader = new Header("X-Trino-User", backendStateConfiguration.getUsername()); + } } else { - identityHeader = new Header("X-Trino-User", backendStateConfiguration.getUsername()); + identityHeader = null; } metricsEndpoint = monitorConfiguration.getMetricsEndpoint(); runningQueriesMetricName = monitorConfiguration.getRunningQueriesMetricName(); @@ -139,8 +146,10 @@ private Map getMetrics(String baseUrl, int retriesRemaining) Request.Builder requestBuilder = prepareGet() .setUri(uri.build()) - .addHeader(identityHeader.name, identityHeader.value) .addHeader("Content-Type", "application/openmetrics-text; version=1.0.0; charset=utf-8"); + if (identityHeader != null) { + requestBuilder.addHeader(identityHeader.name, identityHeader.value); + } if (xForwardedProtoHeader) { requestBuilder.addHeader(X_FORWARDED_PROTO, "https"); } diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/config/BackendStateConfiguration.java b/gateway-ha/src/main/java/io/trino/gateway/ha/config/BackendStateConfiguration.java index 77436950e..13ac5cad6 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/config/BackendStateConfiguration.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/config/BackendStateConfiguration.java @@ -19,6 +19,7 @@ public class BackendStateConfiguration private String password = ""; private Boolean ssl = false; private boolean xForwardedProtoHeader; + private boolean monitorMtlsEnabled; public BackendStateConfiguration() {} @@ -61,4 +62,14 @@ public void setXForwardedProtoHeader(boolean xForwardedProtoHeader) { this.xForwardedProtoHeader = xForwardedProtoHeader; } + + public boolean isMonitorMtlsEnabled() + { + return monitorMtlsEnabled; + } + + public void setMonitorMtlsEnabled(boolean monitorMtlsEnabled) + { + this.monitorMtlsEnabled = monitorMtlsEnabled; + } } diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/module/HaGatewayProviderModule.java b/gateway-ha/src/main/java/io/trino/gateway/ha/module/HaGatewayProviderModule.java index 2973655b8..3f8a49722 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/module/HaGatewayProviderModule.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/module/HaGatewayProviderModule.java @@ -106,6 +106,10 @@ public HaGatewayProviderModule(HaGatewayConfiguration configuration) { this.configuration = requireNonNull(configuration, "configuration is null"); pathFilter = new PathFilter(configuration.getStatementPaths(), configuration.getExtraWhitelistPaths()); + // Enforce required TLS properties for the named HttpClient "monitor" when mTLS is enabled for monitors + if (configuration.getBackendState() != null && configuration.getBackendState().isMonitorMtlsEnabled()) { + validateMonitorMtlsConfig(configuration.getServerConfig()); + } Map presetUsers = configuration.getPresetUsers(); oauthManager = getOAuthManager(configuration); @@ -127,6 +131,28 @@ public HaGatewayProviderModule(HaGatewayConfiguration configuration) queryHistoryManager = new HaQueryHistoryManager(jdbi, configuration.getDataStore().getJdbcUrl().startsWith("jdbc:oracle")); } + private static void validateMonitorMtlsConfig(Map serverConfig) + { + String[] requiredKeys = new String[] { + "monitor.http-client.key-store-path", + "monitor.http-client.key-store-password", + "monitor.http-client.trust-store-path", + "monitor.http-client.trust-store-password" + }; + java.util.List missing = new java.util.ArrayList<>(); + for (String key : requiredKeys) { + String value = serverConfig.get(key); + if (value == null || value.isBlank()) { + missing.add(key); + } + } + if (!missing.isEmpty()) { + throw new IllegalArgumentException( + "backendState.monitorMtlsEnabled=true requires monitor HttpClient TLS configuration. Missing: " + + String.join(", ", missing)); + } + } + private LbOAuthManager getOAuthManager(HaGatewayConfiguration configuration) { AuthenticationConfiguration authenticationConfiguration = configuration.getAuthentication(); From 5d266edeb02e00d3d2755835fc2d9432b794fac9 Mon Sep 17 00:00:00 2001 From: felicity3786 Date: Thu, 11 Dec 2025 16:15:03 -0800 Subject: [PATCH 2/3] small fix --- .../ha/module/HaGatewayProviderModule.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/module/HaGatewayProviderModule.java b/gateway-ha/src/main/java/io/trino/gateway/ha/module/HaGatewayProviderModule.java index 3f8a49722..6e8131d9b 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/module/HaGatewayProviderModule.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/module/HaGatewayProviderModule.java @@ -70,6 +70,7 @@ import jakarta.ws.rs.container.ContainerRequestFilter; import org.jdbi.v3.core.Jdbi; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -81,6 +82,11 @@ public class HaGatewayProviderModule extends AbstractModule { + private static final String MONITOR_HTTP_CLIENT_KEY_STORE_PATH = "monitor.http-client.key-store-path"; + private static final String MONITOR_HTTP_CLIENT_KEY_STORE_PASSWORD = "monitor.http-client.key-store-password"; + private static final String MONITOR_HTTP_CLIENT_TRUST_STORE_PATH = "monitor.http-client.trust-store-path"; + private static final String MONITOR_HTTP_CLIENT_TRUST_STORE_PASSWORD = "monitor.http-client.trust-store-password"; + private final LbOAuthManager oauthManager; private final LbFormAuthManager formAuthManager; private final AuthorizationManager authorizationManager; @@ -134,12 +140,12 @@ public HaGatewayProviderModule(HaGatewayConfiguration configuration) private static void validateMonitorMtlsConfig(Map serverConfig) { String[] requiredKeys = new String[] { - "monitor.http-client.key-store-path", - "monitor.http-client.key-store-password", - "monitor.http-client.trust-store-path", - "monitor.http-client.trust-store-password" + MONITOR_HTTP_CLIENT_KEY_STORE_PATH, + MONITOR_HTTP_CLIENT_KEY_STORE_PASSWORD, + MONITOR_HTTP_CLIENT_TRUST_STORE_PATH, + MONITOR_HTTP_CLIENT_TRUST_STORE_PASSWORD }; - java.util.List missing = new java.util.ArrayList<>(); + List missing = new ArrayList<>(); for (String key : requiredKeys) { String value = serverConfig.get(key); if (value == null || value.isBlank()) { From 00524ae93a2a8fa232f2ea74295ff2368267d872 Mon Sep 17 00:00:00 2001 From: felicity3786 Date: Thu, 11 Dec 2025 16:24:59 -0800 Subject: [PATCH 3/3] add docs --- docs/installation.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/installation.md b/docs/installation.md index b8995a159..37458e366 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -558,3 +558,32 @@ a username and password using `backendState` as with the `JDBC` option. #### NOOP This option disables health checks. + +### Enable mTLS for monitor HttpClient (JMX and METRICS) + +For JMX and Metrics monitors, you can enable mutual TLS (mTLS) for the monitor HttpClient. +When enabled, the monitors authenticate to backends with a client certificate and ignore any +configured `backendState.username` and `backendState.password`. + +Enable it with a toggle in `backendState` and provide TLS material for the named `monitor` HttpClient +under `serverConfig`: + +```yaml +backendState: + monitorMtlsEnabled: true + +serverConfig: + monitor.http-client.key-store-path: /path/to/keystore + monitor.http-client.key-store-password: keystore_password + monitor.http-client.trust-store-path: /path/to/truststore + monitor.http-client.trust-store-password: changeit +``` + +Notes: + +- The `monitor` client is bound via Airlift's `httpClientBinder`, so properties must use the + `monitor.http-client.*` prefix. +- With `monitorMtlsEnabled: true`: + - JMX and Metrics monitors do not send `Authorization` or `X-Trino-User` headers and do not apply Basic Auth. + - Authentication relies solely on the configured client certificate. +- The gateway validates these TLS properties on startup when mTLS is enabled and fails fast if any are missing.