Skip to content

HADOOP-16524. Reloading SSL keystore for both DataNode and NameNode #2470

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
saintstack merged 3 commits into from
Jan 8, 2021
Merged

HADOOP-16524. Reloading SSL keystore for both DataNode and NameNode #2470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d36f054
HADOOP-16524. Reloading SSL keystore for both DataNode and NameNode
Nov 18, 2020
be6b9cc
addressing PR comments by saintstack
Dec 9, 2020
142740f
few more cosmetic fixes
Dec 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 57 additions & 5 deletions hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,17 @@
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.nio.file.Paths;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.HashMap;
import java.util.Collections;
import java.util.Optional;
import java.util.Properties;
import java.util.Enumeration;
import java.util.Arrays;
import java.util.Timer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

Expand Down Expand Up @@ -74,6 +77,8 @@
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory;
import org.apache.hadoop.security.ssl.FileMonitoringTimerTask;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Shell;
Expand Down Expand Up @@ -184,6 +189,7 @@ public final class HttpServer2 implements FilterContainer {
static final String STATE_DESCRIPTION_ALIVE = " - alive";
static final String STATE_DESCRIPTION_NOT_LIVE = " - not live";
private final SignerSecretProvider secretProvider;
private final Optional<java.util.Timer> configurationChangeMonitor;
private XFrameOption xFrameOption;
private boolean xFrameOptionIsEnabled;
public static final String HTTP_HEADER_PREFIX = "hadoop.http.header.";
Expand Down Expand Up @@ -239,6 +245,8 @@ public static class Builder {

private boolean sniHostCheckEnabled;

private Optional<Timer> configurationChangeMonitor = Optional.empty();

public Builder setName(String name){
this.name = name;
return this;
Expand Down Expand Up @@ -569,12 +577,45 @@ private ServerConnector createHttpsChannelConnector(
}

setEnabledProtocols(sslContextFactory);

long storesReloadInterval =
conf.getLong(FileBasedKeyStoresFactory.SSL_STORES_RELOAD_INTERVAL_TPL_KEY,
FileBasedKeyStoresFactory.DEFAULT_SSL_STORES_RELOAD_INTERVAL);

if (storesReloadInterval > 0) {
this.configurationChangeMonitor = Optional.of(
this.makeConfigurationChangeMonitor(storesReloadInterval, sslContextFactory));
}

conn.addFirstConnectionFactory(new SslConnectionFactory(sslContextFactory,
HttpVersion.HTTP_1_1.asString()));

return conn;
}

private Timer makeConfigurationChangeMonitor(long reloadInterval,
SslContextFactory.Server sslContextFactory) {
Timer timer = new Timer("SSL Certificates Store Monitor", true);
//
// The Jetty SSLContextFactory provides a 'reload' method which will reload both
// truststore and keystore certificates.
//
timer.schedule(new FileMonitoringTimerTask(
Paths.get(keyStore),
path -> {
LOG.info("Reloading certificates from store keystore " + keyStore);
try {
sslContextFactory.reload(factory -> { });
} catch (Exception ex) {
LOG.error("Failed to reload SSL keystore certificates", ex);
}
},null),
reloadInterval,
reloadInterval
);
return timer;
}

private void setEnabledProtocols(SslContextFactory sslContextFactory) {
String enabledProtocols = conf.get(SSLFactory.SSL_ENABLED_PROTOCOLS_KEY,
SSLFactory.SSL_ENABLED_PROTOCOLS_DEFAULT);
Expand Down Expand Up @@ -617,6 +658,7 @@ private HttpServer2(final Builder b) throws IOException {
this.webAppContext = createWebAppContext(b, adminsAcl, appDir);
this.xFrameOptionIsEnabled = b.xFrameEnabled;
this.xFrameOption = b.xFrameOption;
this.configurationChangeMonitor = b.configurationChangeMonitor;

try {
this.secretProvider =
Expand Down Expand Up @@ -1384,6 +1426,16 @@ void openListeners() throws Exception {
*/
public void stop() throws Exception {
MultiException exception = null;
if (this.configurationChangeMonitor.isPresent()) {
try {
this.configurationChangeMonitor.get().cancel();
} catch (Exception e) {
LOG.error(
"Error while canceling configuration monitoring timer for webapp"
+ webAppContext.getDisplayName(), e);
exception = addMultiException(exception, e);
}
}
for (ServerConnector c : listeners) {
try {
c.close();
Expand Down
216 changes: 137 additions & 79 deletions ...hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,20 @@
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.TrustManager;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.text.MessageFormat;
import java.util.Timer;

/**
* {@link KeyStoresFactory} implementation that reads the certificates from
* keystore files.
* <p>
* if the trust certificates keystore file changes, the {@link TrustManager}
* is refreshed with the new trust certificate entries (using a
* {@link ReloadingX509TrustManager} trustmanager).
* If either the truststore or the keystore certificates file changes, it
* would be refreshed under the corresponding wrapper implementation -
* {@link ReloadingX509KeystoreManager} or {@link ReloadingX509TrustManager}.
* </p>
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
Expand All @@ -51,6 +51,13 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory {
private static final Logger LOG =
LoggerFactory.getLogger(FileBasedKeyStoresFactory.class);

/**
* The refresh interval used to check if either of the truststore or keystore
* certificate file has changed.
*/
public static final String SSL_STORES_RELOAD_INTERVAL_TPL_KEY =
"ssl.{0}.stores.reload.interval";

public static final String SSL_KEYSTORE_LOCATION_TPL_KEY =
"ssl.{0}.keystore.location";
public static final String SSL_KEYSTORE_PASSWORD_TPL_KEY =
Expand All @@ -77,14 +84,119 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory {
public static final String DEFAULT_KEYSTORE_TYPE = "jks";

/**
* Reload interval in milliseconds.
* The default time interval in milliseconds used to check if either
* of the truststore or keystore certificates file has changed and needs reloading.
*/
public static final int DEFAULT_SSL_TRUSTSTORE_RELOAD_INTERVAL = 10000;
public static final int DEFAULT_SSL_STORES_RELOAD_INTERVAL = 10000;

private Configuration conf;
private KeyManager[] keyManagers;
private TrustManager[] trustManagers;
private ReloadingX509TrustManager trustManager;
private Timer fileMonitoringTimer;


private void createTrustManagersFromConfiguration(SSLFactory.Mode mode,
String truststoreType,
String truststoreLocation,
long storesReloadInterval)
Copy link
Contributor

@saintstack saintstack Dec 4, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does the rest of this class do this extreme rightward shifting of parameters?

Copy link
Contributor

@saintstack saintstack Dec 4, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm... maybe it does. Ignore the above comment then.

Copy link
Contributor Author

@bolerio bolerio Dec 8, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, but it's true the most of the code base doesn't have it. Should I change? I don't mind...just inadvertently sneaking in my old habits I suppose....

Copy link
Contributor

@saintstack saintstack Dec 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems like your code habits align w/ the backing file here. I'd leave it as is.

throws IOException, GeneralSecurityException {
String passwordProperty = resolvePropertyName(mode,
SSL_TRUSTSTORE_PASSWORD_TPL_KEY);
String truststorePassword = getPassword(conf, passwordProperty, "");
if (truststorePassword.isEmpty()) {
// An empty trust store password is legal; the trust store password
// is only required when writing to a trust store. Otherwise it's
// an optional integrity check.
truststorePassword = null;
}

// Check if obsolete truststore specific reload interval is present for backward compatible
long truststoreReloadInterval =
conf.getLong(
resolvePropertyName(mode, SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY),
storesReloadInterval);

if (LOG.isDebugEnabled()) {
LOG.debug(mode.toString() + " TrustStore: " + truststoreLocation +
", reloading at " + truststoreReloadInterval + " millis.");
}

trustManager = new ReloadingX509TrustManager(
truststoreType,
truststoreLocation,
truststorePassword);

if (truststoreReloadInterval > 0) {
fileMonitoringTimer.schedule(
new FileMonitoringTimerTask(
Paths.get(truststoreLocation),
path -> trustManager.loadFrom(path),
exception -> LOG.error(ReloadingX509TrustManager.RELOAD_ERROR_MESSAGE, exception)),
truststoreReloadInterval,
truststoreReloadInterval);
}

if (LOG.isDebugEnabled()) {
LOG.debug(mode.toString() + " Loaded TrustStore: " + truststoreLocation);
}
trustManagers = new TrustManager[]{trustManager};
}

/**
* Implements logic of initializing the KeyManagers with the options
* to reload keystores.
* @param mode client or server
* @param keystoreType The keystore type.
* @param storesReloadInterval The interval to check if the keystore certificates
* file has changed.
*/
private void createKeyManagersFromConfiguration(SSLFactory.Mode mode,
String keystoreType, long storesReloadInterval)
throws GeneralSecurityException, IOException {
String locationProperty =
resolvePropertyName(mode, SSL_KEYSTORE_LOCATION_TPL_KEY);
String keystoreLocation = conf.get(locationProperty, "");
if (keystoreLocation.isEmpty()) {
throw new GeneralSecurityException("The property '" + locationProperty +
"' has not been set in the ssl configuration file.");
}
String passwordProperty =
resolvePropertyName(mode, SSL_KEYSTORE_PASSWORD_TPL_KEY);
String keystorePassword = getPassword(conf, passwordProperty, "");
if (keystorePassword.isEmpty()) {
throw new GeneralSecurityException("The property '" + passwordProperty +
"' has not been set in the ssl configuration file.");
}
String keyPasswordProperty =
resolvePropertyName(mode, SSL_KEYSTORE_KEYPASSWORD_TPL_KEY);
// Key password defaults to the same value as store password for
// compatibility with legacy configurations that did not use a separate
// configuration property for key password.
String keystoreKeyPassword = getPassword(
conf, keyPasswordProperty, keystorePassword);
if (LOG.isDebugEnabled()) {
LOG.debug(mode.toString() + " KeyStore: " + keystoreLocation);
}

ReloadingX509KeystoreManager keystoreManager = new ReloadingX509KeystoreManager(
keystoreType,
keystoreLocation,
keystorePassword,
keystoreKeyPassword);

if (storesReloadInterval > 0) {
fileMonitoringTimer.schedule(
new FileMonitoringTimerTask(
Paths.get(keystoreLocation),
path -> keystoreManager.loadFrom(path),
exception -> LOG.error(ReloadingX509KeystoreManager.RELOAD_ERROR_MESSAGE, exception)),
storesReloadInterval,
storesReloadInterval);
}

keyManagers = new KeyManager[] { keystoreManager };
}

/**
* Resolves a property name to its client/server version if applicable.
Expand Down Expand Up @@ -139,56 +251,28 @@ public void init(SSLFactory.Mode mode)
conf.getBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY,
SSLFactory.SSL_REQUIRE_CLIENT_CERT_DEFAULT);

long storesReloadInterval = conf.getLong(
resolvePropertyName(mode, SSL_STORES_RELOAD_INTERVAL_TPL_KEY),
DEFAULT_SSL_STORES_RELOAD_INTERVAL);

fileMonitoringTimer = new Timer("SSL Certificates Store Monitor", true);

// certificate store
String keystoreType =
conf.get(resolvePropertyName(mode, SSL_KEYSTORE_TYPE_TPL_KEY),
DEFAULT_KEYSTORE_TYPE);
KeyStore keystore = KeyStore.getInstance(keystoreType);
String keystoreKeyPassword = null;
if (requireClientCert || mode == SSLFactory.Mode.SERVER) {
String locationProperty =
resolvePropertyName(mode, SSL_KEYSTORE_LOCATION_TPL_KEY);
String keystoreLocation = conf.get(locationProperty, "");
if (keystoreLocation.isEmpty()) {
throw new GeneralSecurityException("The property '" + locationProperty +
"' has not been set in the ssl configuration file.");
}
String passwordProperty =
resolvePropertyName(mode, SSL_KEYSTORE_PASSWORD_TPL_KEY);
String keystorePassword = getPassword(conf, passwordProperty, "");
if (keystorePassword.isEmpty()) {
throw new GeneralSecurityException("The property '" + passwordProperty +
"' has not been set in the ssl configuration file.");
}
String keyPasswordProperty =
resolvePropertyName(mode, SSL_KEYSTORE_KEYPASSWORD_TPL_KEY);
// Key password defaults to the same value as store password for
// compatibility with legacy configurations that did not use a separate
// configuration property for key password.
keystoreKeyPassword = getPassword(
conf, keyPasswordProperty, keystorePassword);
if (LOG.isDebugEnabled()) {
LOG.debug(mode.toString() + " KeyStore: " + keystoreLocation);
}
conf.get(resolvePropertyName(mode, SSL_KEYSTORE_TYPE_TPL_KEY),
DEFAULT_KEYSTORE_TYPE);

InputStream is = Files.newInputStream(Paths.get(keystoreLocation));
try {
keystore.load(is, keystorePassword.toCharArray());
} finally {
is.close();
}
if (LOG.isDebugEnabled()) {
LOG.debug(mode.toString() + " Loaded KeyStore: " + keystoreLocation);
}
if (requireClientCert || mode == SSLFactory.Mode.SERVER) {
createKeyManagersFromConfiguration(mode, keystoreType, storesReloadInterval);
} else {
KeyStore keystore = KeyStore.getInstance(keystoreType);
keystore.load(null, null);
KeyManagerFactory keyMgrFactory = KeyManagerFactory
.getInstance(SSLFactory.SSLCERTIFICATE);

keyMgrFactory.init(keystore, null);
keyManagers = keyMgrFactory.getKeyManagers();
}
KeyManagerFactory keyMgrFactory = KeyManagerFactory
.getInstance(SSLFactory.SSLCERTIFICATE);

keyMgrFactory.init(keystore, (keystoreKeyPassword != null) ?
keystoreKeyPassword.toCharArray() : null);
keyManagers = keyMgrFactory.getKeyManagers();

//trust store
String truststoreType =
Expand All @@ -199,33 +283,7 @@ public void init(SSLFactory.Mode mode)
resolvePropertyName(mode, SSL_TRUSTSTORE_LOCATION_TPL_KEY);
String truststoreLocation = conf.get(locationProperty, "");
if (!truststoreLocation.isEmpty()) {
String passwordProperty = resolvePropertyName(mode,
SSL_TRUSTSTORE_PASSWORD_TPL_KEY);
String truststorePassword = getPassword(conf, passwordProperty, "");
if (truststorePassword.isEmpty()) {
// An empty trust store password is legal; the trust store password
// is only required when writing to a trust store. Otherwise it's
// an optional integrity check.
truststorePassword = null;
}
long truststoreReloadInterval =
conf.getLong(
resolvePropertyName(mode, SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY),
DEFAULT_SSL_TRUSTSTORE_RELOAD_INTERVAL);

if (LOG.isDebugEnabled()) {
LOG.debug(mode.toString() + " TrustStore: " + truststoreLocation);
}

trustManager = new ReloadingX509TrustManager(truststoreType,
truststoreLocation,
truststorePassword,
truststoreReloadInterval);
trustManager.init();
if (LOG.isDebugEnabled()) {
LOG.debug(mode.toString() + " Loaded TrustStore: " + truststoreLocation);
}
trustManagers = new TrustManager[]{trustManager};
createTrustManagersFromConfiguration(mode, truststoreType, truststoreLocation, storesReloadInterval);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("The property '" + locationProperty + "' has not been set, " +
Expand Down Expand Up @@ -256,7 +314,7 @@ String getPassword(Configuration conf, String alias, String defaultPass) {
@Override
public synchronized void destroy() {
if (trustManager != null) {
trustManager.destroy();
fileMonitoringTimer.cancel();
trustManager = null;
keyManagers = null;
trustManagers = null;
Expand Down
Loading