diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java index ace44ba9a39e..ffd763280476 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java @@ -32,6 +32,7 @@ import org.apache.hadoop.security.token.Token; import java.io.IOException; +import java.time.Duration; import java.util.EnumSet; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -69,11 +70,20 @@ class TokenHelper { HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT, TimeUnit.MILLISECONDS); - String certId = certClient.getCertificate().getSerialNumber().toString(); + long certificateGracePeriod = Duration.parse( + conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT)) + .toMillis(); + if (expiryTime > certificateGracePeriod) { + throw new IllegalArgumentException("Certificate grace period " + + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + + " should be greater than maximum block/container token lifetime " + + HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME); + } if (blockTokenEnabled) { blockTokenMgr = new OzoneBlockTokenSecretManager( - securityConfig, expiryTime, certId); + securityConfig, expiryTime); blockTokenMgr.start(certClient); } else { blockTokenMgr = null; @@ -81,7 +91,7 @@ class TokenHelper { if (containerTokenEnabled) { containerTokenMgr = new ContainerTokenSecretManager( - securityConfig, expiryTime, certId); + securityConfig, expiryTime); containerTokenMgr.start(certClient); } else { containerTokenMgr = null; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretKey.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretKey.java index 0dd5fd4d289f..833a4668b342 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretKey.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretKey.java @@ -49,13 +49,16 @@ public class OzoneSecretKey implements Writable { private PrivateKey privateKey; private PublicKey publicKey; private SecurityConfig securityConfig; + private String certSerialId; - public OzoneSecretKey(int keyId, long expiryDate, KeyPair keyPair) { + public OzoneSecretKey(int keyId, long expiryDate, KeyPair keyPair, + String certificateSerialId) { Preconditions.checkNotNull(keyId); this.keyId = keyId; this.expiryDate = expiryDate; this.privateKey = keyPair.getPrivate(); this.publicKey = keyPair.getPublic(); + this.certSerialId = certificateSerialId; } /* @@ -89,6 +92,10 @@ public PublicKey getPublicKey() { return publicKey; } + public String getCertSerialId() { + return certSerialId; + } + public byte[] getEncodedPrivateKey() { return privateKey.getEncoded(); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java index 575dc7e39169..4b1bdc053d36 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateNotification; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.token.SecretManager; @@ -36,7 +37,9 @@ import java.security.PrivateKey; import java.security.Signature; import java.security.SignatureException; +import java.security.cert.X509Certificate; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** * SecretManager for Ozone Master. Responsible for signing identifiers with @@ -45,7 +48,7 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public abstract class OzoneSecretManager - extends SecretManager { + extends SecretManager implements CertificateNotification { private final Logger logger; /** @@ -57,7 +60,7 @@ public abstract class OzoneSecretManager private final Text service; private CertificateClient certClient; private volatile boolean running; - private OzoneSecretKey currentKey; + private AtomicReference currentKey; private AtomicInteger currentKeyId; private AtomicInteger tokenSequenceNumber; @@ -81,6 +84,7 @@ public OzoneSecretManager(SecurityConfig secureConf, long tokenMaxLifetime, tokenSequenceNumber = new AtomicInteger(); this.service = service; this.logger = logger; + this.currentKey = new AtomicReference<>(); } @@ -112,12 +116,12 @@ public byte[] createPassword(byte[] identifier, PrivateKey privateKey) public byte[] createPassword(T identifier) { if (logger.isDebugEnabled()) { logger.debug("Creating password for identifier: {}, currentKey: {}", - formatTokenId(identifier), currentKey.getKeyId()); + formatTokenId(identifier), currentKey.get().getKeyId()); } byte[] password = null; try { password = createPassword(identifier.getBytes(), - currentKey.getPrivateKey()); + currentKey.get().getPrivateKey()); } catch (IOException ioe) { logger.error("Could not store token {}!!", formatTokenId(identifier), ioe); @@ -166,16 +170,33 @@ public int incrementDelegationTokenSeqNum() { * Update the current master key. This is called once by start method before * tokenRemoverThread is created, */ - private OzoneSecretKey updateCurrentKey(KeyPair keyPair) throws IOException { - logger.info("Updating the current master key for generating tokens"); + private OzoneSecretKey updateCurrentKey(KeyPair keyPair, + X509Certificate certificate) { + logger.info("Updating current master key for generating tokens. Cert id {}", + certificate.getSerialNumber().toString()); - // TODO: fix me based on the certificate expire time to set the key - // expire time. int newCurrentId = incrementCurrentKeyId(); - OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, -1, - keyPair); - currentKey = newKey; - return currentKey; + OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, + certificate.getNotAfter().getTime(), keyPair, + certificate.getSerialNumber().toString()); + currentKey.set(newKey); + return newKey; + } + + public void notifyCertificateRenewed(String oldCertId, String newCertId) { + if (!oldCertId.equals(getCertSerialId())) { + logger.info("Old certificate Id doesn't match. Holding {}, oldCertId {}", + getCertSerialId(), oldCertId); + } + if (!newCertId.equals( + certClient.getCertificate().getSerialNumber().toString())) { + logger.info("New certificate Id doesn't match. Holding in caClient {}," + + " newCertId {}", newCertId, + certClient.getCertificate().getSerialNumber().toString()); + } + logger.info("Certificate is changed from {} to {}", oldCertId, newCertId); + updateCurrentKey(new KeyPair(certClient.getPublicKey(), + certClient.getPrivateKey()), certClient.getCertificate()); } public String formatTokenId(T id) { @@ -193,7 +214,8 @@ public synchronized void start(CertificateClient client) Preconditions.checkState(!isRunning()); setCertClient(client); updateCurrentKey(new KeyPair(certClient.getPublicKey(), - certClient.getPrivateKey())); + certClient.getPrivateKey()), certClient.getCertificate()); + client.registerNotificationReceiver(this); setIsRunning(true); } @@ -236,13 +258,17 @@ public void setIsRunning(boolean val) { } public OzoneSecretKey getCurrentKey() { - return currentKey; + return currentKey.get(); } public AtomicInteger getCurrentKeyId() { return currentKeyId; } + public String getCertSerialId() { + return currentKey.get().getCertSerialId(); + } + public AtomicInteger getTokenSequenceNumber() { return tokenSequenceNumber; } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java index 1a8b33f403b4..0d3cb7293846 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java @@ -141,7 +141,7 @@ private void createKeyManagers(Mode mode) throws public synchronized void init(Mode mode, boolean requireClientAuth) throws IOException, GeneralSecurityException { - monitoringTimer = new Timer(caClient.getComponentName() + "-" + monitoringTimer = new Timer(caClient.getComponentName() + "-" + mode + "-" + SSL_MONITORING_THREAD_NAME, true); // key manager diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java index 2015c56645e9..e9f37f3de120 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java @@ -41,9 +41,8 @@ public class ContainerTokenSecretManager private static final Logger LOG = LoggerFactory.getLogger(ContainerTokenSecretManager.class); - public ContainerTokenSecretManager(SecurityConfig conf, - long tokenLifetime, String certSerialId) { - super(conf, tokenLifetime, certSerialId, LOG); + public ContainerTokenSecretManager(SecurityConfig conf, long tokenLifetime) { + super(conf, tokenLifetime, LOG); } public ContainerTokenIdentifier createIdentifier(String user, diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java index 886d6b354ab6..3dc7a395a1b4 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java @@ -44,9 +44,8 @@ public class OzoneBlockTokenSecretManager extends private static final Logger LOG = LoggerFactory .getLogger(OzoneBlockTokenSecretManager.class); - public OzoneBlockTokenSecretManager(SecurityConfig conf, - long tokenLifetime, String omCertSerialId) { - super(conf, tokenLifetime, omCertSerialId, LOG); + public OzoneBlockTokenSecretManager(SecurityConfig conf, long tokenLifetime) { + super(conf, tokenLifetime, LOG); } @Override diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java index a40317880916..966cf1cf5f39 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java @@ -39,12 +39,9 @@ private static final Text SERVICE = new Text("HDDS_SERVICE"); - private final String certSerialId; - protected ShortLivedTokenSecretManager(SecurityConfig conf, - long tokenLifetime, String certSerialId, Logger logger) { + long tokenLifetime, Logger logger) { super(conf, tokenLifetime, tokenLifetime, SERVICE, logger); - this.certSerialId = certSerialId; } @Override @@ -95,10 +92,6 @@ protected Instant getTokenExpiryTime() { return Instant.now().plusMillis(getTokenMaxLifetime()); } - protected String getCertSerialId() { - return certSerialId; - } - public Token generateToken(T tokenIdentifier) { return new Token<>(tokenIdentifier.getBytes(), createPassword(tokenIdentifier), tokenIdentifier.getKind(), diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java index 641b26edb1a2..caeca376bf3b 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java @@ -362,4 +362,10 @@ default void assertValidKeysAndCertificate() throws OzoneSecurityException { * Return the store factory for key manager and trust manager for client. */ KeyStoresFactory getClientKeyStoresFactory() throws CertificateException; + + /** + * Register a receiver that will be called after the certificate renewed. + * @param receiver + */ + void registerNotificationReceiver(CertificateNotification receiver); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java new file mode 100644 index 000000000000..948acaf2f724 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.hdds.security.x509.certificate.client; + +/** + * Class should implement this interface if it wants to be notified when there + * is some changes in Certificate. + */ +public interface CertificateNotification { + /** + * Notify the class implementing this interface that certificate is renewed. + * Note that the new leader can possibly be this server. + * + * @param oldCertId The old cert id before renew. + * @param newCertId The new cert id after renew. + */ + void notifyCertificateRenewed(String oldCertId, String newCertId); +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index 8647c324f54d..21a7dc59d301 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@ -46,10 +46,12 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Random; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -142,6 +144,7 @@ public abstract class DefaultCertificateClient implements CertificateClient { private Consumer certIdSaveCallback; private Runnable shutdownCallback; private SCMSecurityProtocolClientSideTranslatorPB scmSecurityProtocolClient; + private Set notificationReceivers; DefaultCertificateClient(SecurityConfig securityConfig, Logger log, String certSerialId, String component, @@ -155,6 +158,7 @@ public abstract class DefaultCertificateClient implements CertificateClient { this.component = component; this.certIdSaveCallback = saveCertId; this.shutdownCallback = shutdown; + this.notificationReceivers = new HashSet<>(); loadAllCertificates(); } @@ -1098,6 +1102,17 @@ public KeyStoresFactory getClientKeyStoresFactory() return clientKeyStoresFactory; } + /** + * Register a receiver that will be called after the certificate renewed. + * @param receiver + */ + @Override + public void registerNotificationReceiver(CertificateNotification receiver) { + synchronized (notificationReceivers) { + notificationReceivers.add(receiver); + } + } + @Override public synchronized void close() throws IOException { if (executorService != null) { @@ -1390,8 +1405,9 @@ public synchronized void startCertificateMonitor() { } this.executorService.scheduleAtFixedRate(new CertificateLifetimeMonitor(), timeBeforeGracePeriod, interval, TimeUnit.MILLISECONDS); - getLogger().info("CertificateLifetimeMonitor is started with first delay" + - " {} ms and interval {} ms.", timeBeforeGracePeriod, interval); + getLogger().info("CertificateLifetimeMonitor for {} is started with " + + "first delay {} ms and interval {} ms.", component, + timeBeforeGracePeriod, interval); } /** @@ -1403,7 +1419,8 @@ public void run() { renewLock.lock(); try { - Duration timeLeft = timeBeforeExpiryGracePeriod(getCertificate()); + X509Certificate currentCert = getCertificate(); + Duration timeLeft = timeBeforeExpiryGracePeriod(currentCert); if (timeLeft.isZero()) { String newCertId; try { @@ -1434,6 +1451,9 @@ public void run() { reloadKeyAndCertificate(newCertId); // cleanup backup directory cleanBackupDir(); + // notify notification receivers + notificationReceivers.forEach(r -> r.notifyCertificateRenewed( + currentCert.getSerialNumber().toString(), newCertId)); } } finally { renewLock.unlock(); diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java index ad435a0f30b5..00b8b339e99b 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java @@ -105,7 +105,7 @@ public void setUp() throws Exception { .generateCertificate("CN=OzoneMaster", keyPair, 30, ALGORITHM); omCertSerialId = x509Certificate.getSerialNumber().toString(); secretManager = new OzoneBlockTokenSecretManager(securityConfig, - TimeUnit.HOURS.toMillis(1), omCertSerialId); + TimeUnit.HOURS.toMillis(1)); client = Mockito.mock(DefaultCertificateClient.class); when(client.getCertificate()).thenReturn(x509Certificate); when(client.getCertificate(anyString())). diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java index 2744823cb248..009b7a0105c3 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java @@ -231,7 +231,7 @@ private static Token anyToken() { private class MockTokenManager extends ShortLivedTokenSecretManager { MockTokenManager(SecurityConfig conf) { - super(conf, TimeUnit.HOURS.toMillis(1), CERT_ID, LOG); + super(conf, TimeUnit.HOURS.toMillis(1), LOG); } @Override diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java index 1c5e1118e0bb..6c4444babe75 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateNotification; import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.crl.CRLInfo; import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; @@ -253,6 +254,10 @@ public KeyStoresFactory getClientKeyStoresFactory() return null; } + @Override + public void registerNotificationReceiver(CertificateNotification receiver) { + } + public void renewKey() throws Exception { KeyPair newKeyPair = KeyStoreTestUtil.generateKeyPair("RSA"); X509Certificate newCert = KeyStoreTestUtil.generateCertificate( diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java index ac6716c7d45c..c2214635959e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.hdds.utils.db.RocksDBCheckpoint; @@ -58,11 +58,11 @@ public class SCMSnapshotProvider { private Map peerNodesMap; - private final SCMCertificateClient scmCertificateClient; + private final CertificateClient scmCertificateClient; public SCMSnapshotProvider(ConfigurationSource conf, List peerNodes, - SCMCertificateClient scmCertificateClient) { + CertificateClient scmCertificateClient) { LOG.info("Initializing SCM Snapshot Provider"); this.conf = conf; this.scmCertificateClient = scmCertificateClient; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 1452c7f6b3a5..0131d1b7a33b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -25,6 +25,7 @@ import com.google.common.base.Preconditions; import com.google.protobuf.BlockingService; +import java.time.Duration; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; @@ -74,6 +75,7 @@ import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore; import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultCAProfile; import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile; +import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.server.OzoneAdmins; @@ -254,7 +256,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl private ReplicationManager replicationManager; private SCMSafeModeManager scmSafeModeManager; - private SCMCertificateClient scmCertificateClient; + private CertificateClient scmCertificateClient; private ContainerTokenSecretManager containerTokenMgr; private JvmPauseMonitor jvmPauseMonitor; @@ -892,10 +894,15 @@ public CertificateServer getScmCertificateServer() { return getSecurityProtocolServer().getScmCertificateServer(); } - public SCMCertificateClient getScmCertificateClient() { + public CertificateClient getScmCertificateClient() { return scmCertificateClient; } + @VisibleForTesting + public void setScmCertificateClient(CertificateClient client) { + scmCertificateClient = client; + } + private ContainerTokenSecretManager createContainerTokenSecretManager( OzoneConfiguration conf) throws IOException { @@ -903,6 +910,15 @@ private ContainerTokenSecretManager createContainerTokenSecretManager( HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT, TimeUnit.MILLISECONDS); + long certificateGracePeriod = Duration.parse( + conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT)).toMillis(); + if (expiryTime > certificateGracePeriod) { + throw new IllegalArgumentException("Certificate grace period " + + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + + " should be greater than maximum block/container token lifetime " + + HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME); + } // Means this is an upgraded cluster and it has no sub-ca, // so SCM Certificate client is not initialized. To make Tokens @@ -926,10 +942,8 @@ private ContainerTokenSecretManager createContainerTokenSecretManager( scmCertificateClient = new SCMCertificateClient(securityConfig, certSerialNumber, SCM_ROOT_CA_COMPONENT_NAME); } - String certId = scmCertificateClient.getCertificate().getSerialNumber() - .toString(); return new ContainerTokenSecretManager(securityConfig, - expiryTime, certId); + expiryTime); } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java index 01fcaf893237..19f3e7c4a25c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java @@ -382,7 +382,8 @@ public String toString() { .append(", strToSign=").append(getStrToSign()) .append(", signature=").append(getSignature()) .append(", awsAccessKeyId=").append(getAwsAccessId()) - .append(", omServiceId=").append(getOmServiceId()); + .append(", omServiceId=").append(getOmServiceId()) + .append(", omCertSerialId=").append(getOmCertSerialId()); return buffer.toString(); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java index 2a8a945a0b27..9b7b3d90c392 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java @@ -628,11 +628,11 @@ public static void startCluster(OzoneConfiguration conf) throws Exception { conf.setFromObject(writableECContainerProviderConfig); OzoneManager.setTestSecureOmFlag(true); - certClient = new CertificateClientTestImpl(config); + certClient = new CertificateClientTestImpl(conf); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(NUM_DN) .setScmId(SCM_ID).setClusterId(CLUSTER_ID) - .setCertificateClient(new CertificateClientTestImpl(conf)) + .setCertificateClient(certClient) .build(); cluster.waitForClusterToBeReady(); cluster.getOzoneManager().startSecretManager(); @@ -681,10 +681,10 @@ public static void prepareData(int[][] ranges) throws Exception { SecurityConfig conf = new SecurityConfig(tweakedConfig); long tokenLifetime = TimeUnit.DAYS.toMillis(1); containerTokenGenerator = new ContainerTokenSecretManager( - conf, tokenLifetime, "1"); + conf, tokenLifetime); containerTokenGenerator.start(certClient); blockTokenGenerator = new OzoneBlockTokenSecretManager( - conf, tokenLifetime, "1"); + conf, tokenLifetime); blockTokenGenerator.start(certClient); containerToken = containerTokenGenerator .generateToken(ANY_USER, new ContainerID(containerID)); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index 5952ce5947cc..a55faa679d07 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.ozone; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; import java.io.File; import java.io.IOException; import java.net.InetAddress; @@ -46,15 +48,19 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.HddsTestUtils; +import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.ha.HASecurityUtils; import org.apache.hadoop.hdds.scm.ha.SCMHANodeDetails; import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider; import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.security.exception.SCMSecurityException; +import org.apache.hadoop.hdds.security.token.ContainerTokenIdentifier; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; @@ -72,6 +78,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.client.CertificateClientTestImpl; import org.apache.hadoop.ozone.common.Storage; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMStorage; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -95,6 +102,10 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY; @@ -161,7 +172,7 @@ @InterfaceAudience.Private public final class TestSecureOzoneCluster { - private static final String COMPONENT = "test"; + private static final String COMPONENT = "om"; private static final String OM_CERT_SERIAL_ID = "9879877970576"; private static final Logger LOG = LoggerFactory .getLogger(TestSecureOzoneCluster.class); @@ -187,6 +198,10 @@ public final class TestSecureOzoneCluster { private String scmId; private String omId; private OzoneManagerProtocolClientSideTranslatorPB omClient; + private KeyPair keyPair; + private Path omMetaDirPath; + private int certGraceTime = 10 * 1000; // 10s + private int delegationTokenMaxTime = 9 * 1000; // 9s @Before public void init() { @@ -211,11 +226,14 @@ public void init() { DefaultMetricsSystem.setMiniClusterMode(true); ExitUtils.disableSystemExit(); final String path = folder.newFolder().toString(); - Path metaDirPath = Paths.get(path, "om-meta"); - conf.set(OZONE_METADATA_DIRS, metaDirPath.toString()); + omMetaDirPath = Paths.get(path, "om-meta"); + conf.set(OZONE_METADATA_DIRS, omMetaDirPath.toString()); conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name()); - conf.set(HDDS_X509_RENEW_GRACE_DURATION, "PT5S"); // 5s + conf.set(HDDS_X509_RENEW_GRACE_DURATION, + Duration.ofMillis(certGraceTime).toString()); + conf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, + delegationTokenMaxTime); workDir = GenericTestUtils.getTestDir(getClass().getSimpleName()); clusterId = UUID.randomUUID().toString(); @@ -544,7 +562,7 @@ public void testAccessControlExceptionOnClient() throws Exception { private void generateKeyPair() throws Exception { HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(conf); - KeyPair keyPair = keyGenerator.generateKey(); + keyPair = keyGenerator.generateKey(); KeyCodec pemWriter = new KeyCodec(new SecurityConfig(conf), COMPONENT); pemWriter.writeKey(keyPair, true); } @@ -1036,6 +1054,7 @@ public void testCertificateRotationUnRecoverableFailure() throws Exception { omStorage.setOmCertSerialId(certId); omStorage.forceInitialize(); + // second cert as renew response X509CertificateHolder newCertHolder = generateX509CertHolder(conf, null, null, Duration.ofSeconds(certificateLifetime)); DNCertificateClient mockClient = mock(DNCertificateClient.class); @@ -1058,6 +1077,160 @@ public void testCertificateRotationUnRecoverableFailure() throws Exception { 1000, certificateLifetime * 1000); } + /** + * Tests delegation token renewal after a certificate renew. + */ + @Test + public void testDelegationTokenRenewCrossCertificateRenew() throws Exception { + try { + // Setup secure OM for start. + final int certLifetime = 40 * 1000; // 40s + OzoneConfiguration newConf = new OzoneConfiguration(conf); + newConf.set(HDDS_X509_DEFAULT_DURATION, + Duration.ofMillis(certLifetime).toString()); + newConf.set(HDDS_X509_RENEW_GRACE_DURATION, + Duration.ofMillis(certLifetime - 15 * 1000).toString()); + newConf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, + certLifetime - 20 * 1000); + + setupOm(newConf); + OzoneManager.setTestSecureOmFlag(true); + + CertificateClientTestImpl certClient = + new CertificateClientTestImpl(newConf, true); + X509Certificate omCert = certClient.getCertificate(); + String omCertId1 = omCert.getSerialNumber().toString(); + // Start OM + om.setCertClient(certClient); + om.start(); + GenericTestUtils.waitFor(() -> om.isLeaderReady(), 100, 10000); + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + + // Get first OM client which will authenticate via Kerberos + omClient = new OzoneManagerProtocolClientSideTranslatorPB( + OmTransportFactory.create(newConf, ugi, null), + RandomStringUtils.randomAscii(5)); + + // Since client is already connected get a delegation token + Token token1 = omClient.getDelegationToken( + new Text("om")); + + // Check if token is of right kind and renewer is running om instance + assertNotNull(token1); + assertEquals("OzoneToken", token1.getKind().toString()); + assertEquals(OmUtils.getOmRpcAddress(newConf), + token1.getService().toString()); + OzoneTokenIdentifier temp = new OzoneTokenIdentifier(); + ByteArrayInputStream buf = new ByteArrayInputStream( + token1.getIdentifier()); + DataInputStream in = new DataInputStream(buf); + temp.readFields(in); + assertEquals(omCertId1, temp.getOmCertSerialId()); + + // Renew delegation token + long expiryTime = omClient.renewDelegationToken(token1); + assertTrue(expiryTime > 0); + + // Wait for OM certificate to renew + GenericTestUtils.waitFor(() -> !omCertId1.equals( + certClient.getCertificate().getSerialNumber().toString()), + 100, certLifetime); + String omCertId2 = + certClient.getCertificate().getSerialNumber().toString(); + assertNotEquals(omCertId1, omCertId2); + // Get a new delegation token + Token token2 = omClient.getDelegationToken( + new Text("om")); + buf = new ByteArrayInputStream(token2.getIdentifier()); + in = new DataInputStream(buf); + temp.readFields(in); + assertEquals(omCertId2, temp.getOmCertSerialId()); + + // Because old certificate is still valid, so renew old token will succeed + expiryTime = omClient.renewDelegationToken(token1); + assertTrue(expiryTime > 0); + assertTrue(new Date(expiryTime).before(omCert.getNotAfter())); + } finally { + if (om != null) { + om.stop(); + } + IOUtils.closeQuietly(om); + } + } + + /** + * Tests container token renewal after a certificate renew. + */ + @Test + public void testContainerTokenRenewCrossCertificateRenew() throws Exception { + // Setup secure SCM for start. + final int certLifetime = 40 * 1000; // 40s + conf.set(HDDS_X509_DEFAULT_DURATION, + Duration.ofMillis(certLifetime).toString()); + conf.set(HDDS_X509_MAX_DURATION, + Duration.ofMillis(certLifetime).toString()); + conf.set(HDDS_X509_RENEW_GRACE_DURATION, + Duration.ofMillis(certLifetime - 15 * 1000).toString()); + conf.setBoolean(HDDS_CONTAINER_TOKEN_ENABLED, true); + conf.setLong(HDDS_BLOCK_TOKEN_EXPIRY_TIME, certLifetime - 20 * 1000); + + initSCM(); + scm = HddsTestUtils.getScmSimple(conf); + try { + CertificateClientTestImpl certClient = + new CertificateClientTestImpl(conf, true); + X509Certificate scmCert = certClient.getCertificate(); + String scmCertId1 = scmCert.getSerialNumber().toString(); + // Start SCM + scm.setScmCertificateClient(certClient); + scm.start(); + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + // Get SCM client which will authenticate via Kerberos + SCMContainerLocationFailoverProxyProvider proxyProvider = + new SCMContainerLocationFailoverProxyProvider(conf, ugi); + StorageContainerLocationProtocolClientSideTranslatorPB scmClient = + new StorageContainerLocationProtocolClientSideTranslatorPB( + proxyProvider); + + // Since client is already connected get a delegation token + ContainerID containerID = new ContainerID(1); + Token token1 = scmClient.getContainerToken(containerID); + + // Check if token is of right kind and renewer is running instance + assertNotNull(token1); + assertEquals(ContainerTokenIdentifier.KIND, token1.getKind()); + assertEquals(containerID.toString(), token1.getService().toString()); + ContainerTokenIdentifier temp = new ContainerTokenIdentifier(); + ByteArrayInputStream buf = new ByteArrayInputStream( + token1.getIdentifier()); + DataInputStream in = new DataInputStream(buf); + temp.readFields(in); + assertEquals(scmCertId1, temp.getCertSerialId()); + + // Wait for SCM certificate to renew + GenericTestUtils.waitFor(() -> !scmCertId1.equals( + certClient.getCertificate().getSerialNumber().toString()), + 100, certLifetime); + String scmCertId2 = + certClient.getCertificate().getSerialNumber().toString(); + assertNotEquals(scmCertId1, scmCertId2); + + // Get a new container token + containerID = new ContainerID(2); + Token token2 = scmClient.getContainerToken(containerID); + buf = new ByteArrayInputStream(token2.getIdentifier()); + in = new DataInputStream(buf); + temp.readFields(in); + assertEquals(scmCertId2, temp.getCertSerialId()); + } finally { + if (scm != null) { + scm.stop(); + } + } + } + public void validateCertificate(X509Certificate cert) throws Exception { // Assert that we indeed have a self signed certificate. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java index aed1a3852233..627a62907d60 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java @@ -19,9 +19,14 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.security.InvalidKeyException; import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; import java.security.PrivateKey; import java.security.PublicKey; +import java.security.Signature; +import java.security.SignatureException; import java.security.cert.CertStore; import java.security.cert.X509Certificate; import java.time.Duration; @@ -29,14 +34,23 @@ import java.time.ZoneId; import java.util.Collections; import java.util.Date; +import java.util.HashSet; import java.util.List; - +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultApprover; import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateNotification; import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate; import org.apache.hadoop.hdds.security.x509.crl.CRLInfo; @@ -52,6 +66,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION_DEFAULT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT; +import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateException.ErrorCode.CRYPTO_SIGNATURE_VERIFICATION_ERROR; /** * Test implementation for CertificateClient. To be used only for test @@ -70,13 +85,18 @@ public class CertificateClientTestImpl implements CertificateClient { private DefaultApprover approver; private KeyStoresFactory serverKeyStoresFactory; private KeyStoresFactory clientKeyStoresFactory; + private Map certificateMap; + private ScheduledExecutorService executorService; + private Set notificationReceivers; - public CertificateClientTestImpl(OzoneConfiguration conf) throws Exception { - this(conf, true); + public CertificateClientTestImpl(OzoneConfiguration conf) + throws Exception { + this(conf, false); } - public CertificateClientTestImpl(OzoneConfiguration conf, boolean rootCA) + public CertificateClientTestImpl(OzoneConfiguration conf, boolean autoRenew) throws Exception { + certificateMap = new ConcurrentHashMap<>(); securityConfig = new SecurityConfig(conf); keyGen = new HDDSKeyGenerator(securityConfig.getConfiguration()); keyPair = keyGen.generateKey(); @@ -100,6 +120,7 @@ public CertificateClientTestImpl(OzoneConfiguration conf, boolean rootCA) .makeCA(); rootCert = new JcaX509CertificateConverter().getCertificate( builder.build()); + certificateMap.put(rootCert.getSerialNumber().toString(), rootCert); // Generate normal certificate, signed by RootCA certificate approver = new DefaultApprover(new DefaultProfile(), securityConfig); @@ -126,11 +147,30 @@ public CertificateClientTestImpl(OzoneConfiguration conf, boolean rootCA) csrBuilder.build(), "scm1", "cluster1"); x509Certificate = new JcaX509CertificateConverter().getCertificate(certificateHolder); + certificateMap.put(x509Certificate.getSerialNumber().toString(), + x509Certificate); serverKeyStoresFactory = SecurityUtil.getServerKeyStoresFactory( securityConfig, this, true); clientKeyStoresFactory = SecurityUtil.getClientKeyStoresFactory( securityConfig, this, true); + + if (autoRenew) { + Duration gracePeriod = securityConfig.getRenewalGracePeriod(); + Date expireDate = x509Certificate.getNotAfter(); + LocalDateTime gracePeriodStart = expireDate.toInstant() + .atZone(ZoneId.systemDefault()).toLocalDateTime().minus(gracePeriod); + LocalDateTime currentTime = LocalDateTime.now(); + Duration delay = gracePeriodStart.isBefore(currentTime) ? Duration.ZERO : + Duration.between(currentTime, gracePeriodStart); + + executorService = Executors.newScheduledThreadPool(1, + new ThreadFactoryBuilder().setNameFormat("CertificateLifetimeMonitor") + .setDaemon(true).build()); + this.executorService.schedule(new RenewCertTask(), + delay.toMillis(), TimeUnit.MILLISECONDS); + } + notificationReceivers = new HashSet<>(); } @Override @@ -152,7 +192,7 @@ public PublicKey getPublicKey() { @Override public X509Certificate getCertificate(String certSerialId) throws CertificateException { - return x509Certificate; + return certificateMap.get(certSerialId); } @Override @@ -194,7 +234,18 @@ public boolean verifySignature(InputStream stream, byte[] signature, @Override public boolean verifySignature(byte[] data, byte[] signature, X509Certificate cert) throws CertificateException { - return true; + try { + Signature sign = Signature.getInstance(getSignatureAlgorithm(), + getSecurityProvider()); + sign.initVerify(cert); + sign.update(data); + return sign.verify(signature); + } catch (NoSuchAlgorithmException | NoSuchProviderException + | InvalidKeyException | SignatureException e) { + System.out.println("Error while signing the stream " + e.getMessage()); + throw new CertificateException("Error while signing the stream", e, + CRYPTO_SIGNATURE_VERIFICATION_ERROR); + } } @Override @@ -344,9 +395,31 @@ public void renewKey() throws Exception { // Save the new private key and certificate to file // Save certificate and private key to keyStore + X509Certificate oldCert = x509Certificate; keyPair = newKeyPair; x509Certificate = newX509Certificate; + certificateMap.put(x509Certificate.getSerialNumber().toString(), + x509Certificate); System.out.println(new Date() + " certificated is renewed"); + + // notify notification receivers + notificationReceivers.forEach(r -> r.notifyCertificateRenewed( + oldCert.getSerialNumber().toString(), + x509Certificate.getSerialNumber().toString())); + } + + /** + * Task to renew certificate. + */ + public class RenewCertTask implements Runnable { + @Override + public void run() { + try { + renewKey(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } @Override @@ -359,6 +432,13 @@ public KeyStoresFactory getClientKeyStoresFactory() { return clientKeyStoresFactory; } + @Override + public void registerNotificationReceiver(CertificateNotification receiver) { + synchronized (notificationReceivers) { + notificationReceivers.add(receiver); + } + } + @Override public void close() throws IOException { if (serverKeyStoresFactory != null) { @@ -368,5 +448,9 @@ public void close() throws IOException { if (clientKeyStoresFactory != null) { clientKeyStoresFactory.destroy(); } + + if (executorService != null) { + executorService.shutdown(); + } } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java index 82af642e5b5a..cbf57a96d8d0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java @@ -119,8 +119,7 @@ public static void init() throws Exception { .setCertificateClient(certificateClientTest) .build(); secretManager = new OzoneBlockTokenSecretManager(new SecurityConfig(conf), - 60 * 60, certificateClientTest.getCertificate(). - getSerialNumber().toString()); + 60 * 60); secretManager.start(certificateClientTest); cluster.getOzoneManager().startSecretManager(); cluster.waitForClusterToBeReady(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java index 19fe829efe1a..ad96d4fa4b5f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java @@ -146,9 +146,9 @@ public void setup() throws Exception { HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, "1s", TimeUnit.MILLISECONDS); - caClient = new CertificateClientTestImpl(conf, false); + caClient = new CertificateClientTestImpl(conf); secretManager = new ContainerTokenSecretManager(new SecurityConfig(conf), - expiryTime, caClient.getCertificate().getSerialNumber().toString()); + expiryTime); } @Test(expected = CertificateExpiredException.class) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java index c94bfb20fd00..95eeadc08e9c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java @@ -123,9 +123,7 @@ public void setup() throws Exception { secConfig = new SecurityConfig(conf); caClient = new CertificateClientTestImpl(conf); secretManager = new ContainerTokenSecretManager( - new SecurityConfig(conf), - TimeUnit.DAYS.toMillis(1), - caClient.getCertificate().getSerialNumber().toString()); + new SecurityConfig(conf), TimeUnit.DAYS.toMillis(1)); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java index 62e7ecbda682..4b87ed1e7a3c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java @@ -130,17 +130,15 @@ public static void setup() throws Exception { CONF.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true); caClient = new CertificateClientTestImpl(CONF); - String certSerialId = - caClient.getCertificate().getSerialNumber().toString(); SecurityConfig secConf = new SecurityConfig(CONF); long tokenLifetime = TimeUnit.HOURS.toMillis(1); blockTokenSecretManager = new OzoneBlockTokenSecretManager( - secConf, tokenLifetime, certSerialId); + secConf, tokenLifetime); blockTokenSecretManager.start(caClient); containerTokenSecretManager = new ContainerTokenSecretManager( - secConf, tokenLifetime, certSerialId); + secConf, tokenLifetime); containerTokenSecretManager.start(caClient); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 215f0e5daec0..fbcedcf0b44a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -31,6 +31,7 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.security.PrivilegedExceptionAction; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -953,6 +954,15 @@ private OzoneDelegationTokenSecretManager createDelegationTokenSecretManager( conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + long certificateGracePeriod = Duration.parse( + conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT)).toMillis(); + if (tokenMaxLifetime > certificateGracePeriod) { + throw new IllegalArgumentException("Certificate grace period " + + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + + " should be greater than maximum delegation token lifetime " + + OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY); + } return new OzoneDelegationTokenSecretManager.Builder() .setConf(conf) @@ -974,13 +984,21 @@ private OzoneBlockTokenSecretManager createBlockTokenSecretManager( HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT, TimeUnit.MILLISECONDS); + long certificateGracePeriod = Duration.parse( + conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT)).toMillis(); + if (expiryTime > certificateGracePeriod) { + throw new IllegalArgumentException("Certificate grace period " + + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + + " should be greater than maximum block token lifetime " + + HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME); + } // TODO: Pass OM cert serial ID. if (testSecureOmFlag) { - return new OzoneBlockTokenSecretManager(secConfig, expiryTime, "1"); + return new OzoneBlockTokenSecretManager(secConfig, expiryTime); } Objects.requireNonNull(certClient); - return new OzoneBlockTokenSecretManager(secConfig, expiryTime, - certClient.getCertificate().getSerialNumber().toString()); + return new OzoneBlockTokenSecretManager(secConfig, expiryTime); } private void stopSecretManager() { @@ -2091,6 +2109,9 @@ public void stop() { OMPerformanceMetrics.unregister(); RatisDropwizardExports.clear(ratisMetricsMap, ratisReporterList); scmClient.close(); + if (certClient != null) { + certClient.close(); + } } catch (Exception e) { LOG.error("OzoneManager stop failed.", e); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java index 3e023d1f92ec..0565c1507a92 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java @@ -264,21 +264,10 @@ private void updateIdentifierDetails(OzoneTokenIdentifier identifier) { identifier.setMasterKeyId(getCurrentKey().getKeyId()); identifier.setSequenceNumber(sequenceNum); identifier.setMaxDate(now + getTokenMaxLifetime()); - identifier.setOmCertSerialId(getOmCertificateSerialId()); + identifier.setOmCertSerialId(getCertSerialId()); identifier.setOmServiceId(getOmServiceId()); } - /** - * Get OM certificate serial id. - * */ - private String getOmCertificateSerialId() { - if (omCertificateSerialId == null) { - omCertificateSerialId = - getCertClient().getCertificate().getSerialNumber().toString(); - } - return omCertificateSerialId; - } - private String getOmServiceId() { return omServiceId; } @@ -480,6 +469,7 @@ public boolean verifySignature(OzoneTokenIdentifier identifier, try { signerCert.checkValidity(); } catch (CertificateExpiredException | CertificateNotYetValidException e) { + LOG.error("signerCert {} is invalid", signerCert, e); return false; } @@ -487,6 +477,7 @@ public boolean verifySignature(OzoneTokenIdentifier identifier, return getCertClient().verifySignature(identifier.getBytes(), password, signerCert); } catch (CertificateException e) { + LOG.error("verifySignature with signerCert {} failed", signerCert, e); return false; } }