diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java index e9a6e59ff16c..26abac2d95c5 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java @@ -327,7 +327,7 @@ public void testCertificateRotation() throws Exception { client.getCertificate().getSerialNumber().toString())); // start monitor task to renew key and cert - client.startCertificateMonitor(); + client.startCertificateRenewerService(); // check after renew, client will have the new cert ID GenericTestUtils.waitFor(() -> { @@ -402,7 +402,7 @@ public void testCertificateRotationRecoverableFailure() throws Exception { client.getCertificate().getSerialNumber().toString())); // start monitor task to renew key and cert - client.startCertificateMonitor(); + client.startCertificateRenewerService(); // certificate failed to renew, client still hold the old expired cert. Thread.sleep(CERT_LIFETIME * 1000); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index fb7587a838cb..a26bd12188ce 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@ -52,6 +52,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -127,6 +128,7 @@ public abstract class DefaultCertificateClient implements CertificateClient { private Runnable shutdownCallback; private SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient; private final Set notificationReceivers; + private RootCaRotationPoller rootCaRotationPoller; protected DefaultCertificateClient( SecurityConfig securityConfig, @@ -169,19 +171,33 @@ private synchronized void loadAllCertificates() { return; } - if (shouldStartCertificateMonitor()) { + if (shouldStartCertificateRenewerService()) { + startRootCaRotationPoller(); if (certPath != null && executorService == null) { - startCertificateMonitor(); + startCertificateRenewerService(); } else { if (executorService != null) { - getLogger().debug("CertificateLifetimeMonitor is already started."); + getLogger().debug("CertificateRenewerService is already started."); } else { getLogger().warn("Component certificate was not loaded."); } } } else { - getLogger().info("CertificateLifetimeMonitor is disabled for {}", - component); + getLogger().info("CertificateRenewerService and root ca rotation " + + "polling is disabled for {}", component); + } + } + + private void startRootCaRotationPoller() { + if (rootCaRotationPoller == null) { + rootCaRotationPoller = new RootCaRotationPoller(securityConfig, + rootCaCertificates, scmSecurityClient); + rootCaRotationPoller.addRootCARotationProcessor( + this::getRootCaRotationListener); + rootCaRotationPoller.run(); + } else { + getLogger().debug("Root CA certificate rotation poller is already " + + "started."); } } @@ -985,6 +1001,10 @@ public synchronized void close() throws IOException { executorService = null; } + if (rootCaRotationPoller != null) { + rootCaRotationPoller.close(); + } + if (serverKeyStoresFactory != null) { serverKeyStoresFactory.destroy(); } @@ -1292,11 +1312,21 @@ public SCMSecurityProtocolClientSideTranslatorPB getScmSecureClient() return scmSecurityClient; } - protected boolean shouldStartCertificateMonitor() { + protected boolean shouldStartCertificateRenewerService() { return true; } - public synchronized void startCertificateMonitor() { + public synchronized CompletableFuture getRootCaRotationListener( + List rootCAs) { + if (rootCaCertificates.containsAll(rootCAs)) { + return CompletableFuture.completedFuture(null); + } + CertificateRenewerService renewerService = + new CertificateRenewerService(true); + return CompletableFuture.runAsync(renewerService, executorService); + } + + public synchronized void startCertificateRenewerService() { Preconditions.checkNotNull(getCertificate(), "Component certificate should not be empty"); // Schedule task to refresh certificate before it expires @@ -1310,13 +1340,13 @@ public synchronized void startCertificateMonitor() { if (executorService == null) { executorService = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat( - getComponentName() + "-CertificateLifetimeMonitor") + getComponentName() + "-CertificateRenewerService") .setDaemon(true).build()); } this.executorService.scheduleAtFixedRate( new CertificateRenewerService(false), timeBeforeGracePeriod, interval, TimeUnit.MILLISECONDS); - getLogger().info("CertificateLifetimeMonitor for {} is started with " + + getLogger().info("CertificateRenewerService for {} is started with " + "first delay {} ms and interval {} ms.", component, timeBeforeGracePeriod, interval); } @@ -1349,10 +1379,11 @@ public void run() { } String newCertId; try { - getLogger().info("Current certificate {} has entered the expiry" + - " grace period {}. Starting renew key and certs.", + getLogger().info("Current certificate {} needs to be renewed " + + "remaining grace period {}. Forced renewal due to root ca " + + "rotation: {}.", currentCert.getSerialNumber().toString(), - timeLeft, securityConfig.getRenewalGracePeriod()); + timeLeft, forceRenewal); newCertId = renewAndStoreKeyAndCertificate(forceRenewal); } catch (CertificateException e) { if (e.errorCode() == diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java index 32a9326e460b..262bb4b57010 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java @@ -173,7 +173,7 @@ public CertificateSignRequest.Builder getCSRBuilder() } @Override - protected boolean shouldStartCertificateMonitor() { + protected boolean shouldStartCertificateRenewerService() { return false; } diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClientTestImpl.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClientTestImpl.java index 57801a5c13b1..40395ac128be 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClientTestImpl.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClientTestImpl.java @@ -166,7 +166,7 @@ public CertificateClientTestImpl(OzoneConfiguration conf, boolean autoRenew) Duration.between(currentTime, gracePeriodStart); executorService = Executors.newScheduledThreadPool(1, - new ThreadFactoryBuilder().setNameFormat("CertificateLifetimeMonitor") + new ThreadFactoryBuilder().setNameFormat("CertificateRenewerService") .setDaemon(true).build()); this.executorService.schedule(new RenewCertTask(), delay.toMillis(), TimeUnit.MILLISECONDS); diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDefaultCertificateClient.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDefaultCertificateClient.java index 1725a0b510ea..3bd352713a34 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDefaultCertificateClient.java @@ -656,7 +656,7 @@ protected String signAndStoreCertificate( Thread.enumerate(threads); Predicate monitorFilterPredicate = t -> t != null - && t.getName().equals(compName + "-CertificateLifetimeMonitor"); + && t.getName().equals(compName + "-CertificateRenewerService"); long monitorThreadCount = Arrays.stream(threads) .filter(monitorFilterPredicate) .count(); diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/root-ca-rotation.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/root-ca-rotation.yaml index 8f7b944b0fb8..9dccb44945df 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/root-ca-rotation.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/root-ca-rotation.yaml @@ -25,6 +25,7 @@ x-root-cert-rotation-config: - OZONE-SITE.XML_hdds.x509.renew.grace.duration=PT45S - OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval=PT1S - OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout=PT20S + - OZONE-SITE.XML_hdds.x509.rootca.certificate.polling.interval=PT10s - OZONE-SITE.XML_hdds.block.token.expiry.time=15s - OZONE-SITE.XML_ozone.manager.delegation.token.max-lifetime=15s - OZONE-SITE.XML_ozone.manager.delegation.token.renew-interval=15s diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-root-ca-rotation.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test-root-ca-rotation.sh index 66f1a6d01aec..9858d41eb8d4 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-root-ca-rotation.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test-root-ca-rotation.sh @@ -40,6 +40,7 @@ wait_for_execute_command scm 30 "jps | grep StorageContainerManagerStarter | se # wait and verify root CA is rotated wait_for_execute_command scm 180 "ozone admin cert info 2" +wait_for_execute_command datanode 30 "find /data/metadata/dn/certs/ROOTCA-2.crt" # verify om operations and data operations execute_commands_in_container scm "ozone sh volume create /r-v1 && ozone sh bucket create /r-v1/r-b1"