HDDS-8593. Add RootCARotationPoller to CertClient

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java

@@ -327,7 +327,7 @@ public void testCertificateRotation() throws Exception {
         client.getCertificate().getSerialNumber().toString()));
 
     // start monitor task to renew key and cert
-    client.startCertificateMonitor();
+    client.startCertificateRenewerService();
 
     // check after renew, client will have the new cert ID
     GenericTestUtils.waitFor(() -> {
@@ -402,7 +402,7 @@ public void testCertificateRotationRecoverableFailure() throws Exception {
         client.getCertificate().getSerialNumber().toString()));
 
     // start monitor task to renew key and cert
-    client.startCertificateMonitor();
+    client.startCertificateRenewerService();
 
     // certificate failed to renew, client still hold the old expired cert.
     Thread.sleep(CERT_LIFETIME * 1000);

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java

@@ -52,6 +52,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -127,6 +128,7 @@ public abstract class DefaultCertificateClient implements CertificateClient {
   private Runnable shutdownCallback;
   private SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient;
   private final Set<CertificateNotification> notificationReceivers;
+  private RootCaRotationPoller rootCaRotationPoller;
 
   protected DefaultCertificateClient(
       SecurityConfig securityConfig,
@@ -169,19 +171,33 @@ private synchronized void loadAllCertificates() {
       return;
     }
 
-    if (shouldStartCertificateMonitor()) {
+    if (shouldStartCertificateRenewerService()) {
+      startRootCaRotationPoller();
       if (certPath != null && executorService == null) {
-        startCertificateMonitor();
+        startCertificateRenewerService();
       } else {
         if (executorService != null) {
-          getLogger().debug("CertificateLifetimeMonitor is already started.");
+          getLogger().debug("CertificateRenewerService is already started.");
         } else {
           getLogger().warn("Component certificate was not loaded.");
         }
       }
     } else {
-      getLogger().info("CertificateLifetimeMonitor is disabled for {}",
-          component);
+      getLogger().info("CertificateRenewerService and root ca rotation " +
+          "polling is disabled for {}", component);
+    }
+  }
+
+  private void startRootCaRotationPoller() {
+    if (rootCaRotationPoller == null) {
+      rootCaRotationPoller = new RootCaRotationPoller(securityConfig,
+          rootCaCertificates, scmSecurityClient);
+      rootCaRotationPoller.addRootCARotationProcessor(
+          this::getRootCaRotationListener);
+      rootCaRotationPoller.run();
+    } else {
+      getLogger().debug("Root CA certificate rotation poller is already " +
+          "started.");
     }
   }
 
@@ -985,6 +1001,10 @@ public synchronized void close() throws IOException {
       executorService = null;
     }
 
+    if (rootCaRotationPoller != null) {
+      rootCaRotationPoller.close();
+    }
+
     if (serverKeyStoresFactory != null) {
       serverKeyStoresFactory.destroy();
     }
@@ -1292,11 +1312,21 @@ public SCMSecurityProtocolClientSideTranslatorPB getScmSecureClient()
     return scmSecurityClient;
   }
 
-  protected boolean shouldStartCertificateMonitor() {
+  protected boolean shouldStartCertificateRenewerService() {
     return true;
   }
 
-  public synchronized void startCertificateMonitor() {
+  public synchronized CompletableFuture<Void> getRootCaRotationListener(
+      List<X509Certificate> rootCAs) {
+    if (rootCaCertificates.containsAll(rootCAs)) {
+      return CompletableFuture.completedFuture(null);
+    }
+    CertificateRenewerService renewerService =
+        new CertificateRenewerService(true);
+    return CompletableFuture.runAsync(renewerService, executorService);
+  }
+
+  public synchronized void startCertificateRenewerService() {
     Preconditions.checkNotNull(getCertificate(),
         "Component certificate should not be empty");
     // Schedule task to refresh certificate before it expires
@@ -1310,13 +1340,13 @@ public synchronized void startCertificateMonitor() {
     if (executorService == null) {
       executorService = Executors.newScheduledThreadPool(1,
           new ThreadFactoryBuilder().setNameFormat(
-                  getComponentName() + "-CertificateLifetimeMonitor")
+                  getComponentName() + "-CertificateRenewerService")
               .setDaemon(true).build());
     }
     this.executorService.scheduleAtFixedRate(
         new CertificateRenewerService(false),
         timeBeforeGracePeriod, interval, TimeUnit.MILLISECONDS);
-    getLogger().info("CertificateLifetimeMonitor for {} is started with " +
+    getLogger().info("CertificateRenewerService for {} is started with " +
             "first delay {} ms and interval {} ms.", component,
         timeBeforeGracePeriod, interval);
   }
@@ -1349,10 +1379,11 @@ public void run() {
         }
         String newCertId;
         try {
-          getLogger().info("Current certificate {} has entered the expiry" +
-                  " grace period {}. Starting renew key and certs.",
+          getLogger().info("Current certificate {} needs to be renewed " +
+                  "remaining grace period {}. Forced renewal due to root ca " +
+                  "rotation: {}.",
               currentCert.getSerialNumber().toString(),
-              timeLeft, securityConfig.getRenewalGracePeriod());
+              timeLeft, forceRenewal);
           newCertId = renewAndStoreKeyAndCertificate(forceRenewal);
         } catch (CertificateException e) {
           if (e.errorCode() ==

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java

@@ -173,7 +173,7 @@ public CertificateSignRequest.Builder getCSRBuilder()
   }
 
   @Override
-  protected boolean shouldStartCertificateMonitor() {
+  protected boolean shouldStartCertificateRenewerService() {
     return false;
   }
 

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClientTestImpl.java

@@ -166,7 +166,7 @@ public CertificateClientTestImpl(OzoneConfiguration conf, boolean autoRenew)
           Duration.between(currentTime, gracePeriodStart);
 
       executorService = Executors.newScheduledThreadPool(1,
-          new ThreadFactoryBuilder().setNameFormat("CertificateLifetimeMonitor")
+          new ThreadFactoryBuilder().setNameFormat("CertificateRenewerService")
               .setDaemon(true).build());
       this.executorService.schedule(new RenewCertTask(),
           delay.toMillis(), TimeUnit.MILLISECONDS);

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDefaultCertificateClient.java

@@ -656,7 +656,7 @@ protected String signAndStoreCertificate(
     Thread.enumerate(threads);
     Predicate<Thread> monitorFilterPredicate =
         t -> t != null
-            && t.getName().equals(compName + "-CertificateLifetimeMonitor");
+            && t.getName().equals(compName + "-CertificateRenewerService");
     long monitorThreadCount = Arrays.stream(threads)
         .filter(monitorFilterPredicate)
         .count();

hadoop-ozone/dist/src/main/compose/ozonesecure/root-ca-rotation.yaml

@@ -25,6 +25,7 @@ x-root-cert-rotation-config:
     - OZONE-SITE.XML_hdds.x509.renew.grace.duration=PT45S
     - OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval=PT1S
     - OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout=PT20S
+    - OZONE-SITE.XML_hdds.x509.rootca.certificate.polling.interval=PT10s
     - OZONE-SITE.XML_hdds.block.token.expiry.time=15s
     - OZONE-SITE.XML_ozone.manager.delegation.token.max-lifetime=15s
     - OZONE-SITE.XML_ozone.manager.delegation.token.renew-interval=15s

hadoop-ozone/dist/src/main/compose/ozonesecure/test-root-ca-rotation.sh

@@ -40,6 +40,7 @@ wait_for_execute_command scm 30 "jps | grep StorageContainerManagerStarter |  se
 
 # wait and verify root CA is rotated
 wait_for_execute_command scm 180 "ozone admin cert info 2"
+wait_for_execute_command datanode 30 "find /data/metadata/dn/certs/ROOTCA-2.crt"
 
 # verify om operations and data operations
 execute_commands_in_container scm "ozone sh volume create /r-v1 && ozone sh bucket create /r-v1/r-b1"