HDDS-8592. Fetch and save all root certificates during service's certificate rotation.

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java

@@ -25,6 +25,8 @@
 import java.security.cert.CertificateExpiredException;
 import java.time.Duration;
 import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.Callable;
 
 import org.apache.hadoop.fs.FileUtil;
@@ -316,6 +318,9 @@ public void testCertificateRotation() throws Exception {
     when(scmClient.getDataNodeCertificateChain(anyObject(), anyString()))
         .thenReturn(responseProto);
 
+    List<String> rootCaList = new ArrayList<>();
+    rootCaList.add(pemCert);
+    when(scmClient.getAllRootCaCertificates()).thenReturn(rootCaList);
     // check that new cert ID should not equal to current cert ID
     String certId = newCertHolder.getSerialNumber().toString();
     Assert.assertFalse(certId.equals(
@@ -338,6 +343,7 @@ public void testCertificateRotation() throws Exception {
     // test the second time certificate rotation, generate a new cert
     newCertHolder = generateX509CertHolder(null, null,
         Duration.ofSeconds(CERT_LIFETIME));
+    rootCaList.remove(pemCert);
     pemCert = CertificateCodec.getPEMEncodedString(newCertHolder);
     responseProto = SCMSecurityProtocolProtos.SCMGetCertResponseProto
         .newBuilder().setResponseCode(SCMSecurityProtocolProtos
@@ -348,6 +354,8 @@ public void testCertificateRotation() throws Exception {
         .build();
     when(scmClient.getDataNodeCertificateChain(anyObject(), anyString()))
         .thenReturn(responseProto);
+    rootCaList.add(pemCert);
+    when(scmClient.getAllRootCaCertificates()).thenReturn(rootCaList);
     String certId2 = newCertHolder.getSerialNumber().toString();
 
     // check after renew, client will have the new cert ID

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DNCertificateClient.java

@@ -20,11 +20,9 @@
 package org.apache.hadoop.hdds.security.x509.certificate.client;
 
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
 import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.security.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType;
-import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
 import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateSignRequest;
 import org.apache.hadoop.hdds.security.x509.exception.CertificateException;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -34,7 +32,6 @@
 
 import java.io.IOException;
 import java.net.InetAddress;
-import java.nio.file.Path;
 import java.security.KeyPair;
 import java.util.function.Consumer;
 
@@ -98,43 +95,10 @@ public CertificateSignRequest.Builder getCSRBuilder()
   }
 
   @Override
-  public String signAndStoreCertificate(PKCS10CertificationRequest csr,
-      Path certificatePath, boolean renew) throws CertificateException {
-    try {
-      // TODO: For SCM CA we should fetch certificate from multiple SCMs.
-      SCMSecurityProtocolProtos.SCMGetCertResponseProto response =
-          getScmSecureClient().getDataNodeCertificateChain(
-              dn.getProtoBufMessage(), getEncodedString(csr));
-
-      // Persist certificates.
-      if (response.hasX509CACertificate()) {
-        String pemEncodedCert = response.getX509Certificate();
-        CertificateCodec certCodec = new CertificateCodec(
-            getSecurityConfig(), certificatePath);
-        // Certs will be added to cert map after reloadAllCertificate called
-        storeCertificate(pemEncodedCert, CAType.NONE,
-            certCodec, false, !renew);
-        storeCertificate(response.getX509CACertificate(),
-            CAType.SUBORDINATE, certCodec, false, !renew);
-
-        // Store Root CA certificate.
-        if (response.hasX509RootCACertificate()) {
-          storeCertificate(response.getX509RootCACertificate(),
-              CAType.ROOT, certCodec, false, !renew);
-        }
-        // Return the default certificate ID
-        return CertificateCodec.getX509Certificate(pemEncodedCert)
-            .getSerialNumber()
-            .toString();
-      } else {
-        throw new CertificateException("Unable to retrieve datanode " +
-            "certificate chain.");
-      }
-    } catch (IOException | java.security.cert.CertificateException e) {
-      LOG.error("Error while signing and storing SCM signed certificate.", e);
-      throw new CertificateException(
-          "Error while signing and storing SCM signed certificate.", e);
-    }
+  public SCMGetCertResponseProto getCertificateSignResponse(
+      PKCS10CertificationRequest csr) throws IOException {
+    return getScmSecureClient().getDataNodeCertificateChain(
+        dn.getProtoBufMessage(), getEncodedString(csr));
   }
 
   @Override

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java

@@ -21,6 +21,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -62,6 +63,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
 import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.security.SecurityConfig;
 import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory;
@@ -228,15 +230,17 @@ private void updateCachedData(
   }
 
   private synchronized void updateCachedRootCAId(String s) {
+    BigInteger candidateNewId = new BigInteger(s);
     if (rootCaCertId == null
-        || Long.parseLong(s) > Long.parseLong(rootCaCertId)) {
+        || new BigInteger(rootCaCertId).compareTo(candidateNewId) < 0) {
       rootCaCertId = s;
     }
   }
 
   private synchronized void updateCachedSubCAId(String s) {
+    BigInteger candidateNewId = new BigInteger(s);
     if (caCertId == null
-        || Long.parseLong(s) > Long.parseLong(caCertId)) {
+        || new BigInteger(caCertId).compareTo(candidateNewId) < 0) {
       caCertId = s;
     }
   }
@@ -1232,9 +1236,50 @@ protected String signAndStoreCertificate(
     return signAndStoreCertificate(request, certificatePath, false);
   }
 
-  protected abstract String signAndStoreCertificate(
+  protected abstract SCMGetCertResponseProto getCertificateSignResponse(
+      PKCS10CertificationRequest request) throws IOException;
+
+  protected String signAndStoreCertificate(
       PKCS10CertificationRequest request, Path certificatePath, boolean renew)
-      throws CertificateException;
+      throws CertificateException {
+    try {
+      SCMGetCertResponseProto response = getCertificateSignResponse(request);
+
+      // Persist certificates.
+      if (response.hasX509CACertificate()) {
+        String pemEncodedCert = response.getX509Certificate();
+        CertificateCodec certCodec = new CertificateCodec(
+            getSecurityConfig(), certificatePath);
+        // Certs will be added to cert map after reloadAllCertificate called
+        storeCertificate(pemEncodedCert, CAType.NONE,
+            certCodec, false, !renew);
+        storeCertificate(response.getX509CACertificate(),
+            CAType.SUBORDINATE, certCodec, false, !renew);
+
+        getAndStoreAllRootCAs(certCodec, renew);
+        // Return the default certificate ID
+        return updateCertSerialId(CertificateCodec
+            .getX509Certificate(pemEncodedCert).getSerialNumber().toString());
+      } else {
+        throw new CertificateException("Unable to retrieve " +
+            "certificate chain.");
+      }
+    } catch (IOException | java.security.cert.CertificateException e) {
+      logger.error("Error while signing and storing SCM signed certificate.",
+          e);
+      throw new CertificateException(
+          "Error while signing and storing SCM signed certificate.", e);
+    }
+  }
+
+  private void getAndStoreAllRootCAs(CertificateCodec certCodec, boolean renew)
+      throws IOException {
+    List<String> rootCAPems = scmSecurityClient.getAllRootCaCertificates();
+    for (String rootCAPem : rootCAPems) {
+      storeCertificate(rootCAPem, CAType.ROOT, certCodec,
+          false, !renew);
+    }
+  }
 
   public String signAndStoreCertificate(
       PKCS10CertificationRequest request) throws CertificateException {
@@ -1265,22 +1310,25 @@ public synchronized void startCertificateMonitor() {
     if (executorService == null) {
       executorService = Executors.newScheduledThreadPool(1,
           new ThreadFactoryBuilder().setNameFormat(
-              getComponentName() + "-CertificateLifetimeMonitor")
+                  getComponentName() + "-CertificateLifetimeMonitor")
               .setDaemon(true).build());
     }
-    this.executorService.scheduleAtFixedRate(new CertificateLifetimeMonitor(),
+    this.executorService.scheduleAtFixedRate(
+        new CertificateRenewerService(false),
         timeBeforeGracePeriod, interval, TimeUnit.MILLISECONDS);
     getLogger().info("CertificateLifetimeMonitor for {} is started with " +
             "first delay {} ms and interval {} ms.", component,
         timeBeforeGracePeriod, interval);
   }
 
   /**
-   *  Task to monitor certificate lifetime and renew the certificate if needed.
+   * Task to monitor certificate lifetime and renew the certificate if needed.
    */
-  public class CertificateLifetimeMonitor implements Runnable {
+  public class CertificateRenewerService implements Runnable {
+    private boolean forceRenewal;
 
-    public CertificateLifetimeMonitor() {
+    public CertificateRenewerService(boolean forceRenewal) {
+      this.forceRenewal = forceRenewal;
     }
 
     @Override
@@ -1295,38 +1343,40 @@ public void run() {
       synchronized (DefaultCertificateClient.class) {
         X509Certificate currentCert = getCertificate();
         Duration timeLeft = timeBeforeExpiryGracePeriod(currentCert);
-        if (timeLeft.isZero()) {
-          String newCertId;
-          try {
-            getLogger().info("Current certificate {} has entered the expiry" +
-                    " grace period {}. Starting renew key and certs.",
-                currentCert.getSerialNumber().toString(),
-                timeLeft, securityConfig.getRenewalGracePeriod());
-            newCertId = renewAndStoreKeyAndCertificate(false);
-          } catch (CertificateException e) {
-            if (e.errorCode() ==
-                CertificateException.ErrorCode.ROLLBACK_ERROR) {
-              if (shutdownCallback != null) {
-                getLogger().error("Failed to rollback key and cert after an " +
-                    " unsuccessful renew try.", e);
-                shutdownCallback.run();
-              }
-            }
-            getLogger().error("Failed to renew and store key and cert." +
-                " Keep using existing certificates.", e);
-            return;
-          }
 
-          // Persist new cert serial id in component VERSION file
-          if (certIdSaveCallback != null) {
-            certIdSaveCallback.accept(newCertId);
+        if (!forceRenewal && !timeLeft.isZero()) {
+          return;
+        }
+        String newCertId;
+        try {
+          getLogger().info("Current certificate {} has entered the expiry" +
+                  " grace period {}. Starting renew key and certs.",
+              currentCert.getSerialNumber().toString(),
+              timeLeft, securityConfig.getRenewalGracePeriod());
+          newCertId = renewAndStoreKeyAndCertificate(forceRenewal);
+        } catch (CertificateException e) {
+          if (e.errorCode() ==
+              CertificateException.ErrorCode.ROLLBACK_ERROR) {
+            if (shutdownCallback != null) {
+              getLogger().error("Failed to rollback key and cert after an " +
+                  " unsuccessful renew try.", e);
+              shutdownCallback.run();
+            }
           }
+          getLogger().error("Failed to renew and store key and cert." +
+              " Keep using existing certificates.", e);
+          return;
+        }
 
-          // reset and reload all certs
-          reloadKeyAndCertificate(newCertId);
-          // cleanup backup directory
-          cleanBackupDir();
+        // Persist new cert serial id in component VERSION file
+        if (certIdSaveCallback != null) {
+          certIdSaveCallback.accept(newCertId);
         }
+
+        // reset and reload all certs
+        reloadKeyAndCertificate(newCertId);
+        // cleanup backup directory
+        cleanBackupDir();
       }
     }
   }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java

@@ -18,10 +18,10 @@
 
 package org.apache.hadoop.hdds.security.x509.certificate.client;
 
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
 import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.security.SecurityConfig;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
 import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType;
 import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
 import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateSignRequest;
@@ -182,6 +182,13 @@ public Logger getLogger() {
     return LOG;
   }
 
+  @Override
+  protected SCMGetCertResponseProto getCertificateSignResponse(
+      PKCS10CertificationRequest request) {
+    throw new UnsupportedOperationException("getCertSignResponse of " +
+        " SCMCertificateClient is not supported currently");
+  }
+
   @Override
   public String signAndStoreCertificate(PKCS10CertificationRequest request,
       Path certPath, boolean renew) throws CertificateException {
@@ -193,7 +200,7 @@ public String signAndStoreCertificate(PKCS10CertificationRequest request,
               .setScmNodeId(scmId).build();
 
       // Get SCM sub CA cert.
-      SCMSecurityProtocolProtos.SCMGetCertResponseProto response =
+      SCMGetCertResponseProto response =
           getScmSecureClient().getSCMCertChain(scmNodeDetailsProto,
               getEncodedString(request), true);
 

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDefaultCertificateClient.java

@@ -20,7 +20,7 @@
 
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
 import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType;
 import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse;
@@ -489,10 +489,16 @@ public String signAndStoreCertificate(
             return null;
           }
 
+          @Override
+          protected SCMGetCertResponseProto getCertificateSignResponse(
+              PKCS10CertificationRequest request) {
+            return null;
+          }
+
           @Override
           public String signAndStoreCertificate(
               PKCS10CertificationRequest request, Path certificatePath,
-              boolean renew) throws CertificateException {
+              boolean renew) {
             return null;
           }
         }) {
@@ -536,10 +542,11 @@ public void testRenewAndStoreKeyAndCertificate() throws Exception {
 
     X509Certificate newCert = generateX509Cert(null);
     String pemCert = CertificateCodec.getPEMEncodedString(newCert);
-    SCMSecurityProtocolProtos.SCMGetCertResponseProto responseProto =
-        SCMSecurityProtocolProtos.SCMGetCertResponseProto
-            .newBuilder().setResponseCode(SCMSecurityProtocolProtos
-                .SCMGetCertResponseProto.ResponseCode.success)
+    SCMGetCertResponseProto responseProto =
+        SCMGetCertResponseProto
+            .newBuilder().setResponseCode(
+                SCMGetCertResponseProto
+                    .ResponseCode.success)
             .setX509Certificate(pemCert)
             .setX509CACertificate(pemCert)
             .build();
@@ -631,10 +638,16 @@ protected String signAndStoreCertificate(
         return "";
       }
 
+      @Override
+      protected SCMGetCertResponseProto getCertificateSignResponse(
+          PKCS10CertificationRequest request) {
+        return null;
+      }
+
       @Override
       protected String signAndStoreCertificate(
           PKCS10CertificationRequest request, Path certificatePath,
-          boolean renew) throws CertificateException {
+          boolean renew) {
         return null;
       }
     };