HDDS-8592. Prepare DefaultCertificateClient for Root CA Rotation

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java

@@ -222,6 +222,10 @@ public final class HddsConfigKeys {
       "hdds.x509.ca.rotation.ack.timeout";
   public static final String HDDS_X509_CA_ROTATION_ACK_TIMEOUT_DEFAULT =
       "PT15M";
+  public static final String HDDS_X509_ROOTCA_CERTIFICATE_POLLING_INTERVAL =
+      "hdds.x509.rootca.certificate.polling.interval";
+  public static final String
+      HDDS_X509_ROOTCA_CERTIFICATE_POLLING_INTERVAL_DEFAULT = "PT2h";
 
   public static final String HDDS_CONTAINER_REPLICATION_COMPRESSION =
       "hdds.container.replication.compression";

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/SecurityConfig.java

@@ -52,6 +52,8 @@
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_TIME_OF_DAY_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_FILE;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_FILE_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_POLLING_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_POLLING_INTERVAL_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_PRIVATE_KEY_FILE;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_PRIVATE_KEY_FILE_DEFAULT;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_PUBLIC_KEY_FILE;
@@ -131,6 +133,7 @@ public class SecurityConfig {
       Pattern.compile("\\d{2}:\\d{2}:\\d{2}");
   private final Duration caAckTimeout;
   private final SslProvider grpcSSLProvider;
+  private final Duration rootCaCertificatePollingInterval;
 
   /**
    * Constructs a SecurityConfig.
@@ -228,6 +231,13 @@ public SecurityConfig(ConfigurationSource configuration) {
 
     validateCertificateValidityConfig();
 
+    String rootCaCertificatePollingIntervalString = configuration.get(
+        HDDS_X509_ROOTCA_CERTIFICATE_POLLING_INTERVAL,
+        HDDS_X509_ROOTCA_CERTIFICATE_POLLING_INTERVAL_DEFAULT);
+
+    this.rootCaCertificatePollingInterval =
+        Duration.parse(rootCaCertificatePollingIntervalString);
+
     this.externalRootCaCert = configuration.get(
         HDDS_X509_ROOTCA_CERTIFICATE_FILE,
         HDDS_X509_ROOTCA_CERTIFICATE_FILE_DEFAULT);
@@ -552,6 +562,10 @@ public Duration getCaAckTimeout() {
     return caAckTimeout;
   }
 
+  public Duration getRootCaCertificatePollingInterval() {
+    return rootCaCertificatePollingInterval;
+  }
+
   /**
    * Return true if using test certificates with authority as localhost. This
    * should be used only for unit test where certificates are generated by

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -2268,6 +2268,28 @@
       is failed. Default is 15 minutes.
     </description>
   </property>
+  <property>
+    <name>hdds.x509.rootca.certificate.polling.interval</name>
+    <value>PT2h</value>
+    <description>Interval to use for polling in certificate clients for a new
+      root ca certificate. Every time the specified time duration elapses,
+      the clients send a request to the SCMs to see if a new root ca
+      certificate was generated. Once there is a change, the system
+      automatically adds the new root ca to the clients'
+      trust stores and requests a new certificate to be signed.
+    </description>
+  </property>
+  <property>
+    <name>hdds.x509.rootca.client.polling.frequency</name>
+    <value>PT2h</value>
+    <description>Frequency to use for polling in certificate clients for a new
+      root ca certificate. Every time the specified time duration elapses,
+      the clients send a request to the SCMs to see if a new root ca
+      certificate was generated. Once there is a change, the system
+      automatically adds the new root ca to the clients'
+      trust stores and requests a new certificate to be signed.
+    </description>
+  </property>
   <property>
     <name>ozone.scm.security.handler.count.key</name>
     <value>2</value>

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java

@@ -25,6 +25,8 @@
 import java.security.cert.CertificateExpiredException;
 import java.time.Duration;
 import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.Callable;
 
 import org.apache.hadoop.fs.FileUtil;
@@ -316,6 +318,10 @@ public void testCertificateRotation() throws Exception {
     when(scmClient.getDataNodeCertificateChain(anyObject(), anyString()))
         .thenReturn(responseProto);
 
+    List<String> rootCaList = new ArrayList<>();
+    rootCaList.add(pemCert);
+    when(scmClient.getAllRootCaCertificates()).thenReturn(rootCaList);
+
     // check that new cert ID should not equal to current cert ID
     String certId = newCertHolder.getSerialNumber().toString();
     Assert.assertFalse(certId.equals(
@@ -338,6 +344,7 @@ public void testCertificateRotation() throws Exception {
     // test the second time certificate rotation, generate a new cert
     newCertHolder = generateX509CertHolder(null, null,
         Duration.ofSeconds(CERT_LIFETIME));
+    rootCaList.remove(pemCert);
     pemCert = CertificateCodec.getPEMEncodedString(newCertHolder);
     responseProto = SCMSecurityProtocolProtos.SCMGetCertResponseProto
         .newBuilder().setResponseCode(SCMSecurityProtocolProtos
@@ -348,6 +355,8 @@ public void testCertificateRotation() throws Exception {
         .build();
     when(scmClient.getDataNodeCertificateChain(anyObject(), anyString()))
         .thenReturn(responseProto);
+    rootCaList.add(pemCert);
+    when(scmClient.getAllRootCaCertificates()).thenReturn(rootCaList);
     String certId2 = newCertHolder.getSerialNumber().toString();
 
     // check after renew, client will have the new cert ID

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DNCertificateClient.java

@@ -20,11 +20,9 @@
 package org.apache.hadoop.hdds.security.x509.certificate.client;
 
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
 import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.security.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType;
-import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
 import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateSignRequest;
 import org.apache.hadoop.hdds.security.x509.exception.CertificateException;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -34,7 +32,6 @@
 
 import java.io.IOException;
 import java.net.InetAddress;
-import java.nio.file.Path;
 import java.security.KeyPair;
 import java.util.function.Consumer;
 
@@ -98,43 +95,10 @@ public CertificateSignRequest.Builder getCSRBuilder()
   }
 
   @Override
-  public String signAndStoreCertificate(PKCS10CertificationRequest csr,
-      Path certificatePath, boolean renew) throws CertificateException {
-    try {
-      // TODO: For SCM CA we should fetch certificate from multiple SCMs.
-      SCMSecurityProtocolProtos.SCMGetCertResponseProto response =
-          getScmSecureClient().getDataNodeCertificateChain(
-              dn.getProtoBufMessage(), getEncodedString(csr));
-
-      // Persist certificates.
-      if (response.hasX509CACertificate()) {
-        String pemEncodedCert = response.getX509Certificate();
-        CertificateCodec certCodec = new CertificateCodec(
-            getSecurityConfig(), certificatePath);
-        // Certs will be added to cert map after reloadAllCertificate called
-        storeCertificate(pemEncodedCert, CAType.NONE,
-            certCodec, false, !renew);
-        storeCertificate(response.getX509CACertificate(),
-            CAType.SUBORDINATE, certCodec, false, !renew);
-
-        // Store Root CA certificate.
-        if (response.hasX509RootCACertificate()) {
-          storeCertificate(response.getX509RootCACertificate(),
-              CAType.ROOT, certCodec, false, !renew);
-        }
-        // Return the default certificate ID
-        return CertificateCodec.getX509Certificate(pemEncodedCert)
-            .getSerialNumber()
-            .toString();
-      } else {
-        throw new CertificateException("Unable to retrieve datanode " +
-            "certificate chain.");
-      }
-    } catch (IOException | java.security.cert.CertificateException e) {
-      LOG.error("Error while signing and storing SCM signed certificate.", e);
-      throw new CertificateException(
-          "Error while signing and storing SCM signed certificate.", e);
-    }
+  public SCMGetCertResponseProto getCertificateSignResponse(
+      PKCS10CertificationRequest csr) throws IOException {
+    return getScmSecureClient().getDataNodeCertificateChain(
+        dn.getProtoBufMessage(), getEncodedString(csr));
   }
 
   @Override