HDDS-7339. Implement Certificate renewal task for services

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java

@@ -193,6 +193,8 @@ public final class HddsConfigKeys {
       "hdds.x509.renew.grace.duration";
 
   public static final String HDDS_X509_RENEW_GRACE_DURATION_DEFAULT = "P28D";
+  public static final String HDDS_NEW_KEY_CERT_DIR_NAME_SUFFIX = "-next";
+  public static final String HDDS_BACKUP_KEY_CERT_DIR_NAME_SUFFIX = "-previous";
 
   public static final String HDDS_CONTAINER_REPLICATION_COMPRESSION =
       "hdds.container.replication.compression";

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java

@@ -79,6 +79,11 @@ public CertificateCodec(SecurityConfig config, String component) {
     this.location = securityConfig.getCertificateLocation(component);
   }
 
+  public CertificateCodec(SecurityConfig config, Path certPath) {
+    this.securityConfig = config;
+    this.location = certPath;
+  }
+
   /**
    * Returns a X509 Certificate from the Certificate Holder.
    *

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java

@@ -21,8 +21,6 @@
 import java.io.File;
 import java.io.IOException;
 import java.net.InetAddress;
-import java.security.KeyPair;
-import java.security.cert.CertificateException;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -38,13 +36,10 @@
 import org.apache.hadoop.hdds.StringUtils;
 import org.apache.hadoop.hdds.cli.GenericCli;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStore;
 import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStoreImpl;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
-import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
 import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
 import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient;
@@ -72,14 +67,11 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
-import static org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec.getX509Certificate;
-import static org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY;
 import static org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY;
 import static org.apache.hadoop.ozone.common.Storage.StorageState.INITIALIZED;
 import static org.apache.hadoop.util.ExitUtil.terminate;
 
-import org.bouncycastle.pkcs.PKCS10CertificationRequest;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
@@ -98,6 +90,7 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
       HddsDatanodeService.class);
 
   private OzoneConfiguration conf;
+  private SecurityConfig secConf;
   private DatanodeDetails datanodeDetails;
   private DatanodeStateMachine datanodeStateMachine;
   private List<ServicePlugin> plugins;
@@ -237,8 +230,10 @@ public void start() {
       if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
         component = "dn-" + datanodeDetails.getUuidString();
 
-        dnCertClient = new DNCertificateClient(new SecurityConfig(conf),
-            datanodeDetails.getCertSerialId());
+        secConf = new SecurityConfig(conf);
+        dnCertClient = new DNCertificateClient(secConf, datanodeDetails,
+            datanodeDetails.getCertSerialId(), this::saveNewCertId,
+            this::terminateDatanode);
 
         if (SecurityUtil.getAuthenticationMethod(conf).equals(
             UserGroupInformation.AuthenticationMethod.KERBEROS)) {
@@ -273,7 +268,7 @@ public void start() {
       dnCRLStore = new DatanodeCRLStoreImpl(conf);
 
       if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
-        initializeCertificateClient(conf);
+        dnCertClient = initializeCertificateClient(dnCertClient);
       }
       datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf,
           dnCertClient, this::terminateDatanode, dnCRLStore);
@@ -333,23 +328,31 @@ private void startRatisForTest() throws IOException {
    * Initializes secure Datanode.
    * */
   @VisibleForTesting
-  public void initializeCertificateClient(OzoneConfiguration config)
-      throws IOException {
+  public CertificateClient initializeCertificateClient(
+      CertificateClient certClient) throws IOException {
     LOG.info("Initializing secure Datanode.");
 
-    CertificateClient.InitResponse response = dnCertClient.init();
+    CertificateClient.InitResponse response = certClient.init();
     if (response.equals(CertificateClient.InitResponse.REINIT)) {
       LOG.info("Re-initialize certificate client.");
-      dnCertClient = new DNCertificateClient(new SecurityConfig(conf));
-      response = dnCertClient.init();
+      certClient = new DNCertificateClient(secConf, datanodeDetails, null,
+          this::saveNewCertId, this::terminateDatanode);
+      response = certClient.init();
     }
     LOG.info("Init response: {}", response);
     switch (response) {
     case SUCCESS:
       LOG.info("Initialization successful, case:{}.", response);
       break;
     case GETCERT:
-      getSCMSignedCert(config);
+      CertificateSignRequest.Builder csrBuilder = certClient.getCSRBuilder();
+      String dnCertSerialId =
+          certClient.signAndStoreCertificate(csrBuilder.build());
+      // persist cert ID to VERSION file
+      datanodeDetails.setCertSerialId(dnCertSerialId);
+      persistDatanodeDetails(datanodeDetails);
+      // set new certificate ID
+      certClient.setCertificateId(dnCertSerialId);
       LOG.info("Successfully stored SCM signed certificate, case:{}.",
           response);
       break;
@@ -365,51 +368,8 @@ public void initializeCertificateClient(OzoneConfiguration config)
           response);
       throw new RuntimeException("DN security initialization failed.");
     }
-  }
 
-  /**
-   * Get SCM signed certificate and store it using certificate client.
-   * @param config
-   * */
-  private void getSCMSignedCert(OzoneConfiguration config) {
-    try {
-      PKCS10CertificationRequest csr = getCSR(config);
-      // TODO: For SCM CA we should fetch certificate from multiple SCMs.
-      SCMSecurityProtocolClientSideTranslatorPB secureScmClient =
-          HddsServerUtil.getScmSecurityClientWithMaxRetry(config);
-      SCMGetCertResponseProto response = secureScmClient.
-          getDataNodeCertificateChain(
-              datanodeDetails.getProtoBufMessage(),
-              getEncodedString(csr));
-      // Persist certificates.
-      if (response.hasX509CACertificate()) {
-        String pemEncodedCert = response.getX509Certificate();
-        dnCertClient.storeCertificate(pemEncodedCert, true);
-        dnCertClient.storeCertificate(response.getX509CACertificate(), true,
-            true);
-
-        // Store Root CA certificate.
-        if (response.hasX509RootCACertificate()) {
-          dnCertClient.storeRootCACertificate(
-              response.getX509RootCACertificate(), true);
-        }
-        String dnCertSerialId = getX509Certificate(pemEncodedCert).
-            getSerialNumber().toString();
-        datanodeDetails.setCertSerialId(dnCertSerialId);
-        persistDatanodeDetails(datanodeDetails);
-        // Rebuild dnCertClient with the new CSR result so that the default
-        // certSerialId and the x509Certificate can be updated.
-        dnCertClient = new DNCertificateClient(
-            new SecurityConfig(config), dnCertSerialId);
-
-      } else {
-        throw new RuntimeException("Unable to retrieve datanode certificate " +
-            "chain");
-      }
-    } catch (IOException | CertificateException e) {
-      LOG.error("Error while storing SCM signed certificate.", e);
-      throw new RuntimeException(e);
-    }
+    return certClient;
   }
 
   private void registerMXBean() {
@@ -427,30 +387,6 @@ private void unregisterMXBean() {
     }
   }
 
-  /**
-   * Creates CSR for DN.
-   * @param config
-   * */
-  @VisibleForTesting
-  public PKCS10CertificationRequest getCSR(ConfigurationSource config)
-      throws IOException {
-    CertificateSignRequest.Builder builder = dnCertClient.getCSRBuilder();
-    KeyPair keyPair = new KeyPair(dnCertClient.getPublicKey(),
-        dnCertClient.getPrivateKey());
-
-    String hostname = InetAddress.getLocalHost().getCanonicalHostName();
-    String subject = UserGroupInformation.getCurrentUser()
-        .getShortUserName() + "@" + hostname;
-
-    builder.setCA(false)
-        .setKey(keyPair)
-        .setConfiguration(config)
-        .setSubject(subject);
-
-    LOG.info("Creating csr for DN-> subject:{}", subject);
-    return builder.build();
-  }
-
   /**
    * Returns DatanodeDetails or null in case of Error.
    *
@@ -584,7 +520,9 @@ public void stop() {
       unregisterMXBean();
       // stop dn crl store
       try {
-        dnCRLStore.stop();
+        if (dnCRLStore != null) {
+          dnCRLStore.stop();
+        }
       } catch (Exception ex) {
         LOG.error("Datanode CRL store stop failed", ex);
       }
@@ -631,4 +569,18 @@ public void setCertificateClient(CertificateClient client) {
   public void printError(Throwable error) {
     LOG.error("Exception in HddsDatanodeService.", error);
   }
+
+  public void saveNewCertId(String newCertId) {
+    // save new certificate Id to VERSION file
+    datanodeDetails.setCertSerialId(newCertId);
+    try {
+      persistDatanodeDetails(datanodeDetails);
+    } catch (IOException ex) {
+      // New cert ID cannot be persisted into VERSION file.
+      String msg = "Failed to persist new cert ID " + newCertId +
+          "to VERSION file. Terminating datanode...";
+      LOG.error(msg, ex);
+      terminateDatanode();
+    }
+  }
 }