HDDS-7723. Refresh Keys and Certificate used in OzoneSecretManager after certificate renewed

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java

@@ -32,6 +32,7 @@
 import org.apache.hadoop.security.token.Token;
 
 import java.io.IOException;
+import java.time.Duration;
 import java.util.EnumSet;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -69,19 +70,28 @@ class TokenHelper {
           HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME,
           HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT,
           TimeUnit.MILLISECONDS);
-      String certId = certClient.getCertificate().getSerialNumber().toString();
+      long certificateGracePeriod = Duration.parse(
+          conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION,
+              HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT))
+          .toMillis();
+      if (expiryTime > certificateGracePeriod) {
+        throw new IllegalArgumentException("Certificate grace period " +
+            HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION +
+            " should be greater than maximum block/container token lifetime " +
+            HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME);
+      }
 
       if (blockTokenEnabled) {
         blockTokenMgr = new OzoneBlockTokenSecretManager(
-            securityConfig, expiryTime, certId);
+            securityConfig, expiryTime);
         blockTokenMgr.start(certClient);
       } else {
         blockTokenMgr = null;
       }
 
       if (containerTokenEnabled) {
         containerTokenMgr = new ContainerTokenSecretManager(
-            securityConfig, expiryTime, certId);
+            securityConfig, expiryTime);
         containerTokenMgr.start(certClient);
       } else {
         containerTokenMgr = null;

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretKey.java

@@ -49,13 +49,16 @@ public class OzoneSecretKey implements Writable {
   private PrivateKey privateKey;
   private PublicKey publicKey;
   private SecurityConfig securityConfig;
+  private String certSerialId;
 
-  public OzoneSecretKey(int keyId, long expiryDate, KeyPair keyPair) {
+  public OzoneSecretKey(int keyId, long expiryDate, KeyPair keyPair,
+      String certificateSerialId) {
     Preconditions.checkNotNull(keyId);
     this.keyId = keyId;
     this.expiryDate = expiryDate;
     this.privateKey = keyPair.getPrivate();
     this.publicKey = keyPair.getPublic();
+    this.certSerialId = certificateSerialId;
   }
 
   /*
@@ -89,6 +92,10 @@ public PublicKey getPublicKey() {
     return publicKey;
   }
 
+  public String getCertSerialId() {
+    return certSerialId;
+  }
+
   public byte[] getEncodedPrivateKey() {
     return privateKey.getEncoded();
   }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java

@@ -22,6 +22,7 @@
 import org.apache.hadoop.hdds.annotation.InterfaceStability;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
 import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateNotification;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.token.SecretManager;
@@ -36,7 +37,9 @@
 import java.security.PrivateKey;
 import java.security.Signature;
 import java.security.SignatureException;
+import java.security.cert.X509Certificate;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * SecretManager for Ozone Master. Responsible for signing identifiers with
@@ -45,7 +48,7 @@
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public abstract class OzoneSecretManager<T extends TokenIdentifier>
-    extends SecretManager<T> {
+    extends SecretManager<T> implements CertificateNotification {
 
   private final Logger logger;
   /**
@@ -57,7 +60,7 @@ public abstract class OzoneSecretManager<T extends TokenIdentifier>
   private final Text service;
   private CertificateClient certClient;
   private volatile boolean running;
-  private OzoneSecretKey currentKey;
+  private AtomicReference<OzoneSecretKey> currentKey;
   private AtomicInteger currentKeyId;
   private AtomicInteger tokenSequenceNumber;
 
@@ -81,6 +84,7 @@ public OzoneSecretManager(SecurityConfig secureConf, long tokenMaxLifetime,
     tokenSequenceNumber = new AtomicInteger();
     this.service = service;
     this.logger = logger;
+    this.currentKey = new AtomicReference<>();
   }
 
 
@@ -112,12 +116,12 @@ public byte[] createPassword(byte[] identifier, PrivateKey privateKey)
   public byte[] createPassword(T identifier) {
     if (logger.isDebugEnabled()) {
       logger.debug("Creating password for identifier: {}, currentKey: {}",
-          formatTokenId(identifier), currentKey.getKeyId());
+          formatTokenId(identifier), currentKey.get().getKeyId());
     }
     byte[] password = null;
     try {
       password = createPassword(identifier.getBytes(),
-          currentKey.getPrivateKey());
+          currentKey.get().getPrivateKey());
     } catch (IOException ioe) {
       logger.error("Could not store token {}!!", formatTokenId(identifier),
           ioe);
@@ -166,16 +170,33 @@ public int incrementDelegationTokenSeqNum() {
    * Update the current master key. This is called once by start method before
    * tokenRemoverThread is created,
    */
-  private OzoneSecretKey updateCurrentKey(KeyPair keyPair) throws IOException {
-    logger.info("Updating the current master key for generating tokens");
+  private OzoneSecretKey updateCurrentKey(KeyPair keyPair,
+      X509Certificate certificate) {
+    logger.info("Updating current master key for generating tokens. Cert id {}",
+        certificate.getSerialNumber().toString());
 
-    // TODO: fix me based on the certificate expire time to set the key
-    // expire time.
     int newCurrentId = incrementCurrentKeyId();
-    OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, -1,
-        keyPair);
-    currentKey = newKey;
-    return currentKey;
+    OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId,
+        certificate.getNotAfter().getTime(), keyPair,
+        certificate.getSerialNumber().toString());
+    currentKey.set(newKey);
+    return newKey;
+  }
+
+  public void notifyCertificateRenewed(String oldCertId, String newCertId) {
+    if (!oldCertId.equals(getCertSerialId())) {
+      logger.info("Old certificate Id doesn't match. Holding {}, oldCertId {}",
+          getCertSerialId(), oldCertId);
+    }
+    if (!newCertId.equals(
+        certClient.getCertificate().getSerialNumber().toString())) {
+      logger.info("New certificate Id doesn't match. Holding in caClient {}," +
+          " newCertId {}", newCertId,
+          certClient.getCertificate().getSerialNumber().toString());
+    }
+    logger.info("Certificate is changed from {} to {}", oldCertId, newCertId);
+    updateCurrentKey(new KeyPair(certClient.getPublicKey(),
+        certClient.getPrivateKey()), certClient.getCertificate());
   }
 
   public String formatTokenId(T id) {
@@ -193,7 +214,8 @@ public synchronized void start(CertificateClient client)
     Preconditions.checkState(!isRunning());
     setCertClient(client);
     updateCurrentKey(new KeyPair(certClient.getPublicKey(),
-        certClient.getPrivateKey()));
+        certClient.getPrivateKey()), certClient.getCertificate());
+    client.registerNotificationReceiver(this);
     setIsRunning(true);
   }
 
@@ -236,13 +258,17 @@ public void setIsRunning(boolean val) {
   }
 
   public OzoneSecretKey getCurrentKey() {
-    return currentKey;
+    return currentKey.get();
   }
 
   public AtomicInteger getCurrentKeyId() {
     return currentKeyId;
   }
 
+  public String getCertSerialId() {
+    return currentKey.get().getCertSerialId();
+  }
+
   public AtomicInteger getTokenSequenceNumber() {
     return tokenSequenceNumber;
   }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java

@@ -141,7 +141,7 @@ private void createKeyManagers(Mode mode) throws
   public synchronized void init(Mode mode, boolean requireClientAuth)
       throws IOException, GeneralSecurityException {
 
-    monitoringTimer = new Timer(caClient.getComponentName() + "-"
+    monitoringTimer = new Timer(caClient.getComponentName() + "-" + mode + "-"
         + SSL_MONITORING_THREAD_NAME, true);
 
     // key manager

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java

@@ -41,9 +41,8 @@ public class ContainerTokenSecretManager
   private static final Logger LOG =
       LoggerFactory.getLogger(ContainerTokenSecretManager.class);
 
-  public ContainerTokenSecretManager(SecurityConfig conf,
-      long tokenLifetime, String certSerialId) {
-    super(conf, tokenLifetime, certSerialId, LOG);
+  public ContainerTokenSecretManager(SecurityConfig conf, long tokenLifetime) {
+    super(conf, tokenLifetime, LOG);
   }
 
   public ContainerTokenIdentifier createIdentifier(String user,

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java

@@ -44,9 +44,8 @@ public class OzoneBlockTokenSecretManager extends
   private static final Logger LOG = LoggerFactory
       .getLogger(OzoneBlockTokenSecretManager.class);
 
-  public OzoneBlockTokenSecretManager(SecurityConfig conf,
-      long tokenLifetime, String omCertSerialId) {
-    super(conf, tokenLifetime, omCertSerialId, LOG);
+  public OzoneBlockTokenSecretManager(SecurityConfig conf, long tokenLifetime) {
+    super(conf, tokenLifetime, LOG);
   }
 
   @Override

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java

@@ -39,12 +39,9 @@
 
   private static final Text SERVICE = new Text("HDDS_SERVICE");
 
-  private final String certSerialId;
-
   protected ShortLivedTokenSecretManager(SecurityConfig conf,
-      long tokenLifetime, String certSerialId, Logger logger) {
+      long tokenLifetime, Logger logger) {
     super(conf, tokenLifetime, tokenLifetime, SERVICE, logger);
-    this.certSerialId = certSerialId;
   }
 
   @Override
@@ -95,10 +92,6 @@ protected Instant getTokenExpiryTime() {
     return Instant.now().plusMillis(getTokenMaxLifetime());
   }
 
-  protected String getCertSerialId() {
-    return certSerialId;
-  }
-
   public Token<T> generateToken(T tokenIdentifier) {
     return new Token<>(tokenIdentifier.getBytes(),
         createPassword(tokenIdentifier), tokenIdentifier.getKind(),

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java

@@ -362,4 +362,10 @@ default void assertValidKeysAndCertificate() throws OzoneSecurityException {
    * Return the store factory for key manager and trust manager for client.
    */
   KeyStoresFactory getClientKeyStoresFactory() throws CertificateException;
+
+  /**
+   * Register a receiver that will be called after the certificate renewed.
+   * @param receiver
+   */
+  void registerNotificationReceiver(CertificateNotification receiver);
 }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java

@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.client;
+
+/**
+ * Class should implement this interface if it wants to be notified when there
+ * is some changes in Certificate.
+ */
+public interface CertificateNotification {
+  /**
+   * Notify the class implementing this interface that certificate is renewed.
+   * Note that the new leader can possibly be this server.
+   *
+   * @param oldCertId The old cert id before renew.
+   * @param newCertId The new cert id after renew.
+   */
+  void notifyCertificateRenewed(String oldCertId, String newCertId);
+}

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java

@@ -46,10 +46,12 @@
 import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Random;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -142,6 +144,7 @@ public abstract class DefaultCertificateClient implements CertificateClient {
   private Consumer<String> certIdSaveCallback;
   private Runnable shutdownCallback;
   private SCMSecurityProtocolClientSideTranslatorPB scmSecurityProtocolClient;
+  private Set<CertificateNotification> notificationReceivers;
 
   DefaultCertificateClient(SecurityConfig securityConfig, Logger log,
       String certSerialId, String component,
@@ -155,6 +158,7 @@ public abstract class DefaultCertificateClient implements CertificateClient {
     this.component = component;
     this.certIdSaveCallback = saveCertId;
     this.shutdownCallback = shutdown;
+    this.notificationReceivers = new HashSet<>();
 
     loadAllCertificates();
   }
@@ -1098,6 +1102,17 @@ public KeyStoresFactory getClientKeyStoresFactory()
     return clientKeyStoresFactory;
   }
 
+  /**
+   * Register a receiver that will be called after the certificate renewed.
+   * @param receiver
+   */
+  @Override
+  public void registerNotificationReceiver(CertificateNotification receiver) {
+    synchronized (notificationReceivers) {
+      notificationReceivers.add(receiver);
+    }
+  }
+
   @Override
   public synchronized void close() throws IOException {
     if (executorService != null) {
@@ -1390,8 +1405,9 @@ public synchronized void startCertificateMonitor() {
     }
     this.executorService.scheduleAtFixedRate(new CertificateLifetimeMonitor(),
         timeBeforeGracePeriod, interval, TimeUnit.MILLISECONDS);
-    getLogger().info("CertificateLifetimeMonitor is started with first delay" +
-        " {} ms and interval {} ms.", timeBeforeGracePeriod, interval);
+    getLogger().info("CertificateLifetimeMonitor for {} is started with " +
+            "first delay {} ms and interval {} ms.", component,
+        timeBeforeGracePeriod, interval);
   }
 
   /**
@@ -1403,7 +1419,8 @@ public void run() {
 
       renewLock.lock();
       try {
-        Duration timeLeft = timeBeforeExpiryGracePeriod(getCertificate());
+        X509Certificate currentCert = getCertificate();
+        Duration timeLeft = timeBeforeExpiryGracePeriod(currentCert);
         if (timeLeft.isZero()) {
           String newCertId;
           try {
@@ -1434,6 +1451,9 @@ public void run() {
           reloadKeyAndCertificate(newCertId);
           // cleanup backup directory
           cleanBackupDir();
+          // notify notification receivers
+          notificationReceivers.forEach(r -> r.notifyCertificateRenewed(
+              currentCert.getSerialNumber().toString(), newCertId));
         }
       } finally {
         renewLock.unlock();

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java

@@ -105,7 +105,7 @@ public void setUp() throws Exception {
         .generateCertificate("CN=OzoneMaster", keyPair, 30, ALGORITHM);
     omCertSerialId = x509Certificate.getSerialNumber().toString();
     secretManager = new OzoneBlockTokenSecretManager(securityConfig,
-        TimeUnit.HOURS.toMillis(1), omCertSerialId);
+        TimeUnit.HOURS.toMillis(1));
     client = Mockito.mock(DefaultCertificateClient.class);
     when(client.getCertificate()).thenReturn(x509Certificate);
     when(client.getCertificate(anyString())).