HADOOP-19639. SecretManager configuration at runtime

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java

@@ -47,6 +47,61 @@
 public abstract class SecretManager<T extends TokenIdentifier> {
 
   public static final Logger LOG = LoggerFactory.getLogger(SecretManager.class);
+
+  private static String selectedAlgorithm;
+  private static int selectedLength;
+
+  /**
+   * Key generator to use.
+   */
+  private static boolean keygenInitialized;
+  private final Object keyGenLock = new Object();
+  private volatile KeyGenerator keyGen;
+
+  /**
+   * A thread local store for the Macs.
+   */
+  private static boolean macInitialized;
+  private static final ThreadLocal<Mac> threadLocalMac =
+      ThreadLocal.withInitial(SecretManager::createMac);
+
+  private static boolean secretKeyInitialized;
+
+  static {
+    update(new Configuration());
+  }
+
+  private static final String UPDATE_LOG_TEMPLATE =
+      "{} was already initialized with older config, those will not be updated." +
+          "Hint: If you turn on debug log you can see when it is happening. Thread: {}";
+  /**
+   * Updates the selected cryptographic algorithm and key length using the provided
+   * Hadoop {@link Configuration}. This method reads the values for
+   * {@code HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY} and
+   * {@code HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_KEY}, or uses default values if not set.
+   *
+   * @param conf the configuration object containing cryptographic settings
+   */
+  public static synchronized void update(Configuration conf) {
+    if (keygenInitialized) {
+      LOG.warn(UPDATE_LOG_TEMPLATE, "KeyGenerator", Thread.currentThread());
+    }
+    if (macInitialized) {
+      LOG.warn(UPDATE_LOG_TEMPLATE, "Mac", Thread.currentThread());
+    }
+    if (secretKeyInitialized) {
+      LOG.warn(UPDATE_LOG_TEMPLATE, "SecretKey", Thread.currentThread());
+    }
+    selectedAlgorithm = conf.get(
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY,
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT);
+    LOG.debug("Selected hash algorithm: {}", selectedAlgorithm);
+    selectedLength = conf.getInt(
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_KEY,
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_DEFAULT);
+    LOG.debug("Selected hash key length: {}", selectedLength);
+  }
+
   /**
    * The token was invalid and the message explains why.
    */
@@ -115,61 +170,17 @@ public void checkAvailableForRead() throws StandbyException {
     // Default to being available for read.
   }
 
-  private static final String SELECTED_ALGORITHM;
-  private static final int SELECTED_LENGTH;
-
-  static {
-    Configuration conf = new Configuration();
-    String algorithm = conf.get(
-      CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY,
-      CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT);
-    LOG.debug("Selected hash algorithm: {}", algorithm);
-    SELECTED_ALGORITHM = algorithm;
-    int length = conf.getInt(
-      CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_KEY,
-      CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_DEFAULT);
-    LOG.debug("Selected hash key length:{}", length);
-    SELECTED_LENGTH = length;
-  }
-
-  /**
-   * A thread local store for the Macs.
-   */
-  private static final ThreadLocal<Mac> threadLocalMac =
-    new ThreadLocal<Mac>(){
-    @Override
-    protected Mac initialValue() {
-      try {
-        return Mac.getInstance(SELECTED_ALGORITHM);
-      } catch (NoSuchAlgorithmException nsa) {
-        throw new IllegalArgumentException("Can't find " + SELECTED_ALGORITHM, nsa);
-      }
-    }
-  };
-
-  /**
-   * Key generator to use.
-   */
-  private final KeyGenerator keyGen;
-  {
-    try {
-      keyGen = KeyGenerator.getInstance(SELECTED_ALGORITHM);
-      keyGen.init(SELECTED_LENGTH);
-    } catch (NoSuchAlgorithmException nsa) {
-      throw new IllegalArgumentException("Can't find " + SELECTED_ALGORITHM, nsa);
-    }
-  }
-
   /**
    * Generate a new random secret key.
    * @return the new key
    */
   protected SecretKey generateSecret() {
-    SecretKey key;
-    synchronized (keyGen) {
-      key = keyGen.generateKey();
+    synchronized (keyGenLock) {
+      if (keyGen == null) {
+        keyGen = createKeyGenerator();
+      }
+      return keyGen.generateKey();
     }
-    return key;
   }
 
   /**
@@ -197,6 +208,46 @@ public static byte[] createPassword(byte[] identifier,
    * @return the secret key
    */
   protected static SecretKey createSecretKey(byte[] key) {
-    return new SecretKeySpec(key, SELECTED_ALGORITHM);
+    LOG.debug("Creating secretKey with algorithm {} with thread {}",
+        selectedAlgorithm, Thread.currentThread());
+    secretKeyInitialized = true;
+    return new SecretKeySpec(key, selectedAlgorithm);
+  }
+
+  /**
+   * Creates a new {@link KeyGenerator} instance configured with the currently selected
+   * algorithm and key length.
+   *
+   * @return a new {@code KeyGenerator} instance
+   * @throws IllegalArgumentException if the specified algorithm is not available
+   */
+  private static synchronized KeyGenerator createKeyGenerator() {
+    LOG.debug("Creating key generator instance {} - {} bit with thread {}",
+        selectedAlgorithm, selectedLength, Thread.currentThread());
+    try {
+      KeyGenerator keyGen = KeyGenerator.getInstance(selectedAlgorithm);
+      keyGen.init(selectedLength);
+      keygenInitialized = true;
+      return keyGen;
+    } catch (NoSuchAlgorithmException nsa) {
+      throw new IllegalArgumentException("Can't find " + selectedAlgorithm, nsa);
+    }
+  }
+
+  /**
+   * Creates a new {@link Mac} instance using the currently selected algorithm.
+   *
+   * @return a new {@code Mac} instance
+   * @throws IllegalArgumentException if the specified algorithm is not available
+   */
+  private static synchronized Mac createMac() {
+    LOG.debug("Creating mac instance {} with thread {}", selectedAlgorithm, Thread.currentThread());
+    try {
+      Mac mac = Mac.getInstance(selectedAlgorithm);
+      macInitialized = true;
+      return mac;
+    } catch (NoSuchAlgorithmException nsa) {
+      throw new IllegalArgumentException("Can't find " + selectedAlgorithm, nsa);
+    }
   }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestSecretManager.java

@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.security.token;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import javax.crypto.SecretKey;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class TestSecretManager {
+
+  private final String defaultAlgorithm =
+      CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT;
+  private final int defaultLength =
+      CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_DEFAULT;
+  private final String strongAlgorithm = "HmacSHA256";
+  private final int strongLength = 256;
+  private SecretManager<TokenIdentifier> secretManager;
+
+  @Test
+  public void testDefaults() {
+    assertKey(secretManager.generateSecret(), defaultAlgorithm, defaultLength);
+  }
+
+  @Test
+  public void testUpdate() {
+    SecretManager.update(createConfiguration(strongAlgorithm, strongLength));
+    assertKey(secretManager.generateSecret(), strongAlgorithm, strongLength);
+  }
+
+  @Test
+  public void testUnknownAlgorithm() {
+    SecretManager.update(createConfiguration("testUnknownAlgorithm_NO_ALG", strongLength));
+    assertThrows(IllegalArgumentException.class, secretManager::generateSecret);
+  }
+
+  @Test
+  public void testUpdateAfterInitialisation() {
+    SecretKey oldSecretKey = secretManager.generateSecret();
+    SecretManager.update(createConfiguration(strongAlgorithm, strongLength));
+    SecretKey newSecretKey = secretManager.generateSecret();
+    assertKey(oldSecretKey, defaultAlgorithm, defaultLength);
+    assertKey(newSecretKey, defaultAlgorithm, defaultLength);
+  }
+
+  @BeforeEach
+  public void setUp() {
+    secretManager = new SecretManager<TokenIdentifier>() {
+      @Override
+      protected byte[] createPassword(TokenIdentifier identifier) {
+        return new byte[0];
+      }
+
+      @Override
+      public byte[] retrievePassword(TokenIdentifier identifier) throws InvalidToken {
+        return new byte[0];
+      }
+
+      @Override
+      public TokenIdentifier createIdentifier() {
+        return null;
+      }
+    };
+  }
+
+  @AfterEach
+  public void tearDown() {
+    SecretManager.update(createConfiguration(defaultAlgorithm, defaultLength));
+  }
+
+  private void assertKey(SecretKey secretKey, String algorithm, int length) {
+    assertEquals(algorithm, secretKey.getAlgorithm(),
+        "Algorithm of created key is not as expected.");
+    assertEquals(length, secretKey.getEncoded().length * 8,
+        "Length of created key is not as expected.");
+  }
+
+  private Configuration createConfiguration(String algorithm, int length) {
+    Configuration conf = new Configuration();
+    conf.set(
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY,
+        algorithm);
+    conf.setInt(CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_KEY,
+        length);
+    return conf;
+  }
+}