Hadoop 17912 pr review 2

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java

@@ -21,7 +21,6 @@
 import java.io.IOException;
 import java.lang.reflect.Field;
 
-import org.apache.hadoop.fs.azurebfs.extensions.EncryptionContextProvider;
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.util.Preconditions;
 
@@ -50,6 +49,7 @@
 import org.apache.hadoop.fs.azurebfs.diagnostics.StringConfigurationBasicValidator;
 import org.apache.hadoop.fs.azurebfs.enums.Trilean;
 import org.apache.hadoop.fs.azurebfs.extensions.CustomTokenProviderAdaptee;
+import org.apache.hadoop.fs.azurebfs.extensions.EncryptionContextProvider;
 import org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider;
 import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
 import org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider;

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java

@@ -111,6 +111,8 @@ public final class AbfsHttpConstants {
   public static final char CHAR_EQUALS = '=';
   public static final char CHAR_STAR = '*';
   public static final char CHAR_PLUS = '+';
+  public static final String DECEMBER_2019_API_VERSION = "2019-12-12";
+  public static final String APRIL_2021_API_VERSION = "2021-04-10";
 
   private AbfsHttpConstants() {}
 }

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java

@@ -188,7 +188,7 @@ public final class ConfigurationKeys {
   public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script";
   /** Setting this true will make the driver use it's own RemoteIterator implementation */
   public static final String FS_AZURE_ENABLE_ABFS_LIST_ITERATOR = "fs.azure.enable.abfslistiterator";
-  /** Server side encryption key encoded in Base6format */
+  /** Server side encryption key encoded in Base6format {@value}.*/
   public static final String FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY =
       "fs.azure.encryption.encoded.client-provided-key";
   /** SHA256 hash of encryption key encoded in Base64format */

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultEntrySchema.java

@@ -77,6 +77,18 @@ public class ListResultEntrySchema {
   @JsonProperty(value = "permissions")
   private String permissions;
 
+  /**
+   *  The encryption context property
+   */
+  @JsonProperty(value = "EncryptionContext")
+  private String xMsEncryptionContext;
+
+  /**
+   * The customer-provided encryption-256 value
+   * */
+  @JsonProperty(value = "CustomerProvidedKeySha256")
+  private String customerProvidedKeySha256;
+
   /**
    * Get the name value.
    *
@@ -238,4 +250,19 @@ public ListResultEntrySchema withPermissions(final String permissions) {
     return this;
   }
 
-}
+  /**
+   * Get the x-ms-encryption-context value.
+   * @return the x-ms-encryption-context value.
+   * */
+  public String getXMsEncryptionContext() {
+    return xMsEncryptionContext;
+  }
+
+  /**
+   * Get the customer-provided sha-256 key
+   * @return the x-ms-encryption-key-sha256 value used by client.
+   * */
+  public String getCustomerProvidedKeySha256() {
+    return customerProvidedKeySha256;
+  }
+}

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/EncryptionContextProvider.java

@@ -24,6 +24,17 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.azurebfs.security.ABFSKey;
 
+/**
+ * This interface has two roles:<br>
+ * <ul>
+ *   <li>
+ *     To create new encryptionContext from a given path: To be used in case of
+ *     create file as there is no encryptionContext in remote server to refer to
+ *     for encryptionKey creation.
+ *   </li>
+ *   <li>To create encryptionKey using encryptionContext.</li>
+ * </ul>
+ * */
 public interface EncryptionContextProvider extends Destroyable {
   /**
    * Initialize instance
@@ -53,6 +64,4 @@ public interface EncryptionContextProvider extends Destroyable {
    * @throws IOException error in fetching encryption key
    */
   ABFSKey getEncryptionKey(String path, ABFSKey encryptionContext) throws IOException;
-
-  void destroy();
 }

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/ABFSKey.java

@@ -21,39 +21,44 @@
 import javax.crypto.SecretKey;
 import java.util.Arrays;
 
-public class ABFSKey implements SecretKey {
-    private byte[] bytes;
+/**
+ * Implementation of SecretKey that would be used by EncryptionAdapter object,
+ * implementations of encryptionContextProvider to maintain the byteArrays of
+ * encryptionContext and encryptionKey.
+ * */
+public final class ABFSKey implements SecretKey {
+  private byte[] bytes;
 
-    public ABFSKey(byte[] bytes) {
-        if (bytes != null) {
-            this.bytes = bytes.clone();
-        }
+  public ABFSKey(byte[] bytes) {
+    if (bytes != null) {
+      this.bytes = bytes.clone();
     }
+  }
 
-    @Override
-    public String getAlgorithm() {
-        return null;
-    }
+  @Override
+  public String getAlgorithm() {
+    return null;
+  }
 
-    @Override
-    public String getFormat() {
-        return null;
-    }
+  @Override
+  public String getFormat() {
+    return null;
+  }
 
-    /**
-     * This method to be called by implementations of EncryptionContextProvider interface.
-     * Method returns clone of the original bytes array to prevent findbugs flags.
-     * */
-    @Override
-    public byte[] getEncoded() {
-        if (bytes == null) {
-            return null;
-        }
-        return bytes.clone();
+  /**
+   * This method to be called by implementations of EncryptionContextProvider interface.
+   * Method returns clone of the original bytes array to prevent findbugs flags.
+   * */
+  @Override
+  public byte[] getEncoded() {
+    if (bytes == null) {
+      return null;
     }
+    return bytes.clone();
+  }
 
-    @Override
-    public void destroy() {
-        Arrays.fill(bytes, (byte) 0);
-    }
+  @Override
+  public void destroy() {
+    Arrays.fill(bytes, (byte) 0);
+  }
 }

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/EncodingHelper.java

@@ -22,22 +22,25 @@
 import java.security.NoSuchAlgorithmException;
 import java.util.Base64;
 
-public class EncodingHelper {
+/**
+ * Utility class for managing encryption of bytes or base64String conversion of
+ * bytes.
+ * */
+public final class EncodingHelper {
 
-    public static byte[] getSHA256Hash(byte[] key) {
-        try {
-            final MessageDigest digester = MessageDigest.getInstance("SHA-256");
-            return digester.digest(key);
-        } catch (NoSuchAlgorithmException ignored) {
-            /**
-             * This exception can be ignored. Reason being SHA-256 is a valid algorithm, and it is constant for all
-             * method calls.
-             */
-            return null;
-        }
+  public static byte[] getSHA256Hash(byte[] key) {
+    try {
+      final MessageDigest digester = MessageDigest.getInstance("SHA-256");
+      return digester.digest(key);
+    } catch (NoSuchAlgorithmException noSuchAlgorithmException) {
+      /*This exception can be ignored. Reason being SHA-256 is a valid algorithm,
+       and it is constant for all method calls.*/
+      throw new RuntimeException("SHA-256 algorithm not found in MessageDigest",
+          noSuchAlgorithmException);
     }
+  }
 
-    public static String getBase64EncodedString(byte[] bytes) {
-        return Base64.getEncoder().encodeToString(bytes);
-    }
+  public static String getBase64EncodedString(byte[] bytes) {
+    return Base64.getEncoder().encodeToString(bytes);
+  }
 }

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/EncryptionAdapter.java

@@ -21,62 +21,81 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Base64;
-import javax.security.auth.DestroyFailedException;
-import javax.security.auth.Destroyable;
-
-import org.apache.hadoop.util.Preconditions;
+import java.util.Objects;
 
 import org.apache.hadoop.fs.azurebfs.extensions.EncryptionContextProvider;
 
-public class EncryptionAdapter implements Destroyable {
+import static org.apache.hadoop.fs.azurebfs.security.EncodingHelper.getBase64EncodedString;
+
+/**
+ * Class manages the encryptionContext and encryptionKey that needs to be added
+ * to the headers to server request if Customer-Encryption-Context is enabled in
+ * the configuration.
+ * <br>
+ * For fileCreation, the object helps in creating encryptionContext through the
+ * implementation of EncryptionContextProvider.
+ * <br>
+ * For all operations, the object helps in converting encryptionContext to
+ * encryptionKey through the implementation of EncryptionContextProvider.
+ * */
+public class EncryptionAdapter {
   private final String path;
-  private ABFSKey encryptionContext;
+  private final ABFSKey encryptionContext;
   private ABFSKey encryptionKey;
   private final EncryptionContextProvider provider;
 
+  /**
+   * Following constructor called when the encryptionContext of file is known.
+   * The server shall send encryptionContext as a String, the constructor shall
+   * convert the string into a byte-array. The converted byte-array would be used
+   * by the implementation of EncryptionContextProvider to create byte-array of
+   * encryptionKey.
+   * */
   public EncryptionAdapter(EncryptionContextProvider provider, String path,
       byte[] encryptionContext) throws IOException {
-    this(provider, path);
-    Preconditions.checkNotNull(encryptionContext,
+    this.provider = provider;
+    this.path = path;
+    Objects.requireNonNull(encryptionContext,
         "Encryption context should not be null.");
     this.encryptionContext = new ABFSKey(Base64.getDecoder().decode(encryptionContext));
     Arrays.fill(encryptionContext, (byte) 0);
+    computeKeys();
   }
 
+  /**
+   * Following constructor called in case of createPath. Since, the path is not
+   * on the server, encryptionContext is not there for the path. Implementation
+   * of the EncryptionContextProvider would be used to create encryptionContext
+   * from the path.
+   * */
   public EncryptionAdapter(EncryptionContextProvider provider, String path)
       throws IOException {
     this.provider = provider;
     this.path = path;
+    encryptionContext = provider.getEncryptionContext(path);
+    Objects.requireNonNull(encryptionContext,
+        "Encryption context should not be null.");
+    computeKeys();
   }
 
   private void computeKeys() throws IOException {
-    if (encryptionContext == null) {
-      encryptionContext = provider.getEncryptionContext(path);
-    }
-    Preconditions.checkNotNull(encryptionContext,
-            "Encryption context should not be null.");
-    if (encryptionKey == null) {
-      encryptionKey = provider.getEncryptionKey(path, encryptionContext);
-    }
-    Preconditions.checkNotNull(encryptionKey, "Encryption key should not be null.");
+    encryptionKey = provider.getEncryptionKey(path, encryptionContext);
+    Objects.requireNonNull(encryptionKey, "Encryption key should not be null.");
   }
 
   public String getEncodedKey() throws IOException {
-    computeKeys();
-    return EncodingHelper.getBase64EncodedString(encryptionKey.getEncoded());
+    return getBase64EncodedString(encryptionKey.getEncoded());
   }
 
   public String getEncodedKeySHA() throws IOException {
-    computeKeys();
-    return EncodingHelper.getBase64EncodedString(EncodingHelper.getSHA256Hash(encryptionKey.getEncoded()));
+    return getBase64EncodedString(EncodingHelper.getSHA256Hash(encryptionKey.getEncoded()));
   }
 
   public String getEncodedContext() throws IOException {
-    computeKeys();
-    return EncodingHelper.getBase64EncodedString(encryptionContext.getEncoded());
+    return getBase64EncodedString(encryptionContext.getEncoded());
   }
 
-  public void destroy() throws DestroyFailedException {
+  public void destroy() {
     if (encryptionContext != null) {
       encryptionContext.destroy();
     }