HADOOP-16319 skip invalid tests when default encryption enabled

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md

@@ -168,6 +168,13 @@ You can also force all the tests to run with a specific SSE encryption method
 by configuring the property `fs.s3a.server-side-encryption-algorithm` in the s3a
 contract file.
 
+### <a name="default_encyption"></a> Default Encryption
+
+Buckets can be configured with [default encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html)
+on the AWS side. Some S3AFileSystem tests are skipped when default encryption is
+enabled due to unpredictability in how [ETags](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html)
+are generated.
+
 ## <a name="running"></a> Running the Tests
 
 After completing the configuration, execute the test run through Maven.

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java

@@ -23,7 +23,10 @@
 import java.io.IOException;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.AccessDeniedException;
 
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GetBucketEncryptionResult;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.PutObjectRequest;
 import org.junit.Assume;
@@ -44,6 +47,7 @@
 import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM;
 import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.hamcrest.Matchers.nullValue;
 
 /**
  * Tests of the S3A FileSystem which don't have a specific home and can share
@@ -151,10 +155,12 @@ Path mkFile(String name, byte[] data) throws IOException {
   /**
    * The assumption here is that 0-byte files uploaded in a single PUT
    * always have the same checksum, including stores with encryption.
+   * This will be skipped if the bucket has S3 default encryption enabled.
    * @throws Throwable on a failure
    */
   @Test
   public void testEmptyFileChecksums() throws Throwable {
+    assumeNoDefaultEncryption();
     final S3AFileSystem fs = getFileSystem();
     Path file1 = touchFile("file1");
     EtagChecksum checksum1 = fs.getFileChecksum(file1, 0);
@@ -167,6 +173,20 @@ public void testEmptyFileChecksums() throws Throwable {
         fs.getFileChecksum(touchFile("file2"), 0));
   }
 
+  /**
+   * Skip a test if we can get the default encryption on a bucket and it is
+   * non-null.
+   */
+  private void assumeNoDefaultEncryption() throws IOException {
+    try {
+      Assume.assumeThat(getDefaultEncryption(), nullValue());
+    } catch (AccessDeniedException e) {
+      // if the user can't check the default encryption, assume that it is
+      // null and keep going
+      LOG.warn("User does not have permission to call getBucketEncryption()");
+    }
+  }
+
   /**
    * Make sure that when checksums are disabled, the caller
    * gets null back.
@@ -207,12 +227,13 @@ public void testNonEmptyFileChecksums() throws Throwable {
   /**
    * Verify that on an unencrypted store, the checksum of two non-empty
    * (single PUT) files is the same if the data is the same.
-   * This will fail if the bucket has S3 default encryption enabled.
+   * This will be skipped if the bucket has S3 default encryption enabled.
    * @throws Throwable failure
    */
   @Test
   public void testNonEmptyFileChecksumsUnencrypted() throws Throwable {
     Assume.assumeTrue(encryptionAlgorithm().equals(S3AEncryptionMethods.NONE));
+    assumeNoDefaultEncryption();
     final S3AFileSystem fs = getFileSystem();
     final EtagChecksum checksum1 =
         fs.getFileChecksum(mkFile("file5", HELLO), 0);
@@ -256,6 +277,7 @@ public void testS3AToStringUnitialized() throws Throwable {
   }
 
   /**
+<<<<<<< ours
    * Verify that paths with a trailing "/" are fixed up.
    */
   @Test
@@ -368,4 +390,21 @@ private static <T> T verifyNoTrailingSlash(String role, T o) {
         s.endsWith("/"));
     return o;
   }
+
+  /**
+   * Gets default encryption settings for the bucket or returns null if default
+   * encryption is disabled.
+   */
+  private GetBucketEncryptionResult getDefaultEncryption() throws IOException {
+    S3AFileSystem fs = getFileSystem();
+    AmazonS3 s3 = fs.getAmazonS3ClientForTesting("check default encryption");
+    try {
+      return Invoker.once("getBucketEncryption()",
+          fs.getBucket(),
+          () -> s3.getBucketEncryption(fs.getBucket()));
+    } catch (FileNotFoundException e) {
+      return null;
+    }
+  }
+
 }