diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractAnalyticsStreamVectoredRead.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractAnalyticsStreamVectoredRead.java index a2b053783bcd2..8cf182680c350 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractAnalyticsStreamVectoredRead.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractAnalyticsStreamVectoredRead.java @@ -23,12 +23,15 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; import static org.apache.hadoop.fs.s3a.S3ATestUtils.enableAnalyticsAccelerator; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipForAnyEncryptionExceptSSES3; /** - * S3A contract tests for vectored reads with the Analytics stream. The analytics stream does - * not explicitly implement the vectoredRead() method, or currently do and vectored-read specific - * optimisations (such as range coalescing). However, this test ensures that the base implementation - * of readVectored {@link org.apache.hadoop.fs.PositionedReadable} still works. + * S3A contract tests for vectored reads with the Analytics stream. + * The analytics stream does not explicitly implement the vectoredRead() method, + * or currently do and vectored-read specific optimisations + * (such as range coalescing). However, this test ensures that the base + * implementation of readVectored {@link org.apache.hadoop.fs.PositionedReadable} + * still works. */ public class ITestS3AContractAnalyticsStreamVectoredRead extends AbstractContractVectoredReadTest { @@ -44,6 +47,16 @@ public ITestS3AContractAnalyticsStreamVectoredRead(String bufferType) { protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); enableAnalyticsAccelerator(conf); + // If encryption is set, some AAL tests will fail. + // This is because AAL caches the head request response, and uses + // the eTag when making a GET request. When using encryption, the eTag is + // no longer a hash of the object content, and is not always the same when + // the same object is created multiple times. This test creates the file + // vectored_file.txt before running each test, which will have a + // different eTag when using encryption, leading to preconditioned failures. + // This issue is tracked in: + // https://github.com/awslabs/analytics-accelerator-s3/issues/218 + skipForAnyEncryptionExceptSSES3(conf); conf.set("fs.contract.vector-io-early-eof-check", "false"); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java index 9a0afdf651315..816ec90646f8a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java @@ -69,6 +69,7 @@ public class ITestS3AAnalyticsAcceleratorStreamReading extends AbstractS3ATestBa @Before public void setUp() throws Exception { super.setup(); + skipIfClientSideEncryption(); externalTestFile = getExternalData(getConfiguration()); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 53e4a68cbb60b..41c098a76dbcd 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -105,6 +105,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.impl.FlagSet.createFlagSet; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.SSE_S3; import static org.apache.hadoop.fs.s3a.impl.streams.InputStreamType.Analytics; import static org.apache.hadoop.fs.s3a.impl.streams.InputStreamType.Prefetch; import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit; @@ -1709,6 +1710,30 @@ public static void skipIfEncryptionNotSet(Configuration configuration, } } + /** + * Skip a test if encryption algorithm is not empty, or if it is set to + * anything other than AES256. + * + * @param configuration configuration + */ + public static void skipForAnyEncryptionExceptSSES3(Configuration configuration) { + String bucket = getTestBucketName(configuration); + try { + final EncryptionSecrets secrets = buildEncryptionSecrets(bucket, configuration); + S3AEncryptionMethods s3AEncryptionMethods = secrets.getEncryptionMethod(); + + if (s3AEncryptionMethods.getMethod().equals(SSE_S3.getMethod()) + || s3AEncryptionMethods.getMethod().isEmpty()) { + return; + } + + skip("Encryption method is set to " + s3AEncryptionMethods.getMethod()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + } + /** * Get the input stream statistics of an input stream. * Raises an exception if the inner stream is not an S3A input stream