diff --git a/LICENSE-binary b/LICENSE-binary index 5f5f96d00f1ff..88972a518f0d6 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -362,7 +362,7 @@ org.objenesis:objenesis:2.6 org.xerial.snappy:snappy-java:1.1.10.4 org.yaml:snakeyaml:2.0 org.wildfly.openssl:wildfly-openssl:2.1.4.Final -software.amazon.awssdk:bundle:2.25.53 +software.amazon.awssdk:bundle:jar:2.27.14 -------------------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b9469bc752372..8918218cb5f02 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -204,8 +204,8 @@ 1.0-beta-1 900 1.12.720 - 2.25.53 3.1.1 + 2.27.14 1.0.1 2.7.1 1.11.2 diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md index 53239b1295d18..e4b5c76d20c38 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md @@ -294,6 +294,7 @@ understands the risks. * If an option is to be tuned which may relax semantics, a new option MUST be defined. * Unknown flags are ignored; this is to avoid compatibility. * The option `*` means "turn everything on". This is implicitly unstable across releases. +* Other stores may retain stricter semantics. | *Option* | *Meaning* | Since | |----------|--------------------|:------| diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index 48e112f4f29b4..4176b20e8f54a 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -581,6 +581,10 @@ on third party stores. test.fs.s3a.create.create.acl.enabled false + + test.fs.s3a.performance.enabled + false + ``` See [Third Party Stores](third_party_stores.html) for more on this topic. @@ -720,6 +724,18 @@ Tests in `ITestS3AContentEncoding` may need disabling false ``` + +### Disabling tests running in performance mode + +Some tests running in performance mode turn off the safety checks. They expect operations which break POSIX semantics to succeed. +For stores with stricter semantics, these test cases must be disabled. +```xml + + test.fs.s3a.performance.enabled + false + +``` + ### Tests which may fail (and which you can ignore) * `ITestS3AContractMultipartUploader` tests `testMultipartUploadAbort` and `testSingleUpload` raising `FileNotFoundException` diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java index 000caf328837e..033c2d94c7bf8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java @@ -29,9 +29,11 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.s3a.S3ATestUtils; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_PERFORMANCE_TESTS_ENABLED; import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_EXPECT_CONTINUE; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfNotEnabled; /** * S3A contract tests creating files. @@ -84,6 +86,9 @@ protected Configuration createConfiguration() { conf, CONNECTION_EXPECT_CONTINUE); conf.setBoolean(CONNECTION_EXPECT_CONTINUE, expectContinue); + if (createPerformance) { + skipIfNotEnabled(conf, KEY_PERFORMANCE_TESTS_ENABLED, "Skipping tests running in performance mode"); + } S3ATestUtils.disableFilesystemCaching(conf); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java index 4b2468de97bb8..45dfc391b001d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java @@ -29,7 +29,9 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_PERFORMANCE_TESTS_ENABLED; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfNotEnabled; /** * Test mkdir operations on S3A with create performance mode. @@ -50,6 +52,8 @@ protected AbstractFSContract createContract(Configuration conf) { @Test public void testMkdirOverParentFile() throws Throwable { + skipIfNotEnabled(getContract().getConf(), KEY_PERFORMANCE_TESTS_ENABLED, + "Skipping tests running in performance mode"); describe("try to mkdir where a parent is a file, should pass"); FileSystem fs = getFileSystem(); Path path = methodPath(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java index 86ba242b67730..8743907545b73 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java @@ -63,6 +63,11 @@ public interface S3ATestConstants { */ String KEY_ACL_TESTS_ENABLED = TEST_FS_S3A + "create.acl.enabled"; + /** + * A property set to true if tests running in performance mode are enabled: {@value } + */ + String KEY_PERFORMANCE_TESTS_ENABLED = TEST_FS_S3A + "performance.enabled"; + /** * A property set to true if V1 tests are enabled: {@value}. */