From 072258d33fa5f12e0f796ecd14d73d4073ed8d8d Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:59:02 +0100 Subject: [PATCH 1/4] HDDS-12528. Create new module for S3 integration tests (#8152) (cherry picked from commit d44ebf564ce444c6171d0da05913e500a8038b47) --- .../apache/ozone/test}/InputSubstream.java | 7 +- .../dev-support/checks/_mvn_unit_report.sh | 4 +- hadoop-ozone/dev-support/checks/unit.sh | 2 +- .../dev-support/findbugsExcludeFile.xml | 16 +++ hadoop-ozone/integration-test-s3/pom.xml | 121 +++++++++++++++++ .../hadoop/ozone/s3/S3ClientFactory.java | 0 .../hadoop/ozone/s3/S3GatewayService.java | 0 .../ozone/s3/awssdk/S3SDKTestUtils.java | 2 +- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 0 .../ozone/s3/awssdk/v1/TestS3SDKV1.java | 0 .../v1/TestS3SDKV1WithRatisStreaming.java | 0 .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 0 .../ozone/s3/awssdk/v2/TestS3SDKV2.java | 0 .../v2/TestS3SDKV2WithRatisStreaming.java | 0 .../src/test/resources/ozone-site.xml | 126 ++++++++++++++++++ hadoop-ozone/integration-test/pom.xml | 5 - hadoop-ozone/pom.xml | 7 + 17 files changed, 277 insertions(+), 13 deletions(-) rename {hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/utils => hadoop-hdds/test-utils/src/test/java/org/apache/ozone/test}/InputSubstream.java (96%) create mode 100644 hadoop-ozone/integration-test-s3/dev-support/findbugsExcludeFile.xml create mode 100644 hadoop-ozone/integration-test-s3/pom.xml rename hadoop-ozone/{integration-test => integration-test-s3}/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java (100%) rename hadoop-ozone/{integration-test => integration-test-s3}/src/test/java/org/apache/hadoop/ozone/s3/S3GatewayService.java (100%) rename hadoop-ozone/{integration-test => integration-test-s3}/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java (98%) rename hadoop-ozone/{integration-test => integration-test-s3}/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java (100%) rename hadoop-ozone/{integration-test => integration-test-s3}/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java (100%) rename hadoop-ozone/{integration-test => integration-test-s3}/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1WithRatisStreaming.java (100%) rename hadoop-ozone/{integration-test => integration-test-s3}/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java (100%) rename hadoop-ozone/{integration-test => integration-test-s3}/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java (100%) rename hadoop-ozone/{integration-test => integration-test-s3}/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2WithRatisStreaming.java (100%) create mode 100644 hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/utils/InputSubstream.java b/hadoop-hdds/test-utils/src/test/java/org/apache/ozone/test/InputSubstream.java similarity index 96% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/utils/InputSubstream.java rename to hadoop-hdds/test-utils/src/test/java/org/apache/ozone/test/InputSubstream.java index e71689b9ac74..3c7b73aa9358 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/utils/InputSubstream.java +++ b/hadoop-hdds/test-utils/src/test/java/org/apache/ozone/test/InputSubstream.java @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.hadoop.utils; +package org.apache.ozone.test; -import com.google.common.base.Preconditions; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Objects; /** * A filter input stream implementation that exposes a range of the underlying input stream. @@ -33,8 +33,7 @@ public class InputSubstream extends FilterInputStream { private long markedPosition = 0; public InputSubstream(InputStream in, long skip, long length) { - super(in); - Preconditions.checkNotNull(in); + super(Objects.requireNonNull(in, "in == null")); this.currentPosition = 0; this.requestedSkipOffset = skip; this.requestedLength = length; diff --git a/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh index 4d27d5072874..8136e6ea806c 100755 --- a/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh +++ b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh @@ -38,7 +38,7 @@ cat ${failures} > "${tempfile}" leaks=${REPORT_DIR}/leaks.txt if [[ "${CHECK:-unit}" == "integration" ]]; then - find hadoop-ozone/integration-test -not -path '*/iteration*' -name '*-output.txt' -print0 \ + find hadoop-ozone/integration-test* -not -path '*/iteration*' -name '*-output.txt' -print0 \ | xargs -n1 -0 "grep" -l -E "not closed properly|was not shutdown properly" \ | awk -F/ '{sub("-output.txt",""); print $NF}' \ > "${leaks}" @@ -47,7 +47,7 @@ fi cluster=${REPORT_DIR}/cluster-startup-errors.txt if [[ "${CHECK:-unit}" == "integration" ]]; then - find hadoop-ozone/integration-test -not -path '*/iteration*' -name '*-output.txt' -print0 \ + find hadoop-ozone/integration-test* -not -path '*/iteration*' -name '*-output.txt' -print0 \ | xargs -n1 -0 "grep" -l -E "Unable to build MiniOzoneCluster" \ | awk -F/ '{sub("-output.txt",""); print $NF}' \ > "${cluster}" diff --git a/hadoop-ozone/dev-support/checks/unit.sh b/hadoop-ozone/dev-support/checks/unit.sh index 09f45795fefa..f4a0a834d1bc 100755 --- a/hadoop-ozone/dev-support/checks/unit.sh +++ b/hadoop-ozone/dev-support/checks/unit.sh @@ -17,5 +17,5 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" CHECK=unit source "${DIR}/junit.sh" \ - -pl \!:ozone-integration-test,\!:mini-chaos-tests \ + -pl \!:ozone-integration-test,\!:ozone-integration-test-s3,\!:mini-chaos-tests \ "$@" diff --git a/hadoop-ozone/integration-test-s3/dev-support/findbugsExcludeFile.xml b/hadoop-ozone/integration-test-s3/dev-support/findbugsExcludeFile.xml new file mode 100644 index 000000000000..ee5ed59808bf --- /dev/null +++ b/hadoop-ozone/integration-test-s3/dev-support/findbugsExcludeFile.xml @@ -0,0 +1,16 @@ + + + + diff --git a/hadoop-ozone/integration-test-s3/pom.xml b/hadoop-ozone/integration-test-s3/pom.xml new file mode 100644 index 000000000000..b23d389ef80a --- /dev/null +++ b/hadoop-ozone/integration-test-s3/pom.xml @@ -0,0 +1,121 @@ + + + + 4.0.0 + + org.apache.ozone + ozone + 2.1.0-SNAPSHOT + + ozone-integration-test-s3 + 2.1.0-SNAPSHOT + jar + Apache Ozone S3 Integration Tests + Apache Ozone Integration Tests with S3 Gateway + + + + com.amazonaws + aws-java-sdk-core + test + + + com.amazonaws + aws-java-sdk-s3 + test + + + jakarta.xml.bind + jakarta.xml.bind-api + test + + + org.apache.commons + commons-lang3 + test + + + org.apache.ozone + hdds-common + test + + + org.apache.ozone + hdds-config + test + + + org.apache.ozone + hdds-server-framework + test + + + org.apache.ozone + hdds-test-utils + test-jar + test + + + org.apache.ozone + ozone-client + test + + + org.apache.ozone + ozone-mini-cluster + test + + + org.apache.ozone + ozone-s3gateway + test + + + org.apache.ratis + ratis-common + test + + + org.slf4j + slf4j-api + test + + + software.amazon.awssdk + s3 + test + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/dev-support/findbugsExcludeFile.xml + + + + org.apache.maven.plugins + maven-compiler-plugin + + none + + + + + + diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java rename to hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/S3GatewayService.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3GatewayService.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/S3GatewayService.java rename to hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3GatewayService.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java similarity index 98% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java rename to hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java index 6703e4bbc477..33b7788e6960 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java @@ -23,7 +23,7 @@ import java.io.RandomAccessFile; import java.security.MessageDigest; import org.apache.commons.lang3.RandomUtils; -import org.apache.hadoop.utils.InputSubstream; +import org.apache.ozone.test.InputSubstream; /** * Utilities for S3 SDK tests. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java rename to hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java rename to hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1WithRatisStreaming.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1WithRatisStreaming.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1WithRatisStreaming.java rename to hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1WithRatisStreaming.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java rename to hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java rename to hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2WithRatisStreaming.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2WithRatisStreaming.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2WithRatisStreaming.java rename to hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2WithRatisStreaming.java diff --git a/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml b/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml new file mode 100644 index 000000000000..2354b6f321cb --- /dev/null +++ b/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml @@ -0,0 +1,126 @@ + + + + + + + + + + ozone.om.transport.class + org.apache.hadoop.ozone.om.protocolPB.Hadoop3OmTransportFactory + + + + ozone.om.s3.grpc.server_enabled + false + + + + hdds.container.ratis.num.write.chunk.threads.per.volume + 4 + + + + ozone.scm.handler.count.key + 20 + + + + ozone.om.handler.count.key + 20 + + + + hdds.container.ratis.datastream.enabled + true + + + + + hdds.heartbeat.interval + 1s + + + ozone.scm.heartbeat.thread.interval + 100ms + + + + ozone.scm.ratis.pipeline.limit + 3 + + + + ozone.scm.close.container.wait.duration + 1s + + + + ozone.om.snapshot.diff.job.default.wait.time + 1s + + + + hdds.container.ratis.log.appender.queue.byte-limit + + 32MB + + + ozone.om.ratis.log.appender.queue.byte-limit + 4MB + + + ozone.scm.ha.ratis.log.appender.queue.byte-limit + 4MB + + + + ozone.scm.chunk.size + 1MB + + + ozone.scm.block.size + 4MB + + + ozone.client.stream.buffer.flush.size + 1MB + + + ozone.client.stream.buffer.max.size + 2MB + + + ozone.client.stream.buffer.size + 1MB + + + ozone.client.datastream.buffer.flush.size + 4MB + + + ozone.client.datastream.min.packet.size + 256KB + + + ozone.client.datastream.window.size + 8MB + + + diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml index bbd06127c645..ff899d4e0b99 100644 --- a/hadoop-ozone/integration-test/pom.xml +++ b/hadoop-ozone/integration-test/pom.xml @@ -286,11 +286,6 @@ ozone-mini-cluster test - - software.amazon.awssdk - s3 - test - diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index 69cfcf48205c..7298e26ec68d 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -35,6 +35,7 @@ httpfsgateway insight integration-test + integration-test-s3 interface-client interface-storage mini-cluster @@ -147,6 +148,12 @@ hdds-server-framework ${hdds.version} + + org.apache.ozone + hdds-server-framework + ${hdds.version} + test-jar + org.apache.ozone hdds-server-scm From 7bbd8f4e6b2e7a69b43209839c23e033ae97bcd7 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 25 Mar 2025 17:39:54 +0100 Subject: [PATCH 2/4] HDDS-12693. Disk out of space in S3 integration test --- .../integration-test-s3/src/test/resources/ozone-site.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml b/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml index 2354b6f321cb..93d6e00df879 100644 --- a/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml +++ b/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml @@ -98,6 +98,10 @@ ozone.scm.block.size 4MB + + ozone.scm.container.size + 128MB + ozone.client.stream.buffer.flush.size 1MB From 4668c22628f6f93e3c0b57087d23a0372843b576 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 26 Mar 2025 16:19:54 +0100 Subject: [PATCH 3/4] remove extra newline in ozone-site.xml --- .../integration-test-s3/src/test/resources/ozone-site.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml b/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml index 93d6e00df879..916d7beeb72a 100644 --- a/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml +++ b/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml @@ -77,8 +77,7 @@ - hdds.container.ratis.log.appender.queue.byte-limit - + hdds.container.ratis.log.appender.queue.byte-limit 32MB From 78de49d92c663c10e1514b2439a87ecec8704235 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 26 Mar 2025 16:20:34 +0100 Subject: [PATCH 4/4] remove unnecessary --- hadoop-ozone/integration-test/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml index ff899d4e0b99..96b4cb0ba607 100644 --- a/hadoop-ozone/integration-test/pom.xml +++ b/hadoop-ozone/integration-test/pom.xml @@ -259,7 +259,6 @@ org.apache.ozone hdds-server-framework - ${hdds.version} test-jar test