diff --git a/hadoop-ozone/dev-support/checks/unit.sh b/hadoop-ozone/dev-support/checks/unit.sh index f4a0a834d1bc..20eb4b955fc0 100755 --- a/hadoop-ozone/dev-support/checks/unit.sh +++ b/hadoop-ozone/dev-support/checks/unit.sh @@ -17,5 +17,5 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" CHECK=unit source "${DIR}/junit.sh" \ - -pl \!:ozone-integration-test,\!:ozone-integration-test-s3,\!:mini-chaos-tests \ + -pl \!:ozone-integration-test,\!:ozone-integration-test-recon,\!:ozone-integration-test-s3,\!:mini-chaos-tests \ "$@" diff --git a/hadoop-ozone/integration-test-recon/dev-support/findbugsExcludeFile.xml b/hadoop-ozone/integration-test-recon/dev-support/findbugsExcludeFile.xml new file mode 100644 index 000000000000..ee5ed59808bf --- /dev/null +++ b/hadoop-ozone/integration-test-recon/dev-support/findbugsExcludeFile.xml @@ -0,0 +1,16 @@ + + + + diff --git a/hadoop-ozone/integration-test-recon/pom.xml b/hadoop-ozone/integration-test-recon/pom.xml new file mode 100644 index 000000000000..ce2e8222e3c3 --- /dev/null +++ b/hadoop-ozone/integration-test-recon/pom.xml @@ -0,0 +1,217 @@ + + + + 4.0.0 + + org.apache.ozone + ozone + 2.1.0-SNAPSHOT + + ozone-integration-test-recon + 2.1.0-SNAPSHOT + jar + Apache Ozone Recon Integration Tests + Apache Ozone Integration Tests with Recon + + + + com.fasterxml.jackson.core + jackson-core + test + + + com.fasterxml.jackson.core + jackson-databind + test + + + com.google.guava + guava + test + + + commons-io + commons-io + test + + + jakarta.ws.rs + jakarta.ws.rs-api + test + + + org.apache.hadoop + hadoop-common + test + + + org.apache.hadoop + hadoop-hdfs-client + test + + + org.apache.httpcomponents + httpclient + test + + + org.apache.httpcomponents + httpcore + test + + + org.apache.ozone + hdds-client + test + + + org.apache.ozone + hdds-common + test + + + org.apache.ozone + hdds-common + test-jar + test + + + org.apache.ozone + hdds-config + test + + + org.apache.ozone + hdds-interface-client + test + + + org.apache.ozone + hdds-server-framework + test + + + org.apache.ozone + hdds-server-scm + test + + + org.apache.ozone + hdds-server-scm + test-jar + test + + + org.apache.ozone + hdds-test-utils + test-jar + test + + + org.apache.ozone + ozone-cli-admin + test + + + org.apache.ozone + ozone-client + test + + + org.apache.ozone + ozone-common + test + + + org.apache.ozone + ozone-integration-test + test-jar + test + + + org.apache.ozone + ozone-interface-storage + test + + + org.apache.ozone + ozone-manager + test + + + com.sun.jersey + * + + + + + org.apache.ozone + ozone-manager + test-jar + test + + + com.sun.jersey + * + + + + + org.apache.ozone + ozone-mini-cluster + test + + + org.apache.ozone + ozone-recon + test + + + org.apache.ozone + ozone-reconcodegen + test + + + org.apache.ratis + ratis-common + test + + + org.slf4j + slf4j-api + test + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/dev-support/findbugsExcludeFile.xml + + + + org.apache.maven.plugins + maven-compiler-plugin + + none + + + + + + diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/StandardOutputTestBase.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/StandardOutputTestBase.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/StandardOutputTestBase.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/StandardOutputTestBase.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/ReconService.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/ReconService.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/ReconService.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/ReconService.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryAdmin.java similarity index 98% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryAdmin.java index 89f98d6fcf88..6523ab97ff72 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryAdmin.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hadoop.ozone.shell; +package org.apache.hadoop.ozone.recon; import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY; import static org.assertj.core.api.Assertions.assertThat; @@ -33,7 +33,6 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.ozone.recon.ReconService; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerEndpoint.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerEndpoint.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerEndpoint.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerEndpoint.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconEndpointUtil.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconEndpointUtil.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconEndpointUtil.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconEndpointUtil.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconScmSnapshot.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconScmSnapshot.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconScmSnapshot.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconScmSnapshot.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java rename to hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java diff --git a/hadoop-ozone/integration-test-recon/src/test/resources/ozone-site.xml b/hadoop-ozone/integration-test-recon/src/test/resources/ozone-site.xml new file mode 100644 index 000000000000..a9c6511de7ec --- /dev/null +++ b/hadoop-ozone/integration-test-recon/src/test/resources/ozone-site.xml @@ -0,0 +1,113 @@ + + + + + + + ozone.om.transport.class + org.apache.hadoop.ozone.om.protocolPB.Hadoop3OmTransportFactory + + + ozone.om.s3.grpc.server_enabled + false + + + hdds.container.ratis.num.write.chunk.threads.per.volume + 4 + + + ozone.scm.handler.count.key + 20 + + + ozone.om.handler.count.key + 20 + + + hdds.container.ratis.datastream.enabled + true + + + hdds.heartbeat.interval + 1s + + + ozone.scm.heartbeat.thread.interval + 100ms + + + ozone.scm.ratis.pipeline.limit + 3 + + + ozone.scm.close.container.wait.duration + 1s + + + ozone.om.snapshot.diff.job.default.wait.time + 1s + + + hdds.container.ratis.log.appender.queue.byte-limit + 32MB + + + ozone.om.ratis.log.appender.queue.byte-limit + 4MB + + + ozone.scm.ha.ratis.log.appender.queue.byte-limit + 4MB + + + ozone.scm.chunk.size + 1MB + + + ozone.scm.block.size + 4MB + + + ozone.scm.container.size + 128MB + + + ozone.client.stream.buffer.flush.size + 1MB + + + ozone.client.stream.buffer.max.size + 2MB + + + ozone.client.stream.buffer.size + 1MB + + + ozone.client.datastream.buffer.flush.size + 4MB + + + ozone.client.datastream.min.packet.size + 256KB + + + ozone.client.datastream.window.size + 8MB + + diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index 0c21faa31c07..33ab6235f291 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -37,6 +37,7 @@ httpfsgateway insight integration-test + integration-test-recon integration-test-s3 interface-client interface-storage