From 51a38ac75c9b60e93bad552fc626fc772181fa5d Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 12 Mar 2025 14:23:35 +0100 Subject: [PATCH 1/4] HDDS-12569. Extract MiniOzoneCluster to separate module --- .../mini-chaos-tests/pom.xml | 5 + hadoop-ozone/integration-test/pom.xml | 5 + .../dev-support/findbugsExcludeFile.xml | 16 +++ hadoop-ozone/mini-cluster/pom.xml | 133 ++++++++++++++++++ .../apache/hadoop/ozone/MiniOzoneCluster.java | 0 .../hadoop/ozone/MiniOzoneClusterImpl.java | 0 .../ozone/MiniOzoneClusterProvider.java | 0 .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 0 .../hadoop/ozone/UniformDatanodesFactory.java | 0 .../org/apache/hadoop/ozone/package-info.java | 21 +++ hadoop-ozone/pom.xml | 6 + 11 files changed, 186 insertions(+) create mode 100644 hadoop-ozone/mini-cluster/dev-support/findbugsExcludeFile.xml create mode 100644 hadoop-ozone/mini-cluster/pom.xml rename hadoop-ozone/{integration-test/src/test => mini-cluster/src/main}/java/org/apache/hadoop/ozone/MiniOzoneCluster.java (100%) rename hadoop-ozone/{integration-test/src/test => mini-cluster/src/main}/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java (100%) rename hadoop-ozone/{integration-test/src/test => mini-cluster/src/main}/java/org/apache/hadoop/ozone/MiniOzoneClusterProvider.java (100%) rename hadoop-ozone/{integration-test/src/test => mini-cluster/src/main}/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java (100%) rename hadoop-ozone/{integration-test/src/test => mini-cluster/src/main}/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java (100%) create mode 100644 hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/package-info.java diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml b/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml index cb2725c97fba..12c7f0fc0779 100644 --- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml +++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml @@ -65,6 +65,11 @@ ozone-manager test + + org.apache.ozone + ozone-mini-cluster + test + org.apache.ozone ozone-recon diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml index bce2c1b6f515..1a330963162e 100644 --- a/hadoop-ozone/integration-test/pom.xml +++ b/hadoop-ozone/integration-test/pom.xml @@ -281,6 +281,11 @@ test-jar test + + org.apache.ozone + ozone-mini-cluster + test + software.amazon.awssdk s3 diff --git a/hadoop-ozone/mini-cluster/dev-support/findbugsExcludeFile.xml b/hadoop-ozone/mini-cluster/dev-support/findbugsExcludeFile.xml new file mode 100644 index 000000000000..ee5ed59808bf --- /dev/null +++ b/hadoop-ozone/mini-cluster/dev-support/findbugsExcludeFile.xml @@ -0,0 +1,16 @@ + + + + diff --git a/hadoop-ozone/mini-cluster/pom.xml b/hadoop-ozone/mini-cluster/pom.xml new file mode 100644 index 000000000000..26a9906072c5 --- /dev/null +++ b/hadoop-ozone/mini-cluster/pom.xml @@ -0,0 +1,133 @@ + + + + 4.0.0 + + org.apache.ozone + ozone + 2.0.0-SNAPSHOT + + ozone-mini-cluster + 2.0.0-SNAPSHOT + jar + Apache Ozone Mini Cluster + Apache Ozone Mini Cluster for Integration Tests + + + + com.google.guava + guava + + + commons-io + commons-io + + + org.apache.ozone + hdds-common + + + org.apache.ozone + hdds-common + test-jar + + + org.apache.ozone + hdds-config + + + org.apache.ozone + hdds-container-service + + + org.apache.ozone + hdds-interface-client + + + org.apache.ozone + hdds-managed-rocksdb + + + org.apache.ozone + hdds-server-framework + + + org.apache.ozone + hdds-server-scm + + + org.apache.ozone + hdds-server-scm + test-jar + + + org.apache.ozone + hdds-test-utils + test-jar + + + org.apache.ozone + ozone-client + + + org.apache.ozone + ozone-common + + + org.apache.ozone + ozone-manager + + + org.apache.ozone + ozone-recon + + + org.apache.ozone + ozone-reconcodegen + + + org.apache.ozone + ozone-s3gateway + + + org.apache.ratis + ratis-common + + + org.slf4j + slf4j-api + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/dev-support/findbugsExcludeFile.xml + + + + org.apache.maven.plugins + maven-compiler-plugin + + none + + + + + + diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java rename to hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java rename to hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterProvider.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterProvider.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterProvider.java rename to hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterProvider.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java rename to hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java similarity index 100% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java rename to hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/package-info.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/package-info.java new file mode 100644 index 000000000000..812e63329539 --- /dev/null +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Mini cluster for running Ozone in a single JVM for tests. + */ +package org.apache.hadoop.ozone; diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index 98e982f13ae0..a237ac2b2f53 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -37,6 +37,7 @@ integration-test interface-client interface-storage + mini-cluster ozone-manager ozonefs ozonefs-common @@ -266,6 +267,11 @@ ${ozone.version} test-jar + + org.apache.ozone + ozone-mini-cluster + ${ozone.version} + org.apache.ozone ozone-recon From 7ab7a69c13e6942b0c08d0410aa8a6afbf08ca36 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 18 Mar 2025 18:24:36 +0100 Subject: [PATCH 2/4] selective checks: mini-cluster only for integration test --- dev-support/ci/selective_ci_checks.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-support/ci/selective_ci_checks.sh b/dev-support/ci/selective_ci_checks.sh index 213d071b9119..3cd309e93280 100755 --- a/dev-support/ci/selective_ci_checks.sh +++ b/dev-support/ci/selective_ci_checks.sh @@ -260,6 +260,7 @@ function get_count_integration_files() { "^hadoop-ozone/dev-support/checks/integration.sh" "^hadoop-ozone/dev-support/checks/junit.sh" "^hadoop-ozone/integration-test" + "^hadoop-ozone/mini-cluster" "^hadoop-ozone/fault-injection-test/mini-chaos-tests" "src/test/java" "src/test/resources" From 2a62a462436d3645768e38d2b7eebc59713ccc73 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 18 Mar 2025 18:32:21 +0100 Subject: [PATCH 3/4] s3gateway not needed for mini-cluster --- hadoop-ozone/mini-cluster/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hadoop-ozone/mini-cluster/pom.xml b/hadoop-ozone/mini-cluster/pom.xml index 26a9906072c5..1bbac0a8e651 100644 --- a/hadoop-ozone/mini-cluster/pom.xml +++ b/hadoop-ozone/mini-cluster/pom.xml @@ -97,10 +97,6 @@ org.apache.ozone ozone-reconcodegen - - org.apache.ozone - ozone-s3gateway - org.apache.ratis ratis-common From df82cb6eecb32d50ff10862f49a57a79971f0bd4 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 19 Mar 2025 06:59:38 +0100 Subject: [PATCH 4/4] update to 2.1.0-SNAPSHOT --- hadoop-ozone/mini-cluster/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/mini-cluster/pom.xml b/hadoop-ozone/mini-cluster/pom.xml index 1bbac0a8e651..be6317d9344c 100644 --- a/hadoop-ozone/mini-cluster/pom.xml +++ b/hadoop-ozone/mini-cluster/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.0.0-SNAPSHOT + 2.1.0-SNAPSHOT ozone-mini-cluster - 2.0.0-SNAPSHOT + 2.1.0-SNAPSHOT jar Apache Ozone Mini Cluster Apache Ozone Mini Cluster for Integration Tests