From 03e670b865bf3381d79ea38d7ff36b5623507803 Mon Sep 17 00:00:00 2001 From: mingchao zhao Date: Wed, 19 Oct 2022 15:26:52 +0800 Subject: [PATCH 01/48] HDDS-7149. Update ratis version to 2.4.0 and thirdparty version to 1.0.2. (#3855) --- .../org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 7 +++++-- .../ozone/client/rpc/TestValidateBCSIDOnRestart.java | 2 +- .../hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java | 5 ++++- .../hadoop/ozone/om/ratis/OzoneManagerStateMachine.java | 8 ++++++-- pom.xml | 4 ++-- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java index 145a40a66502..3d0f22646096 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java @@ -353,8 +353,11 @@ public void notifyConfigurationChanged(long term, long index, @Override public void pause() { - getLifeCycle().transition(LifeCycle.State.PAUSING); - getLifeCycle().transition(LifeCycle.State.PAUSED); + final LifeCycle lc = getLifeCycle(); + if (lc.getCurrentState() != LifeCycle.State.NEW) { + lc.transition(LifeCycle.State.PAUSING); + lc.transition(LifeCycle.State.PAUSED); + } } @Override diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java index e70ffc54f2da..7f00825e34e6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java @@ -233,7 +233,7 @@ public void testValidateBCSIDOnDnRestart() throws Exception { // in the and what is there in RockSDB and hence the container would be // marked unhealthy index = cluster.getHddsDatanodeIndex(dn.getDatanodeDetails()); - cluster.restartHddsDatanode(dn.getDatanodeDetails(), false); + cluster.restartHddsDatanode(dn.getDatanodeDetails(), true); // Make sure the container is marked unhealthy Assert.assertTrue( cluster.getHddsDatanodes().get(index) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java index 084d3f08c169..3b8ad5faf347 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java @@ -51,6 +51,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; +import org.apache.ratis.util.LifeCycle; import org.junit.Assert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -273,7 +274,9 @@ public void testInstallCorruptedCheckpointFailure() throws Exception { Assert.assertTrue(logCapture.getOutput() .contains("Failed to reload SCM state and instantiate services.")); - Assert.assertTrue(followerSM.getLifeCycleState().isPausingOrPaused()); + final LifeCycle.State s = followerSM.getLifeCycleState(); + Assert.assertTrue("Unexpected lifeCycle state: " + s, + s == LifeCycle.State.NEW || s.isPausingOrPaused()); // Verify correct reloading followerSM.setInstallingDBCheckpoint( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java index 69b5e564155b..7bcc6706a999 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java @@ -391,8 +391,12 @@ public synchronized void pause() { if (getLifeCycleState() == LifeCycle.State.PAUSED) { return; } - getLifeCycle().transition(LifeCycle.State.PAUSING); - getLifeCycle().transition(LifeCycle.State.PAUSED); + final LifeCycle lc = getLifeCycle(); + if (lc.getCurrentState() != LifeCycle.State.NEW) { + getLifeCycle().transition(LifeCycle.State.PAUSING); + getLifeCycle().transition(LifeCycle.State.PAUSED); + } + ozoneManagerDoubleBuffer.stop(); } diff --git a/pom.xml b/pom.xml index 35ad9d7913af..d4f621172b25 100644 --- a/pom.xml +++ b/pom.xml @@ -72,10 +72,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs ${ozone.version} - 2.3.0 + 2.4.0 - 1.0.0 + 1.0.2 2.3.0 From a6316c832068ecc044de87f4c2671a9ca3b86043 Mon Sep 17 00:00:00 2001 From: Christos Bisias Date: Wed, 19 Oct 2022 11:28:32 +0300 Subject: [PATCH 02/48] HDDS-7352. OM log flooded by AWSV4AuthValidator (#3857) --- .../org/apache/hadoop/ozone/security/AWSV4AuthValidator.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java index 1b526e7873e0..3374039af3df 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/AWSV4AuthValidator.java @@ -84,7 +84,9 @@ private static byte[] getSigningKey(String key, String strToSign) { byte[] kRegion = sign(kDate, regionName); byte[] kService = sign(kRegion, serviceName); byte[] kSigning = sign(kService, "aws4_request"); - LOG.info(Hex.encode(kSigning)); + if (LOG.isDebugEnabled()) { + LOG.debug(Hex.encode(kSigning)); + } return kSigning; } From 237a9a15949299c700c7352019662429b0acf242 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Wed, 19 Oct 2022 09:32:27 +0100 Subject: [PATCH 03/48] HDDS-7058. EC: ReplicationManager - Implement ratis container replication check handler (#3802) --- .../replication/ContainerHealthResult.java | 69 +++ .../replication/LegacyReplicationManager.java | 1 - .../RatisContainerReplicaCount.java | 87 ++- .../replication/ReplicationManager.java | 19 +- .../health/RatisReplicationCheckHandler.java | 201 +++++++ .../TestRatisContainerReplicaCount.java | 58 +- .../TestRatisReplicationCheckHandler.java | 560 ++++++++++++++++++ .../scm/node/TestDatanodeAdminMonitor.java | 2 +- 8 files changed, 987 insertions(+), 10 deletions(-) rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/{ => replication}/RatisContainerReplicaCount.java (76%) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerHealthResult.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerHealthResult.java index de290294e91b..b438d6de4cc5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerHealthResult.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerHealthResult.java @@ -106,6 +106,9 @@ public static class UnderReplicatedHealthResult private final int remainingRedundancy; private final boolean dueToDecommission; private final boolean sufficientlyReplicatedAfterPending; + private boolean dueToMisReplication = false; + private boolean isMisReplicated = false; + private boolean isMisReplicatedAfterPending = false; private final boolean unrecoverable; private int requeueCount = 0; @@ -119,6 +122,44 @@ public UnderReplicatedHealthResult(ContainerInfo containerInfo, this.unrecoverable = unrecoverable; } + /** + * Pass true to indicate the container is mis-replicated - ie it does not + * meet the placement policy. + * @param isMisRep True if the container is mis-replicated, false if not. + * @return this object to allow calls to be chained + */ + public UnderReplicatedHealthResult + setMisReplicated(boolean isMisRep) { + this.isMisReplicated = isMisRep; + return this; + } + + /** + * Pass true to indicate the container is mis-replicated after considering + * pending replicas scheduled for create or delete. + * @param isMisRep True if the container is mis-replicated considering + * pending replicas, or false if not. + * @return this object to allow calls to be chained + */ + public UnderReplicatedHealthResult + setMisReplicatedAfterPending(boolean isMisRep) { + this.isMisReplicatedAfterPending = isMisRep; + return this; + } + + /** + * If the container is ONLY under replicated due to mis-replication, pass + * true, otherwise pass false. + * @param dueToMisRep Pass true if the container has enough replicas but + * does not meet the placement policy. + * @return + */ + public UnderReplicatedHealthResult + setDueToMisReplication(boolean dueToMisRep) { + this.dueToMisReplication = dueToMisRep; + return this; + } + /** * How many more replicas can be lost before the container is * unreadable. For containers which are under-replicated due to decommission @@ -187,6 +228,34 @@ public boolean isSufficientlyReplicatedAfterPending() { return sufficientlyReplicatedAfterPending; } + /** + * Returns true if the container is mis-replicated, ignoring any pending + * replicas scheduled to be created. + * @return True if mis-replicated, ignoring pending + */ + public boolean isMisReplicated() { + return isMisReplicated; + } + + /** + * Returns true if the container is mis-replicated after taking account of + * pending replicas, which are schedule to be created. + * @return true is mis-replicated after pending. + */ + public boolean isMisReplicatedAfterPending() { + return isMisReplicatedAfterPending; + } + + /** + * Returns true if the under replication is only due to mis-replication. + * In other words, the container has enough replicas, but they do not meet + * the placement policy. + * @return true if the under-replication is only due to mis-replication + */ + public boolean isDueToMisReplication() { + return dueToMisReplication; + } + /** * Indicates whether a container has enough replicas to be read. For Ratis * at least one replia must be available. For EC, at least dataNum replicas diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyReplicationManager.java index 27c7a12e661c..b58140ac3167 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyReplicationManager.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hdds.scm.PlacementPolicy; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.RatisContainerReplicaCount; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/RatisContainerReplicaCount.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisContainerReplicaCount.java similarity index 76% rename from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/RatisContainerReplicaCount.java rename to hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisContainerReplicaCount.java index f25423d4ec47..577fc6004d0c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/RatisContainerReplicaCount.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisContainerReplicaCount.java @@ -15,10 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdds.scm.container; +package org.apache.hadoop.hdds.scm.container.replication; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaCount; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; import java.util.Set; @@ -243,11 +244,27 @@ private int missingReplicas() { */ @Override public boolean isSufficientlyReplicated() { - return missingReplicas() + inFlightDel <= 0; + return isSufficientlyReplicated(false); } /** - * Return true is the container is over replicated. Decommission and + * Return true if the container is sufficiently replicated. Decommissioning + * and Decommissioned containers are ignored in this check, assuming they will + * eventually be removed from the cluster. + * This check ignores inflight additions, if includePendingAdd is false, + * otherwise it will assume they complete ok. + * + * @return True if the container is sufficiently replicated and False + * otherwise. + */ + public boolean isSufficientlyReplicated(boolean includePendingAdd) { + // Positive for under-rep, negative for over-rep + int delta = redundancyDelta(true, includePendingAdd); + return delta <= 0; + } + + /** + * Return true if the container is over replicated. Decommission and * maintenance containers are ignored for this check. * The check ignores inflight additions, as they may fail, but it does * consider inflight deletes, as they would reduce the over replication when @@ -257,7 +274,67 @@ public boolean isSufficientlyReplicated() { */ @Override public boolean isOverReplicated() { - return missingReplicas() + inFlightDel < 0; + return isOverReplicated(true); + } + + /** + * Return true if the container is over replicated. Decommission and + * maintenance containers are ignored for this check. + * The check ignores inflight additions, as they may fail, but it does + * consider inflight deletes if includePendingDelete is true. + * + * @return True if the container is over replicated, false otherwise. + */ + public boolean isOverReplicated(boolean includePendingDelete) { + return getExcessRedundancy(includePendingDelete) > 0; + } + + /** + * @return Return Excess Redundancy replica nums. + */ + public int getExcessRedundancy(boolean includePendingDelete) { + int excessRedundancy = redundancyDelta(includePendingDelete, false); + if (excessRedundancy >= 0) { + // either perfectly replicated or under replicated + return 0; + } + return -excessRedundancy; + } + + /** + * Return the delta from the expected number of replicas, optionally + * considering inflight add and deletes. + * @param includePendingDelete + * @param includePendingAdd + * @return zero if perfectly replicated, a negative value for over replication + * and a positive value for under replication. The magnitude of the + * return value indicates how many replias the container is over or + * under replicated by. + */ + private int redundancyDelta(boolean includePendingDelete, + boolean includePendingAdd) { + int excessRedundancy = missingReplicas(); + if (includePendingDelete) { + excessRedundancy += inFlightDel; + } + if (includePendingAdd) { + excessRedundancy -= inFlightAdd; + } + return excessRedundancy; + } + + /** + * How many more replicas can be lost before the container is + * unreadable, assuming any infligh deletes will complete. For containers + * which are under-replicated due to decommission + * or maintenance only, the remaining redundancy will include those + * decommissioning or maintenance replicas, as they are technically still + * available until the datanode processes are stopped. + * @return Count of remaining redundant replicas. + */ + public int getRemainingRedundancy() { + return Math.max(0, + healthyCount + decommissionCount + maintenanceCount - inFlightDel - 1); } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java index 05300d6a08af..77142848b692 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hdds.scm.container.replication.health.HealthCheck; import org.apache.hadoop.hdds.scm.container.replication.health.OpenContainerHandler; import org.apache.hadoop.hdds.scm.container.replication.health.QuasiClosedContainerHandler; +import org.apache.hadoop.hdds.scm.container.replication.health.RatisReplicationCheckHandler; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.ha.SCMService; @@ -74,6 +75,7 @@ import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE; import static org.apache.hadoop.hdds.conf.ConfigTag.SCM; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.EC; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS; /** * Replication Manager (RM) is the one which is responsible for making sure @@ -144,12 +146,14 @@ public class ReplicationManager implements SCMService { private final Clock clock; private final ContainerReplicaPendingOps containerReplicaPendingOps; private final ECReplicationCheckHandler ecReplicationCheckHandler; + private final RatisReplicationCheckHandler ratisReplicationCheckHandler; private final EventPublisher eventPublisher; private final ReentrantLock lock = new ReentrantLock(); private ReplicationQueue replicationQueue; private final ECUnderReplicationHandler ecUnderReplicationHandler; private final ECOverReplicationHandler ecOverReplicationHandler; private final int maintenanceRedundancy; + private final int ratisMaintenanceMinReplicas; private Thread underReplicatedProcessorThread; private Thread overReplicatedProcessorThread; private final UnderReplicatedProcessor underReplicatedProcessor; @@ -190,9 +194,13 @@ public ReplicationManager(final ConfigurationSource conf, this.containerReplicaPendingOps = replicaPendingOps; this.legacyReplicationManager = legacyReplicationManager; this.ecReplicationCheckHandler = new ECReplicationCheckHandler(); + this.ratisReplicationCheckHandler = + new RatisReplicationCheckHandler(containerPlacement); this.nodeManager = nodeManager; this.replicationQueue = new ReplicationQueue(); this.maintenanceRedundancy = rmConf.maintenanceRemainingRedundancy; + this.ratisMaintenanceMinReplicas = rmConf.getMaintenanceReplicaMinimum(); + ecUnderReplicationHandler = new ECUnderReplicationHandler( ecReplicationCheckHandler, containerPlacement, conf, nodeManager); ecOverReplicationHandler = @@ -212,7 +220,8 @@ public ReplicationManager(final ConfigurationSource conf, .addNext(new ClosingContainerHandler(this)) .addNext(new QuasiClosedContainerHandler(this)) .addNext(new ClosedWithMismatchedReplicasHandler(this)) - .addNext(ecReplicationCheckHandler); + .addNext(ecReplicationCheckHandler) + .addNext(ratisReplicationCheckHandler); start(); } @@ -466,10 +475,16 @@ protected void processContainer(ContainerInfo containerInfo, List pendingOps = containerReplicaPendingOps.getPendingOps(containerID); + // There is a different config for EC and Ratis maintenance + // minimum replicas, so we must pass through the correct one. + int maintRedundancy = maintenanceRedundancy; + if (containerInfo.getReplicationType() == RATIS) { + maintRedundancy = ratisMaintenanceMinReplicas; + } ContainerCheckRequest checkRequest = new ContainerCheckRequest.Builder() .setContainerInfo(containerInfo) .setContainerReplicas(replicas) - .setMaintenanceRedundancy(maintenanceRedundancy) + .setMaintenanceRedundancy(maintRedundancy) .setReport(report) .setPendingOps(pendingOps) .setReplicationQueue(repQueue) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java new file mode 100644 index 000000000000..cb53c59e083b --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdds.scm.container.replication.health; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.ContainerPlacementStatus; +import org.apache.hadoop.hdds.scm.PlacementPolicy; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport; +import org.apache.hadoop.hdds.scm.container.replication.ContainerCheckRequest; +import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult; +import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaOp; +import org.apache.hadoop.hdds.scm.container.replication.RatisContainerReplicaCount; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS; + +/** + * Class to determine the health state of a Ratis Container. Given the container + * and current replica details, along with replicas pending add and delete, + * this class will return a ContainerHealthResult indicating if the container + * is healthy, or under / over replicated etc. + */ +public class RatisReplicationCheckHandler extends AbstractCheck { + public static final Logger LOG = + LoggerFactory.getLogger(RatisReplicationCheckHandler.class); + + /** + * PlacementPolicy which is used to identify where a container + * should be replicated. + */ + private final PlacementPolicy ratisContainerPlacement; + + public RatisReplicationCheckHandler(PlacementPolicy containerPlacement) { + this.ratisContainerPlacement = containerPlacement; + } + + @Override + public boolean handle(ContainerCheckRequest request) { + if (request.getContainerInfo().getReplicationType() != RATIS) { + // This handler is only for Ratis containers. + return false; + } + ReplicationManagerReport report = request.getReport(); + ContainerInfo container = request.getContainerInfo(); + ContainerHealthResult health = checkHealth(request); + if (health.getHealthState() == ContainerHealthResult.HealthState.HEALTHY) { + // If the container is healthy, there is nothing else to do in this + // handler so return as unhandled so any further handlers will be tried. + return false; + } + if (health.getHealthState() + == ContainerHealthResult.HealthState.UNDER_REPLICATED) { + report.incrementAndSample( + ReplicationManagerReport.HealthState.UNDER_REPLICATED, + container.containerID()); + ContainerHealthResult.UnderReplicatedHealthResult underHealth + = ((ContainerHealthResult.UnderReplicatedHealthResult) health); + if (underHealth.isUnrecoverable()) { + report.incrementAndSample(ReplicationManagerReport.HealthState.MISSING, + container.containerID()); + } + if (underHealth.isMisReplicated()) { + report.incrementAndSample( + ReplicationManagerReport.HealthState.MIS_REPLICATED, + container.containerID()); + } + // TODO - if it is unrecoverable, should we return false to other + // handlers can be tried? + if (!underHealth.isUnrecoverable() && + (underHealth.isMisReplicatedAfterPending() || + !underHealth.isSufficientlyReplicatedAfterPending())) { + request.getReplicationQueue().enqueue(underHealth); + } + return true; + } + + if (health.getHealthState() + == ContainerHealthResult.HealthState.OVER_REPLICATED) { + report.incrementAndSample( + ReplicationManagerReport.HealthState.OVER_REPLICATED, + container.containerID()); + ContainerHealthResult.OverReplicatedHealthResult overHealth + = ((ContainerHealthResult.OverReplicatedHealthResult) health); + if (!overHealth.isSufficientlyReplicatedAfterPending()) { + request.getReplicationQueue().enqueue(overHealth); + } + return true; + } + return false; + } + + public ContainerHealthResult checkHealth(ContainerCheckRequest request) { + ContainerInfo container = request.getContainerInfo(); + Set replicas = request.getContainerReplicas(); + List replicaPendingOps = request.getPendingOps(); + // Note that this setting is minReplicasForMaintenance. For EC the variable + // is defined as remainingRedundancy which is subtly different. + int minReplicasForMaintenance = request.getMaintenanceRedundancy(); + int pendingAdd = 0; + int pendingDelete = 0; + for (ContainerReplicaOp op : replicaPendingOps) { + if (op.getOpType() == ContainerReplicaOp.PendingOpType.ADD) { + pendingAdd++; + } else if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) { + pendingDelete++; + } + } + int requiredNodes = container.getReplicationConfig().getRequiredNodes(); + + // RatisContainerReplicaCount uses the minReplicasForMaintenance rather + // than remainingRedundancy which ECContainerReplicaCount uses. + RatisContainerReplicaCount replicaCount = + new RatisContainerReplicaCount(container, replicas, pendingAdd, + pendingDelete, requiredNodes, minReplicasForMaintenance); + + ContainerPlacementStatus placementStatus = + getPlacementStatus(replicas, requiredNodes, Collections.emptyList()); + + ContainerPlacementStatus placementStatusWithPending = placementStatus; + if (replicaPendingOps.size() > 0) { + placementStatusWithPending = + getPlacementStatus(replicas, requiredNodes, replicaPendingOps); + } + boolean sufficientlyReplicated + = replicaCount.isSufficientlyReplicated(false); + boolean isPolicySatisfied = placementStatus.isPolicySatisfied(); + if (!sufficientlyReplicated || !isPolicySatisfied) { + ContainerHealthResult.UnderReplicatedHealthResult result = + new ContainerHealthResult.UnderReplicatedHealthResult( + container, replicaCount.getRemainingRedundancy(), + isPolicySatisfied + && replicas.size() - pendingDelete >= requiredNodes, + replicaCount.isSufficientlyReplicated(true), + replicaCount.isUnrecoverable()); + result.setMisReplicated(!isPolicySatisfied) + .setMisReplicatedAfterPending( + !placementStatusWithPending.isPolicySatisfied()) + .setDueToMisReplication( + !isPolicySatisfied && replicaCount.isSufficientlyReplicated()); + return result; + } + + boolean isOverReplicated = replicaCount.isOverReplicated(false); + if (isOverReplicated) { + boolean repOkWithPending = !replicaCount.isOverReplicated(true); + return new ContainerHealthResult.OverReplicatedHealthResult( + container, replicaCount.getExcessRedundancy(false), repOkWithPending); + } + // No issues detected, just return healthy. + return new ContainerHealthResult.HealthyResult(container); + } + + /** + * Given a set of ContainerReplica, transform it to a list of DatanodeDetails + * and then check if the list meets the container placement policy. + * @param replicas List of containerReplica + * @param replicationFactor Expected Replication Factor of the containe + * @return ContainerPlacementStatus indicating if the policy is met or not + */ + private ContainerPlacementStatus getPlacementStatus( + Set replicas, int replicationFactor, + List pendingOps) { + + Set replicaDns = replicas.stream() + .map(ContainerReplica::getDatanodeDetails) + .collect(Collectors.toSet()); + for (ContainerReplicaOp op : pendingOps) { + if (op.getOpType() == ContainerReplicaOp.PendingOpType.ADD) { + replicaDns.add(op.getTarget()); + } + if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) { + replicaDns.remove(op.getTarget()); + } + } + return ratisContainerPlacement.validateContainerPlacement( + new ArrayList<>(replicaDns), replicationFactor); + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisContainerReplicaCount.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisContainerReplicaCount.java index 5ceaec39bfca..7d4947dd64d6 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisContainerReplicaCount.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisContainerReplicaCount.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; -import org.apache.hadoop.hdds.scm.container.RatisContainerReplicaCount; +import org.junit.Assert; import org.junit.jupiter.api.Test; import java.util.HashSet; @@ -433,6 +433,62 @@ void testContainerWithNoReplicasIsMissing() { assertFalse(rcnt.isSufficientlyReplicated()); } + @Test + void testOverReplicatedWithAndWithoutPending() { + Set replica = registerNodes(IN_SERVICE, IN_SERVICE, + IN_SERVICE, IN_SERVICE, IN_SERVICE); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + RatisContainerReplicaCount rcnt = + new RatisContainerReplicaCount(container, replica, 0, 2, 3, 2); + assertTrue(rcnt.isOverReplicated(false)); + assertFalse(rcnt.isOverReplicated(true)); + assertEquals(2, rcnt.getExcessRedundancy(false)); + assertEquals(0, rcnt.getExcessRedundancy(true)); + } + + @Test + void testRemainingRedundancy() { + Set replica = registerNodes(IN_SERVICE, IN_SERVICE, + IN_SERVICE, IN_SERVICE); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + RatisContainerReplicaCount rcnt = + new RatisContainerReplicaCount(container, replica, 0, 1, 3, 2); + Assert.assertEquals(2, rcnt.getRemainingRedundancy()); + replica = registerNodes(IN_SERVICE); + rcnt = + new RatisContainerReplicaCount(container, replica, 0, 0, 3, 2); + Assert.assertEquals(0, rcnt.getRemainingRedundancy()); + rcnt = + new RatisContainerReplicaCount(container, replica, 0, 1, 3, 2); + Assert.assertEquals(0, rcnt.getRemainingRedundancy()); + } + + @Test + void testSufficientlyReplicatedWithAndWithoutPending() { + Set replica = registerNodes(IN_SERVICE, IN_SERVICE); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + RatisContainerReplicaCount rcnt = + new RatisContainerReplicaCount(container, replica, 0, 0, 3, 2); + assertFalse(rcnt.isSufficientlyReplicated(true)); + assertFalse(rcnt.isSufficientlyReplicated(false)); + + rcnt = + new RatisContainerReplicaCount(container, replica, 1, 0, 3, 2); + assertTrue(rcnt.isSufficientlyReplicated(true)); + assertFalse(rcnt.isSufficientlyReplicated(false)); + + replica = registerNodes(IN_SERVICE, IN_SERVICE, IN_SERVICE); + rcnt = + new RatisContainerReplicaCount(container, replica, 0, 1, 3, 2); + assertFalse(rcnt.isSufficientlyReplicated(false)); + assertFalse(rcnt.isSufficientlyReplicated(true)); + rcnt = + new RatisContainerReplicaCount(container, replica, 1, 1, 3, 2); + assertFalse(rcnt.isSufficientlyReplicated(false)); + assertTrue(rcnt.isSufficientlyReplicated(true)); + + } + private void validate(RatisContainerReplicaCount rcnt, boolean sufficientlyReplicated, int replicaDelta, boolean overReplicated) { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java new file mode 100644 index 000000000000..d8dd69284b08 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java @@ -0,0 +1,560 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdds.scm.container.replication.health; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; +import org.apache.hadoop.hdds.scm.PlacementPolicy; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport; +import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementStatusDefault; +import org.apache.hadoop.hdds.scm.container.replication.ContainerCheckRequest; +import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult; +import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult.HealthState; +import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult.OverReplicatedHealthResult; +import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult.UnderReplicatedHealthResult; +import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaOp; +import org.apache.hadoop.hdds.scm.container.replication.ReplicationQueue; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONED; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_MAINTENANCE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaOp.PendingOpType.ADD; +import static org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaOp.PendingOpType.DELETE; +import static org.apache.hadoop.hdds.scm.container.replication.ReplicationTestUtil.createContainerInfo; +import static org.apache.hadoop.hdds.scm.container.replication.ReplicationTestUtil.createReplicas; + +/** + * Tests for the RatisReplicationCheckHandler class. + */ +public class TestRatisReplicationCheckHandler { + + private RatisReplicationCheckHandler healthCheck; + private ReplicationConfig repConfig; + private PlacementPolicy containerPlacementPolicy; + private ReplicationQueue repQueue; + private ContainerCheckRequest.Builder requestBuilder; + private ReplicationManagerReport report; + private int maintenanceRedundancy = 2; + + @Before + public void setup() throws IOException { + containerPlacementPolicy = Mockito.mock(PlacementPolicy.class); + Mockito.when(containerPlacementPolicy.validateContainerPlacement( + Mockito.any(), + Mockito.anyInt() + )).thenAnswer(invocation -> + new ContainerPlacementStatusDefault(2, 2, 3)); + healthCheck = new RatisReplicationCheckHandler(containerPlacementPolicy); + repConfig = RatisReplicationConfig.getInstance(THREE); + repQueue = new ReplicationQueue(); + report = new ReplicationManagerReport(); + requestBuilder = new ContainerCheckRequest.Builder() + .setReplicationQueue(repQueue) + .setMaintenanceRedundancy(maintenanceRedundancy) + .setPendingOps(Collections.emptyList()) + .setReport(report); + } + + @Test + public void testReturnFalseForNonRatis() { + ContainerInfo container = + createContainerInfo(new ECReplicationConfig(3, 2)); + Set replicas = + createReplicas(container.containerID(), 1, 2, 3, 4); + + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + Assert.assertFalse(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(0, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + } + + @Test + public void testHealthyContainerIsHealthy() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas + = createReplicas(container.containerID(), 0, 0, 0); + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + ContainerHealthResult result = + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.HEALTHY, result.getHealthState()); + + Assert.assertFalse(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(0, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + } + + @Test + public void testUnderReplicatedContainerIsUnderReplicated() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas + = createReplicas(container.containerID(), 0, 0); + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + UnderReplicatedHealthResult result = (UnderReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.UNDER_REPLICATED, result.getHealthState()); + Assert.assertEquals(1, result.getRemainingRedundancy()); + Assert.assertFalse(result.isSufficientlyReplicatedAfterPending()); + Assert.assertFalse(result.underReplicatedDueToDecommission()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(1, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + } + + @Test + public void testUnderReplicatedContainerDueToPendingDelete() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas + = createReplicas(container.containerID(), 0, 0, 0); + List pending = new ArrayList<>(); + pending.add(ContainerReplicaOp.create( + DELETE, MockDatanodeDetails.randomDatanodeDetails(), 0)); + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container) + .setPendingOps(pending); + UnderReplicatedHealthResult result = (UnderReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.UNDER_REPLICATED, result.getHealthState()); + Assert.assertEquals(1, result.getRemainingRedundancy()); + Assert.assertFalse(result.isSufficientlyReplicatedAfterPending()); + Assert.assertFalse(result.underReplicatedDueToDecommission()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(1, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + } + + @Test + public void testUnderReplicatedContainerFixedWithPending() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas + = createReplicas(container.containerID(), 0, 0); + List pending = new ArrayList<>(); + pending.add(ContainerReplicaOp.create( + ADD, MockDatanodeDetails.randomDatanodeDetails(), 0)); + requestBuilder.setContainerReplicas(replicas) + .setPendingOps(pending) + .setContainerInfo(container); + UnderReplicatedHealthResult result = (UnderReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.UNDER_REPLICATED, result.getHealthState()); + Assert.assertEquals(1, result.getRemainingRedundancy()); + Assert.assertTrue(result.isSufficientlyReplicatedAfterPending()); + Assert.assertFalse(result.underReplicatedDueToDecommission()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + // Fixed with pending, so nothing added to the queue + Assert.assertEquals(0, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + // Still under replicated until the pending complete + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + } + + @Test + public void testUnderReplicatedDueToDecommission() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas = createReplicas(container.containerID(), + Pair.of(IN_SERVICE, 0), Pair.of(DECOMMISSIONING, 0), + Pair.of(DECOMMISSIONED, 0)); + + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + UnderReplicatedHealthResult result = (UnderReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.UNDER_REPLICATED, result.getHealthState()); + Assert.assertEquals(2, result.getRemainingRedundancy()); + Assert.assertFalse(result.isSufficientlyReplicatedAfterPending()); + Assert.assertTrue(result.underReplicatedDueToDecommission()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(1, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + } + + @Test + public void testUnderReplicatedDueToDecommissionFixedWithPending() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas = createReplicas(container.containerID(), + Pair.of(IN_SERVICE, 0), Pair.of(IN_SERVICE, 0), + Pair.of(DECOMMISSIONED, 0)); + List pending = new ArrayList<>(); + pending.add(ContainerReplicaOp.create( + ADD, MockDatanodeDetails.randomDatanodeDetails(), 0)); + + requestBuilder.setContainerReplicas(replicas) + .setPendingOps(pending) + .setContainerInfo(container); + UnderReplicatedHealthResult result = (UnderReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.UNDER_REPLICATED, result.getHealthState()); + Assert.assertEquals(2, result.getRemainingRedundancy()); + Assert.assertTrue(result.isSufficientlyReplicatedAfterPending()); + Assert.assertTrue(result.underReplicatedDueToDecommission()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + // Nothing queued as inflight replicas will fix it. + Assert.assertEquals(0, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + // Still under replicated in the report until pending complete + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + } + + @Test + public void testUnderReplicatedDueToDecommissionAndMissing() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas = createReplicas(container.containerID(), + Pair.of(IN_SERVICE, 0), Pair.of(DECOMMISSIONED, 0)); + List pending = new ArrayList<>(); + pending.add(ContainerReplicaOp.create( + ADD, MockDatanodeDetails.randomDatanodeDetails(), 0)); + + requestBuilder.setContainerReplicas(replicas) + .setPendingOps(pending) + .setContainerInfo(container); + UnderReplicatedHealthResult result = (UnderReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.UNDER_REPLICATED, result.getHealthState()); + Assert.assertEquals(1, result.getRemainingRedundancy()); + Assert.assertFalse(result.isSufficientlyReplicatedAfterPending()); + Assert.assertFalse(result.underReplicatedDueToDecommission()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(1, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + } + + @Test + public void testUnderReplicatedAndUnrecoverable() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas = Collections.EMPTY_SET; + + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + UnderReplicatedHealthResult result = (UnderReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.UNDER_REPLICATED, result.getHealthState()); + Assert.assertEquals(0, result.getRemainingRedundancy()); + Assert.assertFalse(result.isSufficientlyReplicatedAfterPending()); + Assert.assertFalse(result.underReplicatedDueToDecommission()); + Assert.assertTrue(result.isUnrecoverable()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + // Unrecoverable, so not added to the queue. + Assert.assertEquals(0, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.MISSING)); + } + + @Test + public void testOverReplicatedContainer() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas = createReplicas(container.containerID(), + Pair.of(IN_SERVICE, 0), Pair.of(IN_SERVICE, 0), + Pair.of(IN_SERVICE, 0), Pair.of(IN_SERVICE, 0), + Pair.of(IN_SERVICE, 0), + Pair.of(IN_SERVICE, 0), Pair.of(IN_SERVICE, 0)); + + List pending = new ArrayList<>(); + pending.add(ContainerReplicaOp.create( + DELETE, MockDatanodeDetails.randomDatanodeDetails(), 0)); + pending.add(ContainerReplicaOp.create( + DELETE, MockDatanodeDetails.randomDatanodeDetails(), 0)); + + requestBuilder.setContainerReplicas(replicas) + .setPendingOps(pending) + .setContainerInfo(container); + OverReplicatedHealthResult result = (OverReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.OVER_REPLICATED, result.getHealthState()); + Assert.assertEquals(4, result.getExcessRedundancy()); + Assert.assertFalse(result.isSufficientlyReplicatedAfterPending()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(0, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(1, repQueue.overReplicatedQueueSize()); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.OVER_REPLICATED)); + } + + @Test + public void testOverReplicatedContainerFixedByPending() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas = createReplicas(container.containerID(), + Pair.of(IN_SERVICE, 0), Pair.of(IN_SERVICE, 0), + Pair.of(IN_SERVICE, 0), Pair.of(IN_SERVICE, 0)); + + List pending = new ArrayList<>(); + pending.add(ContainerReplicaOp.create( + DELETE, MockDatanodeDetails.randomDatanodeDetails(), 0)); + + requestBuilder.setContainerReplicas(replicas) + .setPendingOps(pending) + .setContainerInfo(container); + OverReplicatedHealthResult result = (OverReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.OVER_REPLICATED, result.getHealthState()); + Assert.assertEquals(1, result.getExcessRedundancy()); + Assert.assertTrue(result.isSufficientlyReplicatedAfterPending()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(0, repQueue.underReplicatedQueueSize()); + // Fixed by pending so nothing queued. + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + // Still over replicated, so the report should contain it + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.OVER_REPLICATED)); + } + + @Test + public void testOverReplicatedContainerWithMaintenance() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas = createReplicas(container.containerID(), + Pair.of(IN_SERVICE, 0), Pair.of(IN_SERVICE, 0), + Pair.of(IN_SERVICE, 0), Pair.of(IN_SERVICE, 0), + Pair.of(IN_MAINTENANCE, 0), Pair.of(DECOMMISSIONED, 0)); + + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + OverReplicatedHealthResult result = (OverReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.OVER_REPLICATED, result.getHealthState()); + Assert.assertEquals(1, result.getExcessRedundancy()); + Assert.assertFalse(result.isSufficientlyReplicatedAfterPending()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(0, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(1, repQueue.overReplicatedQueueSize()); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.OVER_REPLICATED)); + } + + @Test + public void testOverReplicatedContainerDueToMaintenanceIsHealthy() { + ContainerInfo container = createContainerInfo(repConfig); + Set replicas = createReplicas(container.containerID(), + Pair.of(IN_SERVICE, 0), Pair.of(IN_SERVICE, 0), + Pair.of(IN_SERVICE, 0), Pair.of(IN_MAINTENANCE, 0), + Pair.of(IN_MAINTENANCE, 0)); + + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + ContainerHealthResult result = + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.HEALTHY, result.getHealthState()); + + Assert.assertFalse(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(0, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + Assert.assertEquals(0, report.getStat( + ReplicationManagerReport.HealthState.OVER_REPLICATED)); + Assert.assertEquals(0, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + } + + @Test + public void testUnderReplicatedWithMisReplication() { + Mockito.when(containerPlacementPolicy.validateContainerPlacement( + Mockito.any(), + Mockito.anyInt() + )).thenAnswer(invocation -> + new ContainerPlacementStatusDefault(1, 2, 3)); + + ContainerInfo container = createContainerInfo(repConfig); + Set replicas + = createReplicas(container.containerID(), 0, 0); + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + UnderReplicatedHealthResult result = (UnderReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.UNDER_REPLICATED, result.getHealthState()); + Assert.assertEquals(1, result.getRemainingRedundancy()); + Assert.assertFalse(result.isSufficientlyReplicatedAfterPending()); + Assert.assertFalse(result.underReplicatedDueToDecommission()); + Assert.assertTrue(result.isMisReplicated()); + Assert.assertTrue(result.isMisReplicatedAfterPending()); + Assert.assertFalse(result.isDueToMisReplication()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(1, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.MIS_REPLICATED)); + } + + @Test + public void testUnderReplicatedWithMisReplicationFixedByPending() { + Mockito.when(containerPlacementPolicy.validateContainerPlacement( + Mockito.any(), + Mockito.anyInt() + )).thenAnswer(invocation -> { + List dns = invocation.getArgument(0); + // If the number of DNs is 3 or less make it be mis-replicated + if (dns.size() <= 3) { + return new ContainerPlacementStatusDefault(1, 2, 3); + } else { + return new ContainerPlacementStatusDefault(2, 2, 3); + } + }); + + ContainerInfo container = createContainerInfo(repConfig); + Set replicas + = createReplicas(container.containerID(), 0, 0); + + List pending = new ArrayList<>(); + pending.add(ContainerReplicaOp.create( + ADD, MockDatanodeDetails.randomDatanodeDetails(), 0)); + pending.add(ContainerReplicaOp.create( + ADD, MockDatanodeDetails.randomDatanodeDetails(), 0)); + + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container) + .setPendingOps(pending); + UnderReplicatedHealthResult result = (UnderReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.UNDER_REPLICATED, result.getHealthState()); + Assert.assertEquals(1, result.getRemainingRedundancy()); + Assert.assertTrue(result.isSufficientlyReplicatedAfterPending()); + Assert.assertFalse(result.underReplicatedDueToDecommission()); + Assert.assertTrue(result.isMisReplicated()); + Assert.assertFalse(result.isMisReplicatedAfterPending()); + Assert.assertFalse(result.isDueToMisReplication()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(0, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.MIS_REPLICATED)); + } + + @Test + public void testUnderReplicatedOnlyDueToMisReplication() { + Mockito.when(containerPlacementPolicy.validateContainerPlacement( + Mockito.any(), + Mockito.anyInt() + )).thenAnswer(invocation -> + new ContainerPlacementStatusDefault(1, 2, 3)); + + ContainerInfo container = createContainerInfo(repConfig); + Set replicas + = createReplicas(container.containerID(), 0, 0, 0); + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container); + UnderReplicatedHealthResult result = (UnderReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.UNDER_REPLICATED, result.getHealthState()); + Assert.assertEquals(2, result.getRemainingRedundancy()); + Assert.assertTrue(result.isSufficientlyReplicatedAfterPending()); + Assert.assertFalse(result.underReplicatedDueToDecommission()); + Assert.assertTrue(result.isMisReplicated()); + Assert.assertTrue(result.isMisReplicatedAfterPending()); + Assert.assertTrue(result.isDueToMisReplication()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(1, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.MIS_REPLICATED)); + } + + @Test + public void testUnderReplicatedOnlyDueToMisReplicationFixByPending() { + Mockito.when(containerPlacementPolicy.validateContainerPlacement( + Mockito.any(), + Mockito.anyInt() + )).thenAnswer(invocation -> { + List dns = invocation.getArgument(0); + // If the number of DNs is 3 or less make it be mis-replicated + if (dns.size() <= 3) { + return new ContainerPlacementStatusDefault(1, 2, 3); + } else { + return new ContainerPlacementStatusDefault(2, 2, 3); + } + }); + + ContainerInfo container = createContainerInfo(repConfig); + Set replicas + = createReplicas(container.containerID(), 0, 0, 0); + + List pending = new ArrayList<>(); + pending.add(ContainerReplicaOp.create( + ADD, MockDatanodeDetails.randomDatanodeDetails(), 0)); + pending.add(ContainerReplicaOp.create( + ADD, MockDatanodeDetails.randomDatanodeDetails(), 0)); + + requestBuilder.setContainerReplicas(replicas) + .setContainerInfo(container) + .setPendingOps(pending); + UnderReplicatedHealthResult result = (UnderReplicatedHealthResult) + healthCheck.checkHealth(requestBuilder.build()); + Assert.assertEquals(HealthState.UNDER_REPLICATED, result.getHealthState()); + Assert.assertEquals(2, result.getRemainingRedundancy()); + Assert.assertTrue(result.isSufficientlyReplicatedAfterPending()); + Assert.assertFalse(result.underReplicatedDueToDecommission()); + Assert.assertTrue(result.isMisReplicated()); + Assert.assertFalse(result.isMisReplicatedAfterPending()); + Assert.assertTrue(result.isDueToMisReplication()); + + Assert.assertTrue(healthCheck.handle(requestBuilder.build())); + Assert.assertEquals(0, repQueue.underReplicatedQueueSize()); + Assert.assertEquals(0, repQueue.overReplicatedQueueSize()); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.UNDER_REPLICATED)); + Assert.assertEquals(1, report.getStat( + ReplicationManagerReport.HealthState.MIS_REPLICATED)); + } + +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java index 8afe2a1810ba..095d137a5a7e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.RatisContainerReplicaCount; +import org.apache.hadoop.hdds.scm.container.replication.RatisContainerReplicaCount; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.container.ContainerReplica; From 977ab59e11b0b05b85a0fc945ace336dcfa4b604 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Wed, 19 Oct 2022 09:53:30 +0100 Subject: [PATCH 04/48] HDDS-7341. EC: Close pipelines with unregistered nodes (#3850) --- .../scm/pipeline/PipelineManagerImpl.java | 40 +++++++++++++++--- .../scm/pipeline/TestPipelineManagerImpl.java | 41 +++++++++++++++---- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java index aaa3088f9ab3..044cbc016689 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java @@ -509,22 +509,52 @@ public void scrubPipelines() throws IOException, TimeoutException { if (p.getPipelineState() == Pipeline.PipelineState.ALLOCATED && (currentTime.toEpochMilli() - p.getCreationTimestamp() .toEpochMilli() >= pipelineScrubTimeoutInMills)) { - LOG.info("Scrubbing pipeline: id: " + p.getId().toString() + - " since it stays at ALLOCATED stage for " + + LOG.info("Scrubbing pipeline: id: {} since it stays at ALLOCATED " + + "stage for {} mins.", p.getId(), Duration.between(currentTime, p.getCreationTimestamp()) - .toMinutes() + " mins."); + .toMinutes()); closePipeline(p, false); } // scrub pipelines who stay CLOSED for too long. if (p.getPipelineState() == Pipeline.PipelineState.CLOSED) { - LOG.info("Scrubbing pipeline: id: " + p.getId().toString() + - " since it stays at CLOSED stage."); + LOG.info("Scrubbing pipeline: id: {} since it stays at CLOSED stage.", + p.getId()); closeContainersForPipeline(p.getId()); removePipeline(p); } + // If a datanode is stopped and then SCM is restarted, a pipeline can get + // stuck in an open state. For Ratis, provided some other DNs that were + // part of the open pipeline register to SCM after the restart, the Ratis + // pipeline close will get triggered by the DNs. For EC that will never + // happen, as the DNs are not aware of the pipeline. Therefore we should + // close any pipelines in the scrubber if they have nodes which are not + // registered + if (isOpenWithUnregisteredNodes(p)) { + LOG.info("Scrubbing pipeline: id: {} as it has unregistered nodes", + p.getId()); + closeContainersForPipeline(p.getId()); + closePipeline(p, true); + } } } + /** + * @param pipeline The pipeline to check + * @return True if the pipeline is open and contains unregistered nodes. False + * otherwise. + */ + private boolean isOpenWithUnregisteredNodes(Pipeline pipeline) { + if (!pipeline.isOpen()) { + return false; + } + for (DatanodeDetails dn : pipeline.getNodes()) { + if (nodeManager.getNodeByUuid(dn.getUuidString()) == null) { + return true; + } + } + return false; + } + /** * Schedules a fixed interval job to create pipelines. */ diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java index 8fa45f584233..1ed9b845ac25 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java @@ -20,6 +20,7 @@ import com.google.common.base.Supplier; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -29,12 +30,14 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.PipelineChoosePolicy; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; +import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom; import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.ha.SCMHADBTransactionBuffer; import org.apache.hadoop.hdds.scm.ha.SCMHADBTransactionBufferStub; @@ -125,6 +128,10 @@ public void init() throws Exception { GenericTestUtils.getRandomizedTempPath()); scm = HddsTestUtils.getScm(conf); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath()); + // Mock Node Manager is not able to correctly set up things for the EC + // placement policy (Rack Scatter), so just use the random one. + conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_EC_IMPL_KEY, + SCMContainerPlacementRandom.class.getName()); dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition()); nodeManager = new MockNodeManager(true, 20); maxPipelineCount = nodeManager.getNodeCount( @@ -151,7 +158,7 @@ private PipelineManagerImpl createPipelineManager(boolean isLeader) throws IOException { return PipelineManagerImpl.newPipelineManager(conf, SCMHAManagerStub.getInstance(isLeader), - new MockNodeManager(true, 20), + nodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), new EventQueue(), scmContext, @@ -163,7 +170,7 @@ private PipelineManagerImpl createPipelineManager( boolean isLeader, SCMHADBTransactionBuffer buffer) throws IOException { return PipelineManagerImpl.newPipelineManager(conf, SCMHAManagerStub.getInstance(isLeader, buffer), - new MockNodeManager(true, 20), + nodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), new EventQueue(), SCMContext.emptyContext(), @@ -341,7 +348,6 @@ public void testDeactivatePipelineShouldFailOnFollower() throws Exception { @Test public void testRemovePipeline() throws Exception { PipelineManagerImpl pipelineManager = createPipelineManager(true); - pipelineManager.setScmContext(scmContext); // Create a pipeline Pipeline pipeline = pipelineManager.createPipeline( RatisReplicationConfig.getInstance(ReplicationFactor.THREE)); @@ -391,7 +397,6 @@ public void testRemovePipeline() throws Exception { @Test public void testClosePipelineShouldFailOnFollower() throws Exception { PipelineManagerImpl pipelineManager = createPipelineManager(true); - pipelineManager.setScmContext(scmContext); Pipeline pipeline = pipelineManager.createPipeline( RatisReplicationConfig.getInstance(ReplicationFactor.THREE)); Assertions.assertEquals(1, pipelineManager.getPipelines().size()); @@ -413,7 +418,6 @@ public void testClosePipelineShouldFailOnFollower() throws Exception { @Test public void testPipelineReport() throws Exception { PipelineManagerImpl pipelineManager = createPipelineManager(true); - pipelineManager.setScmContext(scmContext); SCMSafeModeManager scmSafeModeManager = new SCMSafeModeManager(conf, new ArrayList<>(), null, pipelineManager, new EventQueue(), serviceManager, scmContext); @@ -571,7 +575,6 @@ public void testScrubPipelines() throws Exception { OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT, 50, TimeUnit.SECONDS); PipelineManagerImpl pipelineManager = createPipelineManager(true); - pipelineManager.setScmContext(scmContext); Pipeline allocatedPipeline = pipelineManager .createPipeline(RatisReplicationConfig .getInstance(ReplicationFactor.THREE)); @@ -628,13 +631,36 @@ public void testScrubPipelines() throws Exception { pipelineManager.close(); } + @Test + public void testScrubOpenWithUnregisteredNodes() throws Exception { + PipelineManagerImpl pipelineManager = createPipelineManager(true); + Pipeline pipeline = pipelineManager + .createPipeline(new ECReplicationConfig(3, 2)); + pipelineManager.openPipeline(pipeline.getId()); + + // Scrubbing the pipelines should not affect this pipeline + pipelineManager.scrubPipelines(); + pipeline = pipelineManager.getPipeline(pipeline.getId()); + Assertions.assertEquals(Pipeline.PipelineState.OPEN, + pipeline.getPipelineState()); + + // Now, "unregister" one of the nodes in the pipeline + DatanodeDetails firstDN = nodeManager.getNodeByUuid( + pipeline.getNodes().get(0).getUuidString()); + nodeManager.getClusterNetworkTopologyMap().remove(firstDN); + + pipelineManager.scrubPipelines(); + pipeline = pipelineManager.getPipeline(pipeline.getId()); + Assertions.assertEquals(Pipeline.PipelineState.CLOSED, + pipeline.getPipelineState()); + } + @Test public void testScrubPipelinesShouldFailOnFollower() throws Exception { conf.setTimeDuration( OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT, 10, TimeUnit.SECONDS); PipelineManagerImpl pipelineManager = createPipelineManager(true); - pipelineManager.setScmContext(scmContext); Pipeline pipeline = pipelineManager .createPipeline(RatisReplicationConfig .getInstance(ReplicationFactor.THREE)); @@ -765,7 +791,6 @@ public void testPipelineCloseFlow() throws IOException, TimeoutException { GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer .captureLogs(LoggerFactory.getLogger(PipelineManagerImpl.class)); PipelineManagerImpl pipelineManager = createPipelineManager(true); - pipelineManager.setScmContext(scmContext); Pipeline pipeline = pipelineManager.createPipeline( RatisReplicationConfig .getInstance(HddsProtos.ReplicationFactor.THREE)); From d5dc65eaa0ddd06e29110489231b2d491ae5decf Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Wed, 19 Oct 2022 13:28:30 +0200 Subject: [PATCH 05/48] HDDS-7305. Fix Hadoop imports (#3822) --- hadoop-hdds/framework/pom.xml | 8 ++++++++ hadoop-hdds/hadoop-dependency-server/pom.xml | 4 ++++ hadoop-hdds/server-scm/pom.xml | 10 ++++++++++ .../hadoop/hdds/scm/pipeline/PipelineManagerImpl.java | 2 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 1 - 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml index 00272547ecc2..5707688626ed 100644 --- a/hadoop-hdds/framework/pom.xml +++ b/hadoop-hdds/framework/pom.xml @@ -59,6 +59,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.slf4j slf4j-reload4j + + org.apache.commons + commons-compress + + + org.apache.commons + commons-configuration2 + org.apache.logging.log4j log4j-core diff --git a/hadoop-hdds/hadoop-dependency-server/pom.xml b/hadoop-hdds/hadoop-dependency-server/pom.xml index 2aa1a5f0fcd2..ab3ae72e82e8 100644 --- a/hadoop-hdds/hadoop-dependency-server/pom.xml +++ b/hadoop-hdds/hadoop-dependency-server/pom.xml @@ -51,6 +51,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.zookeeper zookeeper + + org.apache.commons + * + org.codehaus.jackson jackson-mapper-asl diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml index 4756c56b9069..60feece1489a 100644 --- a/hadoop-hdds/server-scm/pom.xml +++ b/hadoop-hdds/server-scm/pom.xml @@ -73,6 +73,16 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> provided + + org.apache.commons + commons-math3 + test + + + org.apache.commons + commons-text + + org.apache.hadoop hadoop-hdfs-client diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java index 044cbc016689..d943dc21e2a3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.pipeline; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; @@ -44,7 +45,6 @@ import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; -import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.ratis.protocol.exceptions.NotLeaderException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt index 664d4bf39fb7..6147466757ed 100644 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -28,7 +28,6 @@ share/ozone/lib/commons-io.jar share/ozone/lib/commons-lang3.jar share/ozone/lib/commons-lang.jar share/ozone/lib/commons-logging.jar -share/ozone/lib/commons-math3.jar share/ozone/lib/commons-net.jar share/ozone/lib/commons-pool2.jar share/ozone/lib/commons-text.jar From d32e96c835c5642c71a60de9cc6be36bf487973b Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Wed, 19 Oct 2022 21:01:48 +0200 Subject: [PATCH 06/48] HDDS-7351. Use jackson-bom to ensure consistent Jackson version (#3856) --- pom.xml | 46 ++++++---------------------------------------- 1 file changed, 6 insertions(+), 40 deletions(-) diff --git a/pom.xml b/pom.xml index d4f621172b25..3d6b0316bc65 100644 --- a/pom.xml +++ b/pom.xml @@ -136,8 +136,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 2.34 - 2.13.4 - 2.13.4.2 + 2.13.4.20221013 1.6.0 @@ -1228,39 +1227,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 5.0.3 - com.fasterxml.jackson.core - jackson-core - ${jackson2.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson2.databind.version} - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson2.version} - - - com.fasterxml.jackson.module - jackson-module-jaxb-annotations - ${jackson2.version} - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - ${jackson2.version} - - - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - ${jackson2.version} - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - ${jackson2.version} + com.fasterxml.jackson + jackson-bom + ${jackson2-bom.version} + pom + import org.hamcrest @@ -1554,11 +1525,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs swagger-annotations ${swagger-annotations-version} - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - ${jackson2.version} - org.yaml snakeyaml From e45f9b8333a0ccd605bdc3d18aa36282d4ba5859 Mon Sep 17 00:00:00 2001 From: DaveTeng0 <109315747+DaveTeng0@users.noreply.github.com> Date: Wed, 19 Oct 2022 14:23:20 -0700 Subject: [PATCH 07/48] HDDS-7199. Implement new mix workload Read/Write Freon command which meets specific test requirements (#3754) --- .../main/smoketest/freon/read-write-key.robot | 53 ++++ .../org/apache/hadoop/ozone/freon/Freon.java | 2 + .../hadoop/ozone/freon/KeyGeneratorUtil.java | 50 ++++ .../freon/OzoneClientKeyReadWriteOps.java | 243 ++++++++++++++++++ .../ozone/freon/RangeKeysGenerator.java | 176 +++++++++++++ 5 files changed, 524 insertions(+) create mode 100644 hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java diff --git a/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot new file mode 100644 index 000000000000..25a3c4b9e9a8 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test freon read/write key commands +Resource ../ozone-lib/freon.robot +Test Timeout 5 minutes + +*** Variables *** +${PREFIX} ${EMPTY} + + +*** Test Cases *** +Pre-generate 100 keys of size 1 byte each to Ozone + ${result} = Execute ozone freon ork -n 1 -t 10 -r 100 --size 1 -v voltest -b buckettest -p performanceTest + +Read 10 keys from pre-generated keys + ${keysCount} = BuiltIn.Set Variable 10 + ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 -r 100 -v voltest -b buckettest -p performanceTest + Should contain ${result} Successful executions: ${keysCount} + +Read 10 keys' metadata from pre-generated keys + ${keysCount} = BuiltIn.Set Variable 10 + ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 -m -r 100 -v voltest -b buckettest -p performanceTest + Should contain ${result} Successful executions: ${keysCount} + +Write 10 keys of size 1 byte each from key index 0 to 99 + ${keysCount} = BuiltIn.Set Variable 10 + ${size} = BuiltIn.Set Variable 1 + ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 --percentage-read 0 --size ${size} -r 100 -v voltest -b buckettest -p performanceTest2 + Should contain ${result} Successful executions: ${keysCount} + ${keyName} = Execute echo -n '1' | md5sum | head -c 7 + ${result} = Execute ozone sh key info /voltest/buckettest/performanceTest2/${keyName} + Should contain ${result} \"dataSize\" : 1 + + +Run 90 % of read-key tasks and 10 % of write-key tasks for 10 keys from pre-generated keys + ${keysCount} = BuiltIn.Set Variable 10 + ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 --percentage-read 90 -r 100 -v voltest -b buckettest -p performanceTest + Should contain ${result} Successful executions: ${keysCount} + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java index b6bb6055eefe..dfb52ca9add6 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java @@ -68,6 +68,8 @@ OmBucketReadWriteFileOps.class, OmBucketReadWriteKeyOps.class, OmRPCLoadGenerator.class, + OzoneClientKeyReadWriteOps.class, + RangeKeysGenerator.class }, versionProvider = HddsVersionProvider.class, mixinStandardHelpOptions = true) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java new file mode 100644 index 000000000000..9773caa44070 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.freon; + +import org.apache.commons.codec.digest.DigestUtils; + +import java.util.function.Function; + +/** + * Utility class to generate key name from a given key index. + */ +public class KeyGeneratorUtil { + public static final String PURE_INDEX = "pureIndex"; + public static final String MD5 = "md5"; + public static final String FILE_DIR_SEPARATOR = "/"; + + public String generatePureIndexKeyName(int number) { + return String.valueOf(number); + } + public Function pureIndexKeyNameFunc() { + return number -> String.valueOf(number); + } + + public String generateMd5KeyName(int number) { + String encodedStr = DigestUtils.md5Hex(String.valueOf(number)); + return encodedStr.substring(0, 7); + } + + public Function md5KeyNameFunc() { + return number -> DigestUtils.md5Hex(String.valueOf(number)).substring(0, 7); + } + +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java new file mode 100644 index 000000000000..c4374f4eada6 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.freon; + + +import com.codahale.metrics.Timer; +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneKeyDetails; +import org.apache.hadoop.ozone.client.io.OzoneInputStream; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; + +import java.io.IOException; +import java.util.concurrent.Callable; +import java.util.concurrent.ThreadLocalRandom; +import java.util.HashMap; + +import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR; + +/** + * Ozone key generator/reader for performance test. + */ + +@CommandLine.Command(name = "ockrw", + aliases = "ozone-client-key-read-write-ops", + description = "Generate keys with a fixed name and ranges that can" + + " be written and read as sub-ranges from multiple clients.", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true, + showDefaultValues = true) +public class OzoneClientKeyReadWriteOps extends BaseFreonGenerator + implements Callable { + + @CommandLine.Option(names = {"-v", "--volume"}, + description = "Name of the volume which contains the test data. " + + "Will be created if missing.", + defaultValue = "ockrwvolume") + private String volumeName; + + @CommandLine.Option(names = {"-b", "--bucket"}, + description = "Name of the bucket which contains the test data.", + defaultValue = "ockrwbucket") + private String bucketName; + + @CommandLine.Option(names = {"-m", "--read-metadata-only"}, + description = "If only read key's metadata.", + defaultValue = "false") + private boolean readMetadataOnly; + + @CommandLine.Option(names = {"-s", "--start-index"}, + description = "Start index of keys of read/write operation." + + "This can allow adding keys incrementally or parallel from multiple" + + " clients. Example: Write keys 0-1000000 followed by " + + "keys 1000001-2000000.", + defaultValue = "0") + private int startIndex; + + @CommandLine.Option(names = {"-r", "--range"}, + description = "Range of read/write operations. This in co-ordination" + + " with --start-index can specify the range to read. " + + "Example: Read from --start-index 1000 and read --range 1000 keys.", + defaultValue = "1") + private int range; + + @CommandLine.Option(names = {"--size"}, + description = "Object size (in bytes) " + + "to read/write. If user sets a read size which is larger" + + " than the key size, it only reads bytes up to key size.", + defaultValue = "1") + private int objectSizeInBytes; + + @CommandLine.Option(names = {"--contiguous"}, + description = "By default, the keys are randomized lexically" + + " by calculating the md5 of the key name. If this option is set," + + " the keys are written lexically contiguously.", + defaultValue = "false") + private boolean keySorted; + + @CommandLine.Option(names = {"--percentage-read"}, + description = "Percentage of read tasks in mix workload." + + " The remainder of the percentage will writes to keys." + + " Example --percentage-read 90 will result in 10% writes.", + defaultValue = "100") + private int percentageRead; + + @CommandLine.Option( + names = "--om-service-id", + description = "OM Service ID" + ) + private String omServiceID = null; + + private Timer timer; + + private OzoneClient[] ozoneClients; + + private int clientCount; + + private byte[] keyContent; + + private static final Logger LOG = + LoggerFactory.getLogger(OzoneClientKeyReadWriteOps.class); + + /** + * Task type of read task, or write task. + */ + public enum TaskType { + READ_TASK, + WRITE_TASK + } + private KeyGeneratorUtil kg; + + + @Override + public Void call() throws Exception { + init(); + OzoneConfiguration ozoneConfiguration = createOzoneConfiguration(); + clientCount = getThreadNo(); + ozoneClients = new OzoneClient[clientCount]; + for (int i = 0; i < clientCount; i++) { + ozoneClients[i] = createOzoneClient(omServiceID, ozoneConfiguration); + } + + ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName); + + timer = getMetrics().timer("key-read-write"); + if (objectSizeInBytes >= 0) { + keyContent = RandomUtils.nextBytes(objectSizeInBytes); + } + if (kg == null) { + kg = new KeyGeneratorUtil(); + } + runTests(this::readWriteKeys); + + for (int i = 0; i < clientCount; i++) { + if (ozoneClients[i] != null) { + ozoneClients[i].close(); + } + } + return null; + } + + public void readWriteKeys(long counter) throws RuntimeException, IOException { + int clientIndex = (int)((counter) % clientCount); + TaskType taskType = decideReadOrWriteTask(); + String keyName = getKeyName(); + + timer.time(() -> { + try { + switch (taskType) { + case READ_TASK: + processReadTasks(keyName, ozoneClients[clientIndex]); + break; + case WRITE_TASK: + processWriteTasks(keyName, ozoneClients[clientIndex]); + break; + default: + break; + } + } catch (RuntimeException ex) { + LOG.error(ex.getMessage()); + throw ex; + } catch (IOException ex) { + LOG.error(ex.getMessage()); + throw new RuntimeException(ex.getMessage()); + } + + }); + } + + public void processReadTasks(String keyName, OzoneClient client) + throws RuntimeException, IOException { + OzoneKeyDetails keyDetails = client.getProxy().getKeyDetails(volumeName, bucketName, keyName); + if (!readMetadataOnly) { + byte[] data = new byte[objectSizeInBytes]; + try (OzoneInputStream introStream = keyDetails.getContent()) { + introStream.read(data); + } catch (Exception ex) { + throw ex; + } + } + } + + public void processWriteTasks(String keyName, OzoneClient ozoneClient) + throws RuntimeException, IOException { + try (OzoneOutputStream out = + ozoneClient.getProxy().createKey(volumeName, bucketName, keyName, objectSizeInBytes, null, new HashMap())) { + out.write(keyContent); + } catch (Exception ex) { + throw ex; + } + } + + public TaskType decideReadOrWriteTask() { + if (percentageRead == 100) { + return TaskType.READ_TASK; + } else if (percentageRead == 0) { + return TaskType.WRITE_TASK; + } + //mix workload + int tmp = ThreadLocalRandom.current().nextInt(1,101); + if (tmp <= percentageRead) { + return TaskType.READ_TASK; + } else { + return TaskType.WRITE_TASK; + } + } + + public String getKeyName() { + StringBuilder keyNameSb = new StringBuilder(); + int randomIdxWithinRange = ThreadLocalRandom.current(). + nextInt(startIndex, startIndex + range); + + if (keySorted) { + keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR). + append(randomIdxWithinRange); + } else { + keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR). + append(kg.generateMd5KeyName(randomIdxWithinRange)); + } + return keyNameSb.toString(); + } + +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java new file mode 100644 index 000000000000..bfe23f17818e --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.freon; + +import com.codahale.metrics.Timer; +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; + +import java.util.concurrent.Callable; +import java.util.function.Function; +import java.util.HashMap; + +import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.PURE_INDEX; +import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.MD5; +import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR; + +// import com.fasterxml.jackson.databind.ObjectMapper; +// import java.util.HashMap; +// import java.util.Map; +// import java.io.File; + +/** + * Ozone range keys generator for performance test. + */ +@CommandLine.Command(name = "ork", + description = "write range keys with the help of the ozone clients.", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true, + showDefaultValues = true) +public class RangeKeysGenerator extends BaseFreonGenerator + implements Callable { + + private static final Logger LOG = + LoggerFactory.getLogger(RangeKeysGenerator.class); + + @CommandLine.Option(names = {"-v", "--volume"}, + description = "Name of the volume which contains the test data. " + + "Will be created if missing.", + defaultValue = "ockrwvolume") + private String volumeName; + + @CommandLine.Option(names = {"-b", "--bucket"}, + description = "Name of the bucket which contains the test data.", + defaultValue = "ockrwbucket") + private String bucketName; + + @CommandLine.Option(names = {"-r", "--range-each-client-write"}, + description = "Write range for each client.", + defaultValue = "0") + private int range; + + @CommandLine.Option(names = {"-s", "--key-start-index"}, + description = "Start index of key.", + defaultValue = "0") + private int startIndex; + + + @CommandLine.Option(names = {"-k", "--key-encode"}, + description = "The algorithm to generate key names. " + + "Options are pureIndex, md5", + defaultValue = "md5") + private String encodeFormat; + + @CommandLine.Option(names = {"-g", "--size"}, + description = "Generated object size (in bytes) " + + "to be written.", + defaultValue = "1") + private int objectSizeInBytes; + + private int clientCount; + + @CommandLine.Option( + names = "--debug", + description = "Enable debugging message.", + defaultValue = "false" + ) + private boolean debug; + + + @CommandLine.Option( + names = "--om-service-id", + description = "OM Service ID" + ) + private String omServiceID = null; + private KeyGeneratorUtil kg; + private OzoneClient[] ozoneClients; + private byte[] keyContent; + private Timer timer; + + + @Override + public Void call() throws Exception { + init(); + OzoneConfiguration ozoneConfiguration = createOzoneConfiguration(); + clientCount = getThreadNo(); + ozoneClients = new OzoneClient[clientCount]; + for (int i = 0; i < clientCount; i++) { + ozoneClients[i] = createOzoneClient(omServiceID, ozoneConfiguration); + } + + ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName); + if (objectSizeInBytes >= 0) { + keyContent = RandomUtils.nextBytes(objectSizeInBytes); + } + timer = getMetrics().timer("key-read-write"); + + kg = new KeyGeneratorUtil(); + runTests(this::generateRangeKeys); + for (int i = 0; i < clientCount; i++) { + if (ozoneClients[i] != null) { + ozoneClients[i].close(); + } + } + + return null; + } + + public void generateRangeKeys(long count) throws Exception { + int clientIndex = (int)(count % clientCount); + OzoneClient client = ozoneClients[clientIndex]; + int start = startIndex + (int)count * range; + int end = start + range; + + timer.time(() -> { + switch (encodeFormat) { + case PURE_INDEX: + loopRunner(kg.pureIndexKeyNameFunc(), client, start, end); + break; + case MD5: + loopRunner(kg.md5KeyNameFunc(), client, start, end); + break; + default: + loopRunner(kg.md5KeyNameFunc(), client, start, end); + break; + } + return null; + }); + } + + + public void loopRunner(Function fn, OzoneClient client, + int start, int end) throws Exception { + String keyName; + for (int i = start; i < end + 1; i++) { + keyName = getPrefix() + FILE_DIR_SEPARATOR + fn.apply(i); + try (OzoneOutputStream out = client.getProxy(). + createKey(volumeName, bucketName, keyName, objectSizeInBytes, null, new HashMap())) { + out.write(keyContent); + } + } + } +} From ff6d15f5df822a5a3c776320b032fad3dd380f95 Mon Sep 17 00:00:00 2001 From: Nibiru Date: Thu, 20 Oct 2022 09:40:13 +0800 Subject: [PATCH 08/48] HDDS-7354. SchemaV3 blockData not deleted in table (#3860) --- .../background/BlockDeletingService.java | 4 +- .../common/TestBlockDeletingService.java | 64 +++++++++++++++++-- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java index 68c2bcc6fe50..ee57666a0aaa 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java @@ -501,8 +501,8 @@ private ContainerBackgroundTaskResult deleteViaTransactionStore( for (DeletedBlocksTransaction delTx : delBlocks) { deleter.apply(deleteTxns, batch, delTx.getTxID()); for (Long blk : delTx.getLocalIDList()) { - String bID = blk.toString(); - meta.getStore().getBlockDataTable().deleteWithBatch(batch, bID); + blockDataTable.deleteWithBatch(batch, + containerData.blockKey(blk)); } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java index fe3995968851..357cd7c0c9a3 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.utils.BackgroundService; +import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; @@ -42,6 +43,7 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy; +import org.apache.hadoop.ozone.container.common.interfaces.BlockIterator; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher; import org.apache.hadoop.ozone.container.common.interfaces.DBHandle; @@ -372,8 +374,8 @@ private int getUnderDeletionBlocksCount(DBHandle meta, DatanodeStoreSchemaTwoImpl dnStoreTwoImpl = (DatanodeStoreSchemaTwoImpl) ds; try ( - TableIterator> + TableIterator> iter = dnStoreTwoImpl.getDeleteTransactionTable().iterator()) { while (iter.hasNext()) { StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction @@ -430,16 +432,18 @@ public void testBlockDeletion() throws Exception { containerSet.listContainer(0L, 1, containerData); Assert.assertEquals(1, containerData.size()); KeyValueContainerData data = (KeyValueContainerData) containerData.get(0); + KeyPrefixFilter filter = Objects.equals(schemaVersion, SCHEMA_V1) ? + data.getDeletingBlockKeyFilter() : data.getUnprefixedKeyFilter(); try (DBHandle meta = BlockUtils.getDB(data, conf)) { Map> containerMap = containerSet.getContainerMapCopy(); + assertBlockDataTableRecordCount(3, meta, filter, data.getContainerID()); // NOTE: this test assumes that all the container is KetValueContainer and // have DeleteTransactionId in KetValueContainerData. If other // types is going to be added, this test should be checked. long transactionId = ((KeyValueContainerData) containerMap .get(containerData.get(0).getContainerID()).getContainerData()) .getDeleteTransactionId(); - long containerSpace = containerData.get(0).getBytesUsed(); // Number of deleted blocks in container should be equal to 0 before // block delete @@ -488,6 +492,9 @@ public void testBlockDeletion() throws Exception { Assert.assertEquals(3, deletingServiceMetrics.getSuccessCount() - deleteSuccessCount); + + // check if blockData get deleted + assertBlockDataTableRecordCount(0, meta, filter, data.getContainerID()); } svc.shutdown(); @@ -501,7 +508,6 @@ public void testWithUnrecordedBlocks() throws Exception { if (Objects.equals(schemaVersion, SCHEMA_V1)) { return; } - DatanodeConfiguration dnConf = conf.getObject(DatanodeConfiguration.class); dnConf.setBlockDeletionLimit(2); this.blockLimitPerInterval = dnConf.getBlockDeletionLimit(); @@ -526,6 +532,8 @@ public void testWithUnrecordedBlocks() throws Exception { Assert.assertEquals(2, containerData.size()); KeyValueContainerData ctr1 = (KeyValueContainerData) containerData.get(0); KeyValueContainerData ctr2 = (KeyValueContainerData) containerData.get(1); + KeyPrefixFilter filter = Objects.equals(schemaVersion, SCHEMA_V1) ? + ctr1.getDeletingBlockKeyFilter() : ctr1.getUnprefixedKeyFilter(); try (DBHandle meta = BlockUtils.getDB(ctr1, conf)) { // create unrecorded blocks in a new txn and update metadata, @@ -545,6 +553,8 @@ public void testWithUnrecordedBlocks() throws Exception { Assert.assertEquals(7, getUnderDeletionBlocksCount(meta, ctr1)); } + assertBlockDataTableRecordCount(3, ctr1, filter); + assertBlockDataTableRecordCount(3, ctr2, filter); Assert.assertEquals(3, ctr2.getNumPendingDeletionBlocks()); // Totally 2 container * 3 blocks + 4 unrecorded block = 10 blocks @@ -564,6 +574,9 @@ public void testWithUnrecordedBlocks() throws Exception { Assert.assertEquals(0, ctr2.getBlockCount()); Assert.assertEquals(0, ctr2.getBytesUsed()); + // check if blockData get deleted + assertBlockDataTableRecordCount(0, ctr1, filter); + assertBlockDataTableRecordCount(0, ctr2, filter); svc.shutdown(); } @@ -832,4 +845,47 @@ public void testBlockThrottle() throws Exception { service.shutdown(); } } + + /** + * Check blockData record count of certain container (DBHandle not provided). + * + * @param expectedCount expected records in the table + * @param containerData KeyValueContainerData + * @param filter KeyPrefixFilter + * @throws IOException + */ + private void assertBlockDataTableRecordCount(int expectedCount, + KeyValueContainerData containerData, KeyPrefixFilter filter) + throws IOException { + try (DBHandle handle = BlockUtils.getDB(containerData, conf)) { + long containerID = containerData.getContainerID(); + assertBlockDataTableRecordCount(expectedCount, handle, filter, + containerID); + } + } + + /** + * Check blockData record count of certain container (DBHandle provided). + * + * @param expectedCount expected records in the table + * @param handle DB handle + * @param filter KeyPrefixFilter + * @param containerID the container ID to filter results + * @throws IOException + */ + private void assertBlockDataTableRecordCount(int expectedCount, + DBHandle handle, KeyPrefixFilter filter, long containerID) + throws IOException { + long count = 0L; + BlockIterator iterator = handle.getStore(). + getBlockIterator(containerID, filter); + iterator.seekToFirst(); + while (iterator.hasNext()) { + iterator.nextBlock(); + count += 1; + } + Assert.assertEquals("Excepted: " + expectedCount + + ", but actual: " + count + " in the blockData table of container: " + + containerID + ".", expectedCount, count); + } } From 3fd7cd08962a4eac375e9025c8a90eeb281feb00 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 20 Oct 2022 04:26:50 +0200 Subject: [PATCH 09/48] Revert "HDDS-7199. Implement new mix workload Read/Write Freon command which meets specific test requirements (#3754)" This reverts commit e45f9b8333a0ccd605bdc3d18aa36282d4ba5859. --- .../main/smoketest/freon/read-write-key.robot | 53 ---- .../org/apache/hadoop/ozone/freon/Freon.java | 2 - .../hadoop/ozone/freon/KeyGeneratorUtil.java | 50 ---- .../freon/OzoneClientKeyReadWriteOps.java | 243 ------------------ .../ozone/freon/RangeKeysGenerator.java | 176 ------------- 5 files changed, 524 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java diff --git a/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot deleted file mode 100644 index 25a3c4b9e9a8..000000000000 --- a/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation Test freon read/write key commands -Resource ../ozone-lib/freon.robot -Test Timeout 5 minutes - -*** Variables *** -${PREFIX} ${EMPTY} - - -*** Test Cases *** -Pre-generate 100 keys of size 1 byte each to Ozone - ${result} = Execute ozone freon ork -n 1 -t 10 -r 100 --size 1 -v voltest -b buckettest -p performanceTest - -Read 10 keys from pre-generated keys - ${keysCount} = BuiltIn.Set Variable 10 - ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 -r 100 -v voltest -b buckettest -p performanceTest - Should contain ${result} Successful executions: ${keysCount} - -Read 10 keys' metadata from pre-generated keys - ${keysCount} = BuiltIn.Set Variable 10 - ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 -m -r 100 -v voltest -b buckettest -p performanceTest - Should contain ${result} Successful executions: ${keysCount} - -Write 10 keys of size 1 byte each from key index 0 to 99 - ${keysCount} = BuiltIn.Set Variable 10 - ${size} = BuiltIn.Set Variable 1 - ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 --percentage-read 0 --size ${size} -r 100 -v voltest -b buckettest -p performanceTest2 - Should contain ${result} Successful executions: ${keysCount} - ${keyName} = Execute echo -n '1' | md5sum | head -c 7 - ${result} = Execute ozone sh key info /voltest/buckettest/performanceTest2/${keyName} - Should contain ${result} \"dataSize\" : 1 - - -Run 90 % of read-key tasks and 10 % of write-key tasks for 10 keys from pre-generated keys - ${keysCount} = BuiltIn.Set Variable 10 - ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 --percentage-read 90 -r 100 -v voltest -b buckettest -p performanceTest - Should contain ${result} Successful executions: ${keysCount} - diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java index dfb52ca9add6..b6bb6055eefe 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java @@ -68,8 +68,6 @@ OmBucketReadWriteFileOps.class, OmBucketReadWriteKeyOps.class, OmRPCLoadGenerator.class, - OzoneClientKeyReadWriteOps.class, - RangeKeysGenerator.class }, versionProvider = HddsVersionProvider.class, mixinStandardHelpOptions = true) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java deleted file mode 100644 index 9773caa44070..000000000000 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.ozone.freon; - -import org.apache.commons.codec.digest.DigestUtils; - -import java.util.function.Function; - -/** - * Utility class to generate key name from a given key index. - */ -public class KeyGeneratorUtil { - public static final String PURE_INDEX = "pureIndex"; - public static final String MD5 = "md5"; - public static final String FILE_DIR_SEPARATOR = "/"; - - public String generatePureIndexKeyName(int number) { - return String.valueOf(number); - } - public Function pureIndexKeyNameFunc() { - return number -> String.valueOf(number); - } - - public String generateMd5KeyName(int number) { - String encodedStr = DigestUtils.md5Hex(String.valueOf(number)); - return encodedStr.substring(0, 7); - } - - public Function md5KeyNameFunc() { - return number -> DigestUtils.md5Hex(String.valueOf(number)).substring(0, 7); - } - -} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java deleted file mode 100644 index c4374f4eada6..000000000000 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.ozone.freon; - - -import com.codahale.metrics.Timer; -import org.apache.commons.lang3.RandomUtils; -import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneKeyDetails; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import picocli.CommandLine; - -import java.io.IOException; -import java.util.concurrent.Callable; -import java.util.concurrent.ThreadLocalRandom; -import java.util.HashMap; - -import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR; - -/** - * Ozone key generator/reader for performance test. - */ - -@CommandLine.Command(name = "ockrw", - aliases = "ozone-client-key-read-write-ops", - description = "Generate keys with a fixed name and ranges that can" - + " be written and read as sub-ranges from multiple clients.", - versionProvider = HddsVersionProvider.class, - mixinStandardHelpOptions = true, - showDefaultValues = true) -public class OzoneClientKeyReadWriteOps extends BaseFreonGenerator - implements Callable { - - @CommandLine.Option(names = {"-v", "--volume"}, - description = "Name of the volume which contains the test data. " + - "Will be created if missing.", - defaultValue = "ockrwvolume") - private String volumeName; - - @CommandLine.Option(names = {"-b", "--bucket"}, - description = "Name of the bucket which contains the test data.", - defaultValue = "ockrwbucket") - private String bucketName; - - @CommandLine.Option(names = {"-m", "--read-metadata-only"}, - description = "If only read key's metadata.", - defaultValue = "false") - private boolean readMetadataOnly; - - @CommandLine.Option(names = {"-s", "--start-index"}, - description = "Start index of keys of read/write operation." + - "This can allow adding keys incrementally or parallel from multiple" - + " clients. Example: Write keys 0-1000000 followed by " - + "keys 1000001-2000000.", - defaultValue = "0") - private int startIndex; - - @CommandLine.Option(names = {"-r", "--range"}, - description = "Range of read/write operations. This in co-ordination" - + " with --start-index can specify the range to read. " - + "Example: Read from --start-index 1000 and read --range 1000 keys.", - defaultValue = "1") - private int range; - - @CommandLine.Option(names = {"--size"}, - description = "Object size (in bytes) " + - "to read/write. If user sets a read size which is larger" - + " than the key size, it only reads bytes up to key size.", - defaultValue = "1") - private int objectSizeInBytes; - - @CommandLine.Option(names = {"--contiguous"}, - description = "By default, the keys are randomized lexically" - + " by calculating the md5 of the key name. If this option is set," - + " the keys are written lexically contiguously.", - defaultValue = "false") - private boolean keySorted; - - @CommandLine.Option(names = {"--percentage-read"}, - description = "Percentage of read tasks in mix workload." - + " The remainder of the percentage will writes to keys." - + " Example --percentage-read 90 will result in 10% writes.", - defaultValue = "100") - private int percentageRead; - - @CommandLine.Option( - names = "--om-service-id", - description = "OM Service ID" - ) - private String omServiceID = null; - - private Timer timer; - - private OzoneClient[] ozoneClients; - - private int clientCount; - - private byte[] keyContent; - - private static final Logger LOG = - LoggerFactory.getLogger(OzoneClientKeyReadWriteOps.class); - - /** - * Task type of read task, or write task. - */ - public enum TaskType { - READ_TASK, - WRITE_TASK - } - private KeyGeneratorUtil kg; - - - @Override - public Void call() throws Exception { - init(); - OzoneConfiguration ozoneConfiguration = createOzoneConfiguration(); - clientCount = getThreadNo(); - ozoneClients = new OzoneClient[clientCount]; - for (int i = 0; i < clientCount; i++) { - ozoneClients[i] = createOzoneClient(omServiceID, ozoneConfiguration); - } - - ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName); - - timer = getMetrics().timer("key-read-write"); - if (objectSizeInBytes >= 0) { - keyContent = RandomUtils.nextBytes(objectSizeInBytes); - } - if (kg == null) { - kg = new KeyGeneratorUtil(); - } - runTests(this::readWriteKeys); - - for (int i = 0; i < clientCount; i++) { - if (ozoneClients[i] != null) { - ozoneClients[i].close(); - } - } - return null; - } - - public void readWriteKeys(long counter) throws RuntimeException, IOException { - int clientIndex = (int)((counter) % clientCount); - TaskType taskType = decideReadOrWriteTask(); - String keyName = getKeyName(); - - timer.time(() -> { - try { - switch (taskType) { - case READ_TASK: - processReadTasks(keyName, ozoneClients[clientIndex]); - break; - case WRITE_TASK: - processWriteTasks(keyName, ozoneClients[clientIndex]); - break; - default: - break; - } - } catch (RuntimeException ex) { - LOG.error(ex.getMessage()); - throw ex; - } catch (IOException ex) { - LOG.error(ex.getMessage()); - throw new RuntimeException(ex.getMessage()); - } - - }); - } - - public void processReadTasks(String keyName, OzoneClient client) - throws RuntimeException, IOException { - OzoneKeyDetails keyDetails = client.getProxy().getKeyDetails(volumeName, bucketName, keyName); - if (!readMetadataOnly) { - byte[] data = new byte[objectSizeInBytes]; - try (OzoneInputStream introStream = keyDetails.getContent()) { - introStream.read(data); - } catch (Exception ex) { - throw ex; - } - } - } - - public void processWriteTasks(String keyName, OzoneClient ozoneClient) - throws RuntimeException, IOException { - try (OzoneOutputStream out = - ozoneClient.getProxy().createKey(volumeName, bucketName, keyName, objectSizeInBytes, null, new HashMap())) { - out.write(keyContent); - } catch (Exception ex) { - throw ex; - } - } - - public TaskType decideReadOrWriteTask() { - if (percentageRead == 100) { - return TaskType.READ_TASK; - } else if (percentageRead == 0) { - return TaskType.WRITE_TASK; - } - //mix workload - int tmp = ThreadLocalRandom.current().nextInt(1,101); - if (tmp <= percentageRead) { - return TaskType.READ_TASK; - } else { - return TaskType.WRITE_TASK; - } - } - - public String getKeyName() { - StringBuilder keyNameSb = new StringBuilder(); - int randomIdxWithinRange = ThreadLocalRandom.current(). - nextInt(startIndex, startIndex + range); - - if (keySorted) { - keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR). - append(randomIdxWithinRange); - } else { - keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR). - append(kg.generateMd5KeyName(randomIdxWithinRange)); - } - return keyNameSb.toString(); - } - -} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java deleted file mode 100644 index bfe23f17818e..000000000000 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.ozone.freon; - -import com.codahale.metrics.Timer; -import org.apache.commons.lang3.RandomUtils; -import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import picocli.CommandLine; - -import java.util.concurrent.Callable; -import java.util.function.Function; -import java.util.HashMap; - -import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.PURE_INDEX; -import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.MD5; -import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR; - -// import com.fasterxml.jackson.databind.ObjectMapper; -// import java.util.HashMap; -// import java.util.Map; -// import java.io.File; - -/** - * Ozone range keys generator for performance test. - */ -@CommandLine.Command(name = "ork", - description = "write range keys with the help of the ozone clients.", - versionProvider = HddsVersionProvider.class, - mixinStandardHelpOptions = true, - showDefaultValues = true) -public class RangeKeysGenerator extends BaseFreonGenerator - implements Callable { - - private static final Logger LOG = - LoggerFactory.getLogger(RangeKeysGenerator.class); - - @CommandLine.Option(names = {"-v", "--volume"}, - description = "Name of the volume which contains the test data. " + - "Will be created if missing.", - defaultValue = "ockrwvolume") - private String volumeName; - - @CommandLine.Option(names = {"-b", "--bucket"}, - description = "Name of the bucket which contains the test data.", - defaultValue = "ockrwbucket") - private String bucketName; - - @CommandLine.Option(names = {"-r", "--range-each-client-write"}, - description = "Write range for each client.", - defaultValue = "0") - private int range; - - @CommandLine.Option(names = {"-s", "--key-start-index"}, - description = "Start index of key.", - defaultValue = "0") - private int startIndex; - - - @CommandLine.Option(names = {"-k", "--key-encode"}, - description = "The algorithm to generate key names. " + - "Options are pureIndex, md5", - defaultValue = "md5") - private String encodeFormat; - - @CommandLine.Option(names = {"-g", "--size"}, - description = "Generated object size (in bytes) " + - "to be written.", - defaultValue = "1") - private int objectSizeInBytes; - - private int clientCount; - - @CommandLine.Option( - names = "--debug", - description = "Enable debugging message.", - defaultValue = "false" - ) - private boolean debug; - - - @CommandLine.Option( - names = "--om-service-id", - description = "OM Service ID" - ) - private String omServiceID = null; - private KeyGeneratorUtil kg; - private OzoneClient[] ozoneClients; - private byte[] keyContent; - private Timer timer; - - - @Override - public Void call() throws Exception { - init(); - OzoneConfiguration ozoneConfiguration = createOzoneConfiguration(); - clientCount = getThreadNo(); - ozoneClients = new OzoneClient[clientCount]; - for (int i = 0; i < clientCount; i++) { - ozoneClients[i] = createOzoneClient(omServiceID, ozoneConfiguration); - } - - ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName); - if (objectSizeInBytes >= 0) { - keyContent = RandomUtils.nextBytes(objectSizeInBytes); - } - timer = getMetrics().timer("key-read-write"); - - kg = new KeyGeneratorUtil(); - runTests(this::generateRangeKeys); - for (int i = 0; i < clientCount; i++) { - if (ozoneClients[i] != null) { - ozoneClients[i].close(); - } - } - - return null; - } - - public void generateRangeKeys(long count) throws Exception { - int clientIndex = (int)(count % clientCount); - OzoneClient client = ozoneClients[clientIndex]; - int start = startIndex + (int)count * range; - int end = start + range; - - timer.time(() -> { - switch (encodeFormat) { - case PURE_INDEX: - loopRunner(kg.pureIndexKeyNameFunc(), client, start, end); - break; - case MD5: - loopRunner(kg.md5KeyNameFunc(), client, start, end); - break; - default: - loopRunner(kg.md5KeyNameFunc(), client, start, end); - break; - } - return null; - }); - } - - - public void loopRunner(Function fn, OzoneClient client, - int start, int end) throws Exception { - String keyName; - for (int i = start; i < end + 1; i++) { - keyName = getPrefix() + FILE_DIR_SEPARATOR + fn.apply(i); - try (OzoneOutputStream out = client.getProxy(). - createKey(volumeName, bucketName, keyName, objectSizeInBytes, null, new HashMap())) { - out.write(keyContent); - } - } - } -} From 1fa6d02e8a025e99cbb9a8db4d4daa9072af0d98 Mon Sep 17 00:00:00 2001 From: Navink Date: Thu, 20 Oct 2022 09:29:29 +0530 Subject: [PATCH 10/48] HDDS-6930. SCM,OM,RECON should not print ERROR and exit with code 1 on successful shutdown (#3848) --- .../apache/hadoop/hdds/scm/server/StorageContainerManager.java | 2 +- .../src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java | 2 +- .../ozone/recon/scm/ReconStorageContainerManagerFacade.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 73a5468c397b..1f57b4ffea2b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -1655,7 +1655,7 @@ public void stop() { @Override public void shutDown(String message) { stop(); - ExitUtils.terminate(1, message, LOG); + ExitUtils.terminate(0, message, LOG); } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 70f8359fe832..b5be6c02d769 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2089,7 +2089,7 @@ public void stop() { public void shutDown(String message) { stop(); - ExitUtils.terminate(1, message, LOG); + ExitUtils.terminate(0, message, LOG); } /** diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java index 27c474a1bd15..11dca8628bae 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java @@ -352,7 +352,7 @@ public void stop() { @Override public void shutDown(String message) { stop(); - ExitUtils.terminate(1, message, LOG); + ExitUtils.terminate(0, message, LOG); } public ReconDatanodeProtocolServer getDatanodeProtocolServer() { From df48ca4fe39c963e3c56e03e5218d24817b76bfe Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 20 Oct 2022 12:31:04 +0800 Subject: [PATCH 11/48] HDDS-7356. Update SCM-HA.zh.md to match the English version (#3861) --- hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 73 +++++++++++++++++-- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/docs/content/feature/SCM-HA.zh.md b/hadoop-hdds/docs/content/feature/SCM-HA.zh.md index 4f4ce3b499e9..9875337e6445 100644 --- a/hadoop-hdds/docs/content/feature/SCM-HA.zh.md +++ b/hadoop-hdds/docs/content/feature/SCM-HA.zh.md @@ -25,10 +25,6 @@ summary: Storage Container Manager 的 HA 设置可以避免任何单点故障 Ozone包含两个元数据管理节点(用于键管理的 *Ozone Manager* 和用于块空间管理的 *Storage Container management* )和多个存储节点(数据节点)。通过 RAFT 共识算法实现数据在数据节点之间的复制。 -

- 为了避免任何单点故障，元数据管理器节点还应该具有 HA 设置。 Ozone Manager 和 Storage Container Manager 都支持 HA。在这种模式下，内部状态通过 RAFT (使用 Apache Ratis )复制。 @@ -37,6 +33,12 @@ Ozone Manager 和 Storage Container Manager 都支持 HA。在这种模式下， ## 配置 +> ⚠️ **注意** ⚠️ +> +> SCM HA 目前仅支持新初始化的集群。 +> SCM HA 必须在 Ozone 服务首次启动前开启。 +> 当某个 SCM 以非 HA 的模式启动后，不支持将其改为 HA 模式。 + Storage Container Manager 的 HA 模式可以在 `ozone-site.xml` 中进行以下设置： ```XML @@ -94,7 +96,7 @@ Storage Container Manager 的 HA 模式可以在 `ozone-site.xml` 中进行以 bin/ozone scm --init ``` -第二个和第三个节点应该被 *bootstrapped*，而不是 init。这些集群将加入到配置的 RAFT 仲裁。当前服务器的 id 通过 DNS 名称标识，也可以通过 `ozone.scm.node.id` 明确设置。大多数时候你不需要设置它，因为基于 DNS 的 id 检测可以很好地工作。 +第二个和第三个节点应该被 *bootstrap*，而不是 init。这些集群将加入到配置的 RAFT 仲裁。当前服务器的 id 通过 DNS 名称标识，也可以通过 `ozone.scm.node.id` 明确设置。大多数时候你不需要设置它，因为基于 DNS 的 id 检测可以很好地工作。 ``` bin/ozone scm --bootstrap @@ -111,6 +113,67 @@ bin/ozone scm --bootstrap 根据 `ozone.scm.primordial.node.id`，初始化进程将在第二个/第三个节点上被忽略，引导进程将在除原始节点外的所有节点上被忽略。 +## SCM HA 安全 + +![SCM Secure HA](scm-secure-ha.png) + +在一个安全 SCM HA 集群中，我们将执行初始化的 SCM 称为原始 SCM。 +原始 SCM 使用自签名证书启动根 CA，并用于颁发签名证书到它自己和其他 +引导的 SCM。 只有原始 SCM 可以为其他 SCM 颁发签名证书。 +因此，原始 SCM 在 SCM HA 集群中具有特殊的作用，因为它是唯一可以向 SCM 颁发证书的 SCM。 + +原始 SCM 担任根 CA 角色，它使用子 CA 证书签署所有 SCM 实例。 +SCM 使用子 CA 证书来签署 OM/Datanodes 的证书。 + +引导 SCM 时会从原始 SCM 获取签名证书并启动子 CA。 + +SCM 上的子 CA 用于为集群中的 OM/DN 颁发签名证书。 只有 leader SCM 向 OM/DN 颁发证书。 + +### 如何启用安全 + +```XML + +ozone.security.enable +true + + + +hdds.grpc.tls.enabled +true + +``` + +在正常的 SCM HA 配置的基础上，需要添加上述配置。 + +### 原始 SCM + +原始 SCM 由配置 ozone.scm.primordial.node.id 确定。 +此值可以是 SCM 的节点 ID 或原始机名。 +如果配置是未定义的，则运行 init 的节点被视为原始 SCM。 + +{{< highlight bash >}} +bin/ozone scm --init +{{< /highlight >}} + +这将为根 CA 设置公钥、私钥对和自签名证书 +并生成公钥、私钥对和 CSR 以从根 CA 获取子 CA 的签名证书。 + +### 引导 SCM + +{{< highlight bash >}} +bin/ozone scm --bootstrap +{{< /highlight >}} + +这将为子 CA 设置公钥、私钥对并生成 CSR 以获取来自根 CA 的子 CA 签名证书。 + +**注意**: 当原始 SCM 未定义时，请确保仅在一个 SCM 上运行 **--init**， +在其他 SCM 节点上需使用 **--bootstrap** 进行引导。 + +### 目前 SCM HA 安全的限制 + +1. 当原始 SCM 失效时, 新的 SCM 不能被引导并添加到 HA 节点中。 +2. 尚未支持从非 HA 安全集群升级到 HA 安全集群。 + ## 实现细节 SCM HA 使用 Apache Ratis 在 SCM HA 仲裁的成员之间复制状态。每个节点在本地 RocksDB 中维护块管理元数据。 From 31560fcdfe22b5ffb18f3e847be8b814bb3080ee Mon Sep 17 00:00:00 2001 From: Jie Yao Date: Thu, 20 Oct 2022 14:33:00 +0800 Subject: [PATCH 12/48] HDDS-7355. non-primordial scm fail to get signed cert from primordial SCM when converting an unsecure cluster to secure (#3859) --- .../hdds/scm/server/StorageContainerManager.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 1f57b4ffea2b..09844681ab76 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -1072,7 +1072,8 @@ public static boolean scmBootstrap(OzoneConfiguration conf) scmStorageConfig.getScmId()); // Initialize security if security is enabled later. - initializeSecurityIfNeeded(conf, scmhaNodeDetails, scmStorageConfig); + initializeSecurityIfNeeded( + conf, scmhaNodeDetails, scmStorageConfig, false); return true; } @@ -1097,7 +1098,8 @@ public static boolean scmBootstrap(OzoneConfiguration conf) } // Initialize security if security is enabled later. - initializeSecurityIfNeeded(conf, scmhaNodeDetails, scmStorageConfig); + initializeSecurityIfNeeded( + conf, scmhaNodeDetails, scmStorageConfig, false); } else { try { @@ -1136,14 +1138,15 @@ public static boolean scmBootstrap(OzoneConfiguration conf) * @param scmStorageConfig * @throws IOException */ - private static void initializeSecurityIfNeeded(OzoneConfiguration conf, - SCMHANodeDetails scmhaNodeDetails, SCMStorageConfig scmStorageConfig) + private static void initializeSecurityIfNeeded( + OzoneConfiguration conf, SCMHANodeDetails scmhaNodeDetails, + SCMStorageConfig scmStorageConfig, boolean isPrimordial) throws IOException { // Initialize security if security is enabled later. if (OzoneSecurityUtil.isSecurityEnabled(conf) && scmStorageConfig.getScmCertSerialId() == null) { HASecurityUtils.initializeSecurity(scmStorageConfig, conf, - getScmAddress(scmhaNodeDetails, conf), true); + getScmAddress(scmhaNodeDetails, conf), isPrimordial); scmStorageConfig.forceInitialize(); LOG.info("SCM unsecure cluster is converted to secure cluster. " + "Persisted SCM Certificate SerialID {}", @@ -1233,7 +1236,7 @@ public static boolean scmInit(OzoneConfiguration conf, final boolean isSCMHAEnabled = scmStorageConfig.isSCMHAEnabled(); // Initialize security if security is enabled later. - initializeSecurityIfNeeded(conf, haDetails, scmStorageConfig); + initializeSecurityIfNeeded(conf, haDetails, scmStorageConfig, true); if (SCMHAUtils.isSCMHAEnabled(conf) && !isSCMHAEnabled) { SCMRatisServerImpl.initialize(scmStorageConfig.getClusterID(), From f9b74a25cffe77d4b8031fe332fe247b9bd596c8 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Thu, 20 Oct 2022 08:17:24 -0700 Subject: [PATCH 13/48] HDDS-6210. EC: Add EC metrics (#3851) --- .../hadoop/hdds/scm/XceiverClientMetrics.java | 10 +++ .../client/io/ECBlockInputStreamProxy.java | 7 ++ .../statemachine/DatanodeStateMachine.java | 8 +- .../ECReconstructionCoordinator.java | 15 +++- .../ECReconstructionMetrics.java | 80 +++++++++++++++++++ .../TestECReconstructionSupervisor.java | 3 +- .../scm/storage/TestContainerCommandsEC.java | 10 ++- .../apache/hadoop/ozone/om/TestOmMetrics.java | 61 ++++++++++---- .../org/apache/hadoop/ozone/om/OMMetrics.java | 22 +++++ .../request/bucket/OMBucketCreateRequest.java | 11 +++ .../om/request/key/OMKeyCommitRequest.java | 6 ++ .../om/request/key/OMKeyCreateRequest.java | 3 + 12 files changed, 215 insertions(+), 21 deletions(-) create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionMetrics.java diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java index 399c520fb05a..96db6d13fea5 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java @@ -39,6 +39,8 @@ public class XceiverClientMetrics { private @Metric MutableCounterLong pendingOps; private @Metric MutableCounterLong totalOps; + private @Metric MutableCounterLong ecReconstructionTotal; + private @Metric MutableCounterLong ecReconstructionFailsTotal; private MutableCounterLong[] pendingOpsArray; private MutableCounterLong[] opsArray; private MutableRate[] containerOpsLatency; @@ -100,6 +102,14 @@ public long getPendingContainerOpCountMetrics(ContainerProtos.Type type) { return pendingOpsArray[type.ordinal()].value(); } + public void incECReconstructionTotal() { + ecReconstructionTotal.incr(); + } + + public void incECReconstructionFailsTotal() { + ecReconstructionFailsTotal.incr(); + } + @VisibleForTesting public long getTotalOpCount() { return totalOps.value(); diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamProxy.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamProxy.java index 5427e300cec3..7a8b0d3e8eea 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamProxy.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamProxy.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.XceiverClientFactory; +import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.storage.BlockExtendedInputStream; import org.apache.hadoop.hdds.scm.storage.BlockLocationInfo; @@ -117,6 +118,10 @@ private synchronized void setReaderType() { } private void createBlockReader() { + if (reconstructionReader) { + XceiverClientManager.getXceiverClientMetrics() + .incECReconstructionTotal(); + } blockReader = ecBlockInputStreamFactory.create(reconstructionReader, failedLocations, repConfig, blockInfo, verifyChecksum, xceiverClientFactory, refreshFunction); @@ -162,6 +167,8 @@ public synchronized int read(ByteBuffer buf) throws IOException { // If we get an error from the reconstruction reader, there // is nothing left to try. It will re-try until it has insufficient // locations internally, so if an error comes here, just re-throw it. + XceiverClientManager.getXceiverClientMetrics() + .incECReconstructionFailsTotal(); throw e; } if (e instanceof BadDataLocationException) { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index cc05511b58aa..31864e2868c8 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -54,6 +54,7 @@ import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.ReplicateContainerCommandHandler; import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.SetNodeOperationalStateCommandHandler; import org.apache.hadoop.ozone.container.ec.reconstruction.ECReconstructionCoordinator; +import org.apache.hadoop.ozone.container.ec.reconstruction.ECReconstructionMetrics; import org.apache.hadoop.ozone.container.ec.reconstruction.ECReconstructionSupervisor; import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; @@ -116,6 +117,7 @@ public class DatanodeStateMachine implements Closeable { private final ReadWriteLock constructionLock = new ReentrantReadWriteLock(); private final MeasuredReplicator replicatorMetrics; private final ReplicationSupervisorMetrics replicationSupervisorMetrics; + private final ECReconstructionMetrics ecReconstructionMetrics; /** * Constructs a datanode state machine. @@ -182,8 +184,11 @@ public DatanodeStateMachine(DatanodeDetails datanodeDetails, replicationSupervisorMetrics = ReplicationSupervisorMetrics.create(supervisor); + ecReconstructionMetrics = ECReconstructionMetrics.create(); + ECReconstructionCoordinator ecReconstructionCoordinator = - new ECReconstructionCoordinator(conf, certClient); + new ECReconstructionCoordinator(conf, certClient, + ecReconstructionMetrics); ecReconstructionSupervisor = new ECReconstructionSupervisor(container.getContainerSet(), context, replicationConfig.getReplicationMaxStreams(), @@ -378,6 +383,7 @@ public void close() throws IOException { } context.setState(DatanodeStates.getLastState()); replicationSupervisorMetrics.unRegister(); + ecReconstructionMetrics.unRegister(); executorService.shutdown(); try { if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java index b9da5cba5f11..3fb5361d261c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java @@ -102,10 +102,11 @@ public class ECReconstructionCoordinator implements Closeable { private final BlockInputStreamFactory blockInputStreamFactory; private final TokenHelper tokenHelper; private final ContainerClientMetrics clientMetrics; + private final ECReconstructionMetrics metrics; public ECReconstructionCoordinator(ConfigurationSource conf, - CertificateClient certificateClient) - throws IOException { + CertificateClient certificateClient, + ECReconstructionMetrics metrics) throws IOException { this.containerOperationClient = new ECContainerOperationClient(conf, certificateClient); this.byteBufferPool = new ElasticByteBufferPool(); @@ -121,6 +122,7 @@ public ECReconstructionCoordinator(ConfigurationSource conf, .getInstance(byteBufferPool, () -> ecReconstructExecutor); tokenHelper = new TokenHelper(conf, certificateClient); this.clientMetrics = ContainerClientMetrics.acquire(); + this.metrics = metrics; } public void reconstructECContainerGroup(long containerID, @@ -162,8 +164,13 @@ public void reconstructECContainerGroup(long containerID, containerOperationClient .closeContainer(containerID, dn, repConfig, containerToken); } + metrics.incReconstructionTotal(); + metrics.incBlockGroupReconstructionTotal(blockLocationInfoMap.size()); } catch (Exception e) { // Any exception let's delete the recovering containers. + metrics.incReconstructionFailsTotal(); + metrics.incBlockGroupReconstructionFailsTotal( + blockLocationInfoMap.size()); LOG.warn( "Exception while reconstructing the container {}. Cleaning up" + " all the recovering containers in the reconstruction process.", @@ -445,4 +452,8 @@ private long calcEffectiveBlockGroupLen(BlockData[] blockGroup, } return blockGroupLen == Long.MAX_VALUE ? 0 : blockGroupLen; } + + public ECReconstructionMetrics getECReconstructionMetrics() { + return this.metrics; + } } \ No newline at end of file diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionMetrics.java new file mode 100644 index 000000000000..91442c65f74f --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionMetrics.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.container.ec.reconstruction; + +import org.apache.hadoop.hdds.annotation.InterfaceAudience; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.ozone.OzoneConsts; + +/** + * Metrics class for EC Reconstruction. + */ +@InterfaceAudience.Private +@Metrics(about = "EC Reconstruction Coordinator Metrics", + context = OzoneConsts.OZONE) +public final class ECReconstructionMetrics { + private static final String SOURCE = + ECReconstructionMetrics.class.getSimpleName(); + + private @Metric MutableCounterLong blockGroupReconstructionTotal; + private @Metric MutableCounterLong blockGroupReconstructionFailsTotal; + private @Metric MutableCounterLong reconstructionTotal; + private @Metric MutableCounterLong reconstructionFailsTotal; + + private ECReconstructionMetrics() { + } + + public static ECReconstructionMetrics create() { + MetricsSystem ms = DefaultMetricsSystem.instance(); + return ms.register(SOURCE, "EC Reconstruction Coordinator Metrics", + new ECReconstructionMetrics()); + } + + public void unRegister() { + MetricsSystem ms = DefaultMetricsSystem.instance(); + ms.unregisterSource(SOURCE); + } + + public void incBlockGroupReconstructionTotal(long count) { + blockGroupReconstructionTotal.incr(count); + } + + public void incBlockGroupReconstructionFailsTotal(long count) { + blockGroupReconstructionFailsTotal.incr(count); + } + + public void incReconstructionTotal() { + reconstructionTotal.incr(); + } + + public void incReconstructionFailsTotal() { + reconstructionFailsTotal.incr(); + } + + public long getReconstructionTotal() { + return reconstructionTotal.value(); + } + + public long getBlockGroupReconstructionTotal() { + return blockGroupReconstructionTotal.value(); + } +} diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionSupervisor.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionSupervisor.java index b98eef7f2f07..c40ceb2ea36c 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionSupervisor.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionSupervisor.java @@ -42,7 +42,8 @@ public void testAddTaskShouldExecuteTheGivenTask() final CountDownLatch holdProcessing = new CountDownLatch(1); ECReconstructionSupervisor supervisor = new ECReconstructionSupervisor(null, null, 5, - new ECReconstructionCoordinator(new OzoneConfiguration(), null) { + new ECReconstructionCoordinator(new OzoneConfiguration(), null, + ECReconstructionMetrics.create()) { @Override public void reconstructECContainerGroup(long containerID, ECReplicationConfig repConfig, diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java index 8713dc2a17b8..df8ee103f6df 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java @@ -66,6 +66,7 @@ import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.ozone.container.ec.reconstruction.ECContainerOperationClient; import org.apache.hadoop.ozone.container.ec.reconstruction.ECReconstructionCoordinator; +import org.apache.hadoop.ozone.container.ec.reconstruction.ECReconstructionMetrics; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -390,8 +391,10 @@ private void testECReconstructionCoordinator(List missingIndexes) new XceiverClientManager(config); createKeyAndWriteData(keyString, bucket); ECReconstructionCoordinator coordinator = - new ECReconstructionCoordinator(config, certClient); + new ECReconstructionCoordinator(config, certClient, + ECReconstructionMetrics.create()); + ECReconstructionMetrics metrics = coordinator.getECReconstructionMetrics(); OzoneKeyDetails key = bucket.getKey(keyString); long conID = key.getOzoneKeyLocations().get(0).getContainerID(); Token cToken = containerTokenGenerator @@ -493,7 +496,7 @@ private void testECReconstructionCoordinator(List missingIndexes) readContainerResponseProto.getContainerData().getState()); i++; } - + Assertions.assertEquals(metrics.getReconstructionTotal(), 1L); } private void createKeyAndWriteData(String keyString, OzoneBucket bucket) @@ -565,7 +568,8 @@ public void testECReconstructionCoordinatorShouldCleanupContainersOnFailure() Assert.assertThrows(IOException.class, () -> { ECReconstructionCoordinator coordinator = - new ECReconstructionCoordinator(config, certClient); + new ECReconstructionCoordinator(config, certClient, + ECReconstructionMetrics.create()); coordinator.reconstructECContainerGroup(conID, (ECReplicationConfig) containerPipeline.getReplicationConfig(), sourceNodeMap, targetNodeMap); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java index c9babb892208..9f6141d8f91e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.ContainerBlockID; +import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; @@ -196,7 +197,7 @@ public void testBucketOps() throws Exception { ozoneManager, "bucketManager"); BucketManager mockBm = Mockito.spy(bucketManager); - OmBucketInfo bucketInfo = createBucketInfo(); + OmBucketInfo bucketInfo = createBucketInfo(false); doBucketOps(bucketInfo); MetricsRecordBuilder omMetrics = getMetrics("OMMetrics"); @@ -208,11 +209,18 @@ public void testBucketOps() throws Exception { assertCounter("NumBucketLists", 1L, omMetrics); assertCounter("NumBuckets", 0L, omMetrics); - bucketInfo = createBucketInfo(); + OmBucketInfo ecBucketInfo = createBucketInfo(true); + writeClient.createBucket(ecBucketInfo); + writeClient.deleteBucket(ecBucketInfo.getVolumeName(), + ecBucketInfo.getBucketName()); + omMetrics = getMetrics("OMMetrics"); + assertCounter("EcBucketCreateTotal", 1L, omMetrics); + + bucketInfo = createBucketInfo(false); writeClient.createBucket(bucketInfo); - bucketInfo = createBucketInfo(); + bucketInfo = createBucketInfo(false); writeClient.createBucket(bucketInfo); - bucketInfo = createBucketInfo(); + bucketInfo = createBucketInfo(false); writeClient.createBucket(bucketInfo); writeClient.deleteBucket(bucketInfo.getVolumeName(), bucketInfo.getBucketName()); @@ -232,15 +240,24 @@ public void testBucketOps() throws Exception { mockWritePathExceptions(OmBucketInfo.class); doBucketOps(bucketInfo); + ecBucketInfo = createBucketInfo(true); + try { + writeClient.createBucket(ecBucketInfo); + } catch (Exception e) { + //Expected failure + } + omMetrics = getMetrics("OMMetrics"); + assertCounter("EcBucketCreateFailsTotal", 1L, omMetrics); + omMetrics = getMetrics("OMMetrics"); - assertCounter("NumBucketOps", 14L, omMetrics); - assertCounter("NumBucketCreates", 5L, omMetrics); + assertCounter("NumBucketOps", 17L, omMetrics); + assertCounter("NumBucketCreates", 7L, omMetrics); assertCounter("NumBucketUpdates", 2L, omMetrics); assertCounter("NumBucketInfos", 2L, omMetrics); - assertCounter("NumBucketDeletes", 3L, omMetrics); + assertCounter("NumBucketDeletes", 4L, omMetrics); assertCounter("NumBucketLists", 2L, omMetrics); - assertCounter("NumBucketCreateFails", 1L, omMetrics); + assertCounter("NumBucketCreateFails", 2L, omMetrics); assertCounter("NumBucketUpdateFails", 1L, omMetrics); assertCounter("NumBucketInfoFails", 1L, omMetrics); assertCounter("NumBucketDeleteFails", 1L, omMetrics); @@ -283,6 +300,7 @@ public void testKeyOps() throws Exception { doKeyOps(keyArgs); omMetrics = getMetrics("OMMetrics"); assertCounter("NumKeyOps", 14L, omMetrics); + assertCounter("EcKeyCreateTotal", 1L, omMetrics); keyArgs = createKeyArgs(volumeName, bucketName, RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)); @@ -331,10 +349,19 @@ public void testKeyOps() throws Exception { assertCounter("NumKeyListFails", 1L, omMetrics); assertCounter("NumTrashKeyListFails", 1L, omMetrics); assertCounter("NumInitiateMultipartUploadFails", 1L, omMetrics); - - assertCounter("NumKeys", 2L, omMetrics); + keyArgs = createKeyArgs(volumeName, bucketName, + new ECReplicationConfig("rs-3-2-1024K")); + try { + keySession = writeClient.openKey(keyArgs); + writeClient.commitKey(keyArgs, keySession.getId()); + } catch (Exception e) { + //Expected Failure + } + omMetrics = getMetrics("OMMetrics"); + assertCounter("EcKeyCreateFailsTotal", 1L, omMetrics); + cluster.restartOzoneManager(); assertCounter("NumKeys", 2L, omMetrics); @@ -601,13 +628,19 @@ private OmBucketArgs getBucketArgs(OmBucketInfo info) { .setBucketName(info.getBucketName()) .build(); } - private OmBucketInfo createBucketInfo() throws IOException { + private OmBucketInfo createBucketInfo(boolean isEcBucket) throws IOException { OmVolumeArgs volumeArgs = createVolumeArgs(); writeClient.createVolume(volumeArgs); + DefaultReplicationConfig repConf = new DefaultReplicationConfig( + new ECReplicationConfig("rs-3-2-1024k")); String bucketName = UUID.randomUUID().toString(); - return new OmBucketInfo.Builder() + + OmBucketInfo.Builder builder = new OmBucketInfo.Builder() .setVolumeName(volumeArgs.getVolume()) - .setBucketName(bucketName) - .build(); + .setBucketName(bucketName); + if (isEcBucket) { + builder.setDefaultReplicationConfig(repConf); + } + return builder.build(); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java index fbf6fe27c373..9b86eaa1896c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java @@ -197,6 +197,12 @@ public class OMMetrics { private @Metric MutableCounterLong numDirs; private @Metric MutableCounterLong numFiles; + //EC Metrics + private @Metric MutableCounterLong ecKeyCreateTotal; + private @Metric MutableCounterLong ecKeyCreateFailsTotal; + private @Metric MutableCounterLong ecBucketCreateTotal; + private @Metric MutableCounterLong ecBucketCreateFailsTotal; + private final DBCheckpointMetrics dbCheckpointMetrics; public OMMetrics() { @@ -1172,6 +1178,22 @@ public void incNumTrashAtomicDirDeletes() { numTrashAtomicDirDeletes.incr(); } + public void incEcKeysTotal() { + ecKeyCreateTotal.incr(); + } + + public void incEcBucketsTotal() { + ecBucketCreateTotal.incr(); + } + + public void incEcKeyCreateFailsTotal() { + ecKeyCreateFailsTotal.incr(); + } + + public void incEcBucketCreateFailsTotal() { + ecBucketCreateFailsTotal.incr(); + } + public void unRegister() { MetricsSystem ms = DefaultMetricsSystem.instance(); ms.unregisterSource(SOURCE_NAME); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index c2e0a1bca3bf..091632be34e7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -260,15 +260,26 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, LOG.info("created bucket: {} of layout {} in volume: {}", bucketName, omBucketInfo.getBucketLayout(), volumeName); omMetrics.incNumBuckets(); + if (isECBucket(bucketInfo)) { + omMetrics.incEcBucketsTotal(); + } return omClientResponse; } else { omMetrics.incNumBucketCreateFails(); + if (isECBucket(bucketInfo)) { + omMetrics.incEcBucketCreateFailsTotal(); + } LOG.error("Bucket creation failed for bucket:{} in volume:{}", bucketName, volumeName, exception); return omClientResponse; } } + private boolean isECBucket(BucketInfo bucketInfo) { + return bucketInfo.hasDefaultReplicationConfig() && bucketInfo + .getDefaultReplicationConfig().hasEcReplicationConfig(); + } + private BucketLayout getDefaultBucketLayout(OzoneManager ozoneManager, String volumeName, String bucketName) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index ebff5416f364..de504ea4e52f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -312,6 +312,9 @@ protected void processResult(CommitKeyRequest commitKeyRequest, if (omKeyInfo.getKeyLocationVersions().size() == 1) { omMetrics.incNumKeys(); } + if (commitKeyRequest.getKeyArgs().hasEcReplicationConfig()) { + omMetrics.incEcKeysTotal(); + } omMetrics.incDataCommittedBytes(omKeyInfo.getDataSize()); LOG.debug("Key committed. Volume:{}, Bucket:{}, Key:{}", volumeName, bucketName, keyName); @@ -319,6 +322,9 @@ protected void processResult(CommitKeyRequest commitKeyRequest, case FAILURE: LOG.error("Key commit failed. Volume:{}, Bucket:{}, Key:{}. Exception:{}", volumeName, bucketName, keyName, exception); + if (commitKeyRequest.getKeyArgs().hasEcReplicationConfig()) { + omMetrics.incEcKeyCreateFailsTotal(); + } omMetrics.incNumKeyCommitFails(); break; default: diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index f52607551b9e..ced8b7c48998 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -371,6 +371,9 @@ protected void logResult(CreateKeyRequest createKeyRequest, createKeyRequest.getKeyArgs().getKeyName()); break; case FAILURE: + if (createKeyRequest.getKeyArgs().hasEcReplicationConfig()) { + omMetrics.incEcKeyCreateFailsTotal(); + } LOG.error("Key creation failed. Volume:{}, Bucket:{}, Key:{}. ", createKeyRequest.getKeyArgs().getVolumeName(), createKeyRequest.getKeyArgs().getBucketName(), From 5c2a39365aa8040dbe5bce6043af199ca1cf1d79 Mon Sep 17 00:00:00 2001 From: zhtttylz <54506205+zhtttylz@users.noreply.github.com> Date: Fri, 21 Oct 2022 03:53:35 +0800 Subject: [PATCH 14/48] HDDS-7369. Fix wrong order of command arguments in Nonrolling-Upgrade.md (#3866) --- hadoop-hdds/docs/content/feature/Nonrolling-Upgrade.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/docs/content/feature/Nonrolling-Upgrade.md b/hadoop-hdds/docs/content/feature/Nonrolling-Upgrade.md index 0328e6834b9a..b18f585aebbe 100644 --- a/hadoop-hdds/docs/content/feature/Nonrolling-Upgrade.md +++ b/hadoop-hdds/docs/content/feature/Nonrolling-Upgrade.md @@ -62,15 +62,15 @@ Starting with your current version of Ozone, complete the following steps to upg 4. Start the components 1. Start the SCM and datanodes as usual: ``` - ozone --daemon scm start + ozone --daemon start scm ``` ``` - ozone --daemon datanode start + ozone --daemon start datanode ``` 2. Start the Ozone Manager using the `--upgrade` flag to take it out of prepare mode. ``` - ozone --daemon om start --upgrade + ozone --daemon start om --upgrade ``` - There also exists a `--downgrade` flag which is an alias of `--upgrade`. The name used does not matter. From 13a6d0178e0225f5ecab098b700f75de663ba811 Mon Sep 17 00:00:00 2001 From: smitajoshi12 <112169209+smitajoshi12@users.noreply.github.com> Date: Fri, 21 Oct 2022 12:36:23 +0530 Subject: [PATCH 15/48] HDDS-7141. Recon: Improve Disk Usage Page (#3789) --- .../src/views/diskUsage/diskUsage.less | 36 +++++++++- .../src/views/diskUsage/diskUsage.tsx | 68 +++++++++++-------- 2 files changed, 71 insertions(+), 33 deletions(-) diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.less index fced2c5ba426..154a959710bd 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.less +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.less @@ -38,9 +38,28 @@ margin-right: 25px; } - h3 { - display: inline-block; - padding-right: 5px; + .legendtext { + fill: rgba(0, 0, 0, 0.65) !important; + line-height: 1.5; + font-size: 16px !important; + font-feature-settings: 'tnum', "tnum"; + font-variant: tabular-nums; + } + + .gtitle { + font-size: 22px; + font-weight: 500 !important; + fill: rgba(0, 0, 0, 0.60) !important; + font-feature-settings: 'tnum', "tnum"; + font-variant: tabular-nums; + } + + .slicetext { + fill: rgba(0, 0, 0, 0.60) !important; + font-size: 17px !important; + line-height: 1.5; + font-feature-settings: 'tnum', "tnum"; + font-variant: tabular-nums; } } @@ -49,4 +68,15 @@ margin-right: 10px; display: inline-block; width: 300px; +} + +.metadatainformation { + font-size: 14px; + fill: rgba(0, 0, 0, 0.65) !important; + line-height: 1.5; + font-feature-settings: 'tnum', "tnum"; +} + +.ant-drawer-open .ant-drawer-mask { + opacity: 0 !important; } \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.tsx index 872efe0df131..357567f77c3d 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.tsx +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.tsx @@ -152,26 +152,39 @@ export class DiskUsage extends React.Component, IDUState> subpaths.push(other); } - const pathLabels = subpaths.map(subpath => { - // The return subPath must be normalized in a format with - // a leading slash and without trailing slash - const pieces = subpath.path.split('/'); - const subpathName = pieces[pieces.length - 1]; - // Differentiate key without trailing slash - return (subpath.isKey || subpathName === OTHER_PATH_NAME) ? subpathName : subpathName + '/'; - }); + let pathLabels, values, percentage, sizeStr, pieces, subpathName; + + if (duResponse.subPathCount === 0 || subpaths === 0) { + pieces = duResponse && duResponse.path != null && duResponse.path.split('/'); + subpathName = pieces[pieces.length - 1]; + pathLabels = [subpathName]; + values = [0.1]; + percentage = [100.00]; + sizeStr = [this.byteToSize(duResponse.size, 1)]; + } + else { + pathLabels = subpaths.map(subpath => { + // The return subPath must be normalized in a format with + // a leading slash and without trailing slash + pieces = subpath.path.split('/'); + subpathName = pieces[pieces.length - 1]; + // Differentiate key without trailing slash + return (subpath.isKey || subpathName === OTHER_PATH_NAME) ? subpathName : subpathName + '/'; + }); - const values = subpaths.map(subpath => { - return subpath.size / dataSize; - }); + values = subpaths.map(subpath => { + return subpath.size / dataSize; + }); - const percentage = values.map(value => { - return (value * 100).toFixed(2); - }); + percentage = values.map(value => { + return (value * 100).toFixed(2); + }); - const sizeStr = subpaths.map(subpath => { - return this.byteToSize(subpath.size, 1); - }); + sizeStr = subpaths.map(subpath => { + return this.byteToSize(subpath.size, 1); + }); + } + this.setState({ // Normalized path isLoading: false, @@ -407,13 +420,7 @@ export class DiskUsage extends React.Component, IDUState> {(duResponse.size > 0) ? - ((duResponse.size > 0 && duResponse.subPathCount === 0) ? -

-
{' '} -

This object is a key with a file size of {this.byteToSize(duResponse.size, 1)}.
{' '} - You can also view its metadata details by clicking the top right button. -

-
: +
, IDUState> width: 800, height: 750, font: { - family: 'Arial', - size: 14 + family: 'Roboto, sans-serif', + size: 15 }, showlegend: true, title: 'Disk Usage for ' + returnPath + ' (Total Size: ' + this.byteToSize(duResponse.size, 1) + ')' } } - onClick={e => this.clickPieSection(e, returnPath)}/>) : -

-

This object is empty. Add files to it to see a visualization on disk usage.
{' '} + onClick={(duResponse.subPathCount === 0) ? undefined : e => this.clickPieSection(e, returnPath)}/> +

+ : +

+ This object is empty. Add files to it to see a visualization on disk usage.{' '}
You can also view its metadata details by clicking the top right button. -
} From ae59f8ae5f2149ba1a7749bd3073803345333742 Mon Sep 17 00:00:00 2001 From: smitajoshi12 <112169209+smitajoshi12@users.noreply.github.com> Date: Fri, 21 Oct 2022 12:41:45 +0530 Subject: [PATCH 16/48] HDDS-7248. Recon: Expand the container status page to show all unhealthy container states (#3837) --- .../ozone/recon/api/ContainerEndpoint.java | 1 + .../webapps/recon/ozone-recon-web/api/db.json | 330 +++++++++++++++++- .../recon/ozone-recon-web/api/routes.json | 8 +- .../src/components/navBar/navBar.tsx | 5 + .../missingContainers/missingContainers.tsx | 227 ++++++++---- 5 files changed, 502 insertions(+), 69 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java index 0832700e9604..ce8cc7058942 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java @@ -242,6 +242,7 @@ public Response getReplicaHistoryForContainer( */ @GET @Path("/missing") + @Deprecated public Response getMissingContainers( @DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT) int limit diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json index 04036416f413..1639302e23c5 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json @@ -1260,5 +1260,333 @@ "lastUpdatedTimestamp": 1663421094507, "lastUpdatedSeqNumber": 0 } - ] + ], + "containerUnhealthyMissing": { + "missingCount": 0, + "underReplicatedCount": 1, + "overReplicatedCount": 0, + "misReplicatedCount": 0, + "containers": [ + { + "containerID": 1, + "containerState": "MISSING", + "unhealthySince": 1665590446222, + "expectedReplicaCount": 3, + "actualReplicaCount": 0, + "replicaDeltaCount": 3, + "reason": null, + "keys": 1, + "pipelineID": "a10ffab6-8ed5-414a-aaf5-79890ff3e8a1", + "replicas": [ + { + "containerId": 2, + "datanodeUuid": "15526f1b-76f2-4d8f-876c-c343c94ea476", + "datanodeHost": "ozone_datanode_2.ozone_missing1", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590397315, + "lastBcsId": 2 + }, + { + "containerId": 2, + "datanodeUuid": "f55476ab-4687-464d-a100-1c65de4366e3", + "datanodeHost": "ozone_datanode_3.ozone_missing2", + "firstSeenTime": 1665588176616, + "lastSeenTime": 1665590392293, + "lastBcsId": 2 + }, + { + "containerId": 2, + "datanodeUuid": "7a457bcb-d63e-49cc-b3ff-8b22bf48d130", + "datanodeHost": "ozone_datanode_1.ozone_missing3", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590272289, + "lastBcsId": 0 + } + ] + }, + { + "containerID": 2, + "containerState": "MISSING", + "unhealthySince": 1665590446222, + "expectedReplicaCount": 3, + "actualReplicaCount": 0, + "replicaDeltaCount": 3, + "reason": null, + "keys": 1, + "pipelineID": "a10ffab6-8ed5-414a-aaf5-79890ff3e8a1", + "replicas": [ + { + "containerId": 3, + "datanodeUuid": "15526f1b-76f2-4d8f-876c-c343c94ea476", + "datanodeHost": "ozone_datanode_2.ozone_missing1", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590397315, + "lastBcsId": 2 + }, + { + "containerId": 3, + "datanodeUuid": "f55476ab-4687-464d-a100-1c65de4366e3", + "datanodeHost": "ozone_datanode_3.ozone_missing2", + "firstSeenTime": 1665588176616, + "lastSeenTime": 1665590392293, + "lastBcsId": 2 + }, + { + "containerId": 3, + "datanodeUuid": "7a457bcb-d63e-49cc-b3ff-8b22bf48d130", + "datanodeHost": "ozone_datanode_1.ozone_missing3", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590272289, + "lastBcsId": 0 + } + ] + } + ] + }, + "containerUnhealthyUnderReplicated": { + "missingCount": 0, + "underReplicatedCount": 1, + "overReplicatedCount": 0, + "misReplicatedCount": 0, + "containers": [ + { + "containerID": 2, + "containerState": "UNDER_REPLICATED", + "unhealthySince": 1665590446222, + "expectedReplicaCount": 3, + "actualReplicaCount": 2, + "replicaDeltaCount": 1, + "reason": null, + "keys": 1, + "pipelineID": "a10ffab6-8ed5-414a-aaf5-79890ff3e8a1", + "replicas": [ + { + "containerId": 2, + "datanodeUuid": "15526f1b-76f2-4d8f-876c-c343c94ea476", + "datanodeHost": "ozone_datanode_2.ozone_UnderReplicated2", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590397315, + "lastBcsId": 2 + }, + { + "containerId": 2, + "datanodeUuid": "f55476ab-4687-464d-a100-1c65de4366e3", + "datanodeHost": "ozone_datanode_3.ozone_underreplicated2", + "firstSeenTime": 1665588176616, + "lastSeenTime": 1665590392293, + "lastBcsId": 2 + }, + { + "containerId": 2, + "datanodeUuid": "7a457bcb-d63e-49cc-b3ff-8b22bf48d130", + "datanodeHost": "ozone_datanode_1.ozone_underreplicated2", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590272289, + "lastBcsId": 0 + } + ] + }, + { + "containerID": 3, + "containerState": "UNDER_REPLICATED", + "unhealthySince": 1665590446222, + "expectedReplicaCount": 4, + "actualReplicaCount": 2, + "replicaDeltaCount": 2, + "reason": null, + "keys": 1, + "pipelineID": "a10ffab6-8ed5-414a-aaf5-79890ff3e8a1", + "replicas": [ + { + "containerId": 3, + "datanodeUuid": "15526f1b-76f2-4d8f-876c-c343c94ea476", + "datanodeHost": "ozone_datanode_2.ozone_underreplicated3", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590397315, + "lastBcsId": 2 + }, + { + "containerId": 3, + "datanodeUuid": "f55476ab-4687-464d-a100-1c65de4366e3", + "datanodeHost": "ozone_datanode_3.ozone_underreplicated3", + "firstSeenTime": 1665588176616, + "lastSeenTime": 1665590392293, + "lastBcsId": 2 + }, + { + "containerId": 3, + "datanodeUuid": "7a457bcb-d63e-49cc-b3ff-8b22bf48d130", + "datanodeHost": "ozone_datanode_1.ozone_underreplicated3", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590272289, + "lastBcsId": 0 + } + ] + } + ] + }, + "containerUnhealthyOverReplicated": { + "missingCount": 0, + "underReplicatedCount": 1, + "overReplicatedCount": 0, + "misReplicatedCount": 0, + "containers": [ + { + "containerID": 2, + "containerState": "OVER_REPLICATED", + "unhealthySince": 1665590446222, + "expectedReplicaCount": 3, + "actualReplicaCount": 2, + "replicaDeltaCount": 1, + "reason": null, + "keys": 1, + "pipelineID": "a10ffab6-8ed5-414a-aaf5-79890ff3e8a1", + "replicas": [ + { + "containerId": 2, + "datanodeUuid": "15526f1b-76f2-4d8f-876c-c343c94ea476", + "datanodeHost": "ozone_datanode_2.ozone_overreplicated2", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590397315, + "lastBcsId": 2 + }, + { + "containerId": 2, + "datanodeUuid": "f55476ab-4687-464d-a100-1c65de4366e3", + "datanodeHost": "ozone_datanode_3.ozone_overreplicated22", + "firstSeenTime": 1665588176616, + "lastSeenTime": 1665590392293, + "lastBcsId": 2 + }, + { + "containerId": 2, + "datanodeUuid": "7a457bcb-d63e-49cc-b3ff-8b22bf48d130", + "datanodeHost": "ozone_datanode_1.ozone_overreplicated2", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590272289, + "lastBcsId": 0 + } + ] + }, + { + "containerID": 3, + "containerState": "OVER_REPLICATED", + "unhealthySince": 1665590446222, + "expectedReplicaCount": 4, + "actualReplicaCount": 2, + "replicaDeltaCount": 2, + "reason": null, + "keys": 1, + "pipelineID": "a10ffab6-8ed5-414a-aaf5-79890ff3e8a1", + "replicas": [ + { + "containerId": 3, + "datanodeUuid": "15526f1b-76f2-4d8f-876c-c343c94ea476", + "datanodeHost": "ozone_datanode_2.ozone_overreplicated3", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590397315, + "lastBcsId": 2 + }, + { + "containerId": 3, + "datanodeUuid": "f55476ab-4687-464d-a100-1c65de4366e3", + "datanodeHost": "ozone_datanode_3.ozone_overreplicated3", + "firstSeenTime": 1665588176616, + "lastSeenTime": 1665590392293, + "lastBcsId": 2 + }, + { + "containerId": 3, + "datanodeUuid": "7a457bcb-d63e-49cc-b3ff-8b22bf48d130", + "datanodeHost": "ozone_datanode_1.ozone_overreplicated3", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590272289, + "lastBcsId": 0 + } + ] + } + ] + }, + "containerUnhealthyMisReplicated": { + "missingCount": 0, + "underReplicatedCount": 1, + "overReplicatedCount": 0, + "misReplicatedCount": 0, + "containers": [ + { + "containerID": 2, + "containerState": "MIS_REPLICATED", + "unhealthySince": 1665590446222, + "expectedReplicaCount": 3, + "actualReplicaCount": 2, + "replicaDeltaCount": 1, + "reason": null, + "keys": 1, + "pipelineID": "a10ffab6-8ed5-414a-aaf5-79890ff3e8a1", + "replicas": [ + { + "containerId": 2, + "datanodeUuid": "15526f1b-76f2-4d8f-876c-c343c94ea476", + "datanodeHost": "ozone_datanode_2.ozone_misreplicated2", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590397315, + "lastBcsId": 2 + }, + { + "containerId": 2, + "datanodeUuid": "f55476ab-4687-464d-a100-1c65de4366e3", + "datanodeHost": "ozone_datanode_3.ozone_misreplicated2", + "firstSeenTime": 1665588176616, + "lastSeenTime": 1665590392293, + "lastBcsId": 2 + }, + { + "containerId": 2, + "datanodeUuid": "7a457bcb-d63e-49cc-b3ff-8b22bf48d130", + "datanodeHost": "ozone_datanode_1.ozone_misreplicated2", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590272289, + "lastBcsId": 0 + } + ] + }, + { + "containerID": 3, + "containerState": "MIS_REPLICATED", + "unhealthySince": 1665590446222, + "expectedReplicaCount": 4, + "actualReplicaCount": 2, + "replicaDeltaCount": 2, + "reason": null, + "keys": 1, + "pipelineID": "a10ffab6-8ed5-414a-aaf5-79890ff3e8a1", + "replicas": [ + { + "containerId": 3, + "datanodeUuid": "15526f1b-76f2-4d8f-876c-c343c94ea476", + "datanodeHost": "ozone_datanode_2.ozone_misreplicated3", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590397315, + "lastBcsId": 2 + }, + { + "containerId": 3, + "datanodeUuid": "f55476ab-4687-464d-a100-1c65de4366e3", + "datanodeHost": "ozone_datanode_3.ozone_misreplicated3", + "firstSeenTime": 1665588176616, + "lastSeenTime": 1665590392293, + "lastBcsId": 2 + }, + { + "containerId": 3, + "datanodeUuid": "7a457bcb-d63e-49cc-b3ff-8b22bf48d130", + "datanodeHost": "ozone_datanode_1.ozone_misreplicated3", + "firstSeenTime": 1665588176660, + "lastSeenTime": 1665590272289, + "lastBcsId": 0 + } + ] + } + ] + } } \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json index 4aa303fe29d9..0b0a4bb9d69c 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json @@ -1,7 +1,6 @@ { "/api/v1/*": "/$1", "/containers/:id/keys": "/keys", - "/containers/missing": "/missingContainers", "/utilization/fileCount": "/fileSizeCounts", "/namespace/du?path=/&files=true": "/root", "/namespace/du?path=/vol:id&files=true": "/volume", @@ -23,5 +22,10 @@ "/namespace/du?path=/clunky&files=true": "/clunky", "/namespace/summary?path=*": "/metadata", "/namespace/quota?path=*": "/quota", - "/task/status": "/taskStatus" + "/task/status": "/taskStatus", + "/containers/missing": "/missingContainers", + "/containers/unhealthy/MISSING": "/containerUnhealthyMissing", + "/containers/unhealthy/UNDER_REPLICATED": "/containerUnhealthyUnderReplicated", + "/containers/unhealthy/OVER_REPLICATED": "/containerUnhealthyOverReplicated", + "/containers/unhealthy/MIS_REPLICATED": "/containerUnhealthyMisReplicated" } \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/navBar/navBar.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/navBar/navBar.tsx index acc055040448..d381e8e4d41f 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/navBar/navBar.tsx +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/navBar/navBar.tsx @@ -66,6 +66,11 @@ class NavBar extends React.Component { Pipelines + + + Containers + + Insights diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.tsx index 6a6dca304dcb..c6a76b5b97d2 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.tsx +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/missingContainers/missingContainers.tsx @@ -18,7 +18,7 @@ import React from 'react'; import axios from 'axios'; -import {Icon, Table, Tooltip} from 'antd'; +import {Icon, Table, Tooltip, Tabs} from 'antd'; import {PaginationConfig} from 'antd/lib/pagination'; import filesize from 'filesize'; import moment from 'moment'; @@ -26,6 +26,7 @@ import {showDataFetchError, timeFormat} from 'utils/common'; import './missingContainers.less'; const size = filesize.partial({standard: 'iec'}); +const {TabPane} = Tabs; interface IMissingContainerResponse { containerID: number; @@ -35,6 +36,19 @@ interface IMissingContainerResponse { pipelineID: string; } +interface IContainerResponse { + containerID: number; + containerState: string; + unhealthySince: string; + expectedReplicaCount: number; + actualReplicaCount: number; + replicaDeltaCount: number; + reason: string; + keys: number; + pipelineID: string; + replicas: IContainerReplicas[]; +} + export interface IContainerReplica { containerId: number; datanodeHost: string; @@ -42,11 +56,28 @@ export interface IContainerReplica { lastReportTimestamp: number; } +export interface IContainerReplicas { + containerId: number; + datanodeUuid: string; + datanodeHost: string; + firstSeenTime: number; + lastSeenTime: number; + lastBcsId: number; +} + export interface IMissingContainersResponse { totalCount: number; containers: IMissingContainerResponse[]; } +interface IUnhealthyContainersResponse { + missingCount: number; + underReplicatedCount: number; + overReplicatedCount: number; + misReplicatedCount: number; + containers: IContainerResponse[]; +} + interface IKeyResponse { Volume: string; Bucket: string; @@ -63,30 +94,79 @@ interface IContainerKeysResponse { keys: IKeyResponse[]; } -const COLUMNS = [ +const KEY_TABLE_COLUMNS = [ + { + title: 'Volume', + dataIndex: 'Volume', + key: 'Volume' + }, + { + title: 'Bucket', + dataIndex: 'Bucket', + key: 'Bucket' + }, + { + title: 'Key', + dataIndex: 'Key', + key: 'Key' + }, + { + title: 'Size', + dataIndex: 'DataSize', + key: 'DataSize', + render: (dataSize: number) =>
{size(dataSize)}
+ }, + { + title: 'Date Created', + dataIndex: 'CreationTime', + key: 'CreationTime', + render: (date: string) => moment(date).format('lll') + }, + { + title: 'Date Modified', + dataIndex: 'ModificationTime', + key: 'ModificationTime', + render: (date: string) => moment(date).format('lll') + } +]; + +const CONTAINER_TAB_COLUMNS = [ { title: 'Container ID', dataIndex: 'containerID', key: 'containerID', - sorter: (a: IMissingContainerResponse, b: IMissingContainerResponse) => a.containerID - b.containerID + sorter: (a: IContainerResponse, b: IContainerResponse) => a.containerID - b.containerID }, { title: 'No. of Keys', dataIndex: 'keys', key: 'keys', - sorter: (a: IMissingContainerResponse, b: IMissingContainerResponse) => a.keys - b.keys + sorter: (a: IContainerResponse, b: IContainerResponse) => a.keys - b.keys + }, + { + title: 'Active/Expected Replica(s)', + dataIndex: 'expectedReplicaCount', + key: 'expectedReplicaCount', + render: (expectedReplicaCount: number, record: IContainerResponse) => { + const actualReplicaCount = record.actualReplicaCount; + return ( + + {actualReplicaCount} / {expectedReplicaCount} + + ); + } }, { title: 'Datanodes', dataIndex: 'replicas', key: 'replicas', - render: (replicas: IContainerReplica[]) => ( + render: (replicas: IContainerReplicas[]) => (
- {replicas.map(replica => { + {replicas && replicas.map(replica => { const tooltip = (
-
First Report Time: {timeFormat(replica.firstReportTimestamp)}
-
Last Report Time: {timeFormat(replica.lastReportTimestamp)}
+
First Report Time: {timeFormat(replica.firstSeenTime)}
+
Last Report Time: {timeFormat(replica.lastSeenTime)}
); return ( @@ -111,50 +191,14 @@ const COLUMNS = [ title: 'Pipeline ID', dataIndex: 'pipelineID', key: 'pipelineID', - sorter: (a: IMissingContainerResponse, b: IMissingContainerResponse) => a.pipelineID.localeCompare(b.pipelineID) - }, - { - title: 'Missing Since', - dataIndex: 'missingSince', - key: 'missingSince', - render: (missingSince: number) => timeFormat(missingSince), - sorter: (a: IMissingContainerResponse, b: IMissingContainerResponse) => a.missingSince - b.missingSince - } -]; - -const KEY_TABLE_COLUMNS = [ - { - title: 'Volume', - dataIndex: 'Volume', - key: 'Volume' - }, - { - title: 'Bucket', - dataIndex: 'Bucket', - key: 'Bucket' - }, - { - title: 'Key', - dataIndex: 'Key', - key: 'Key' - }, - { - title: 'Size', - dataIndex: 'DataSize', - key: 'DataSize', - render: (dataSize: number) =>
{size(dataSize)}
+ sorter: (a: IContainerResponse, b: IContainerResponse) => a.pipelineID.localeCompare(b.pipelineID) }, { - title: 'Date Created', - dataIndex: 'CreationTime', - key: 'CreationTime', - render: (date: string) => moment(date).format('lll') - }, - { - title: 'Date Modified', - dataIndex: 'ModificationTime', - key: 'ModificationTime', - render: (date: string) => moment(date).format('lll') + title: 'Unhealthy Since', + dataIndex: 'unhealthySince', + key: 'unhealthySince', + render: (unhealthySince: number) => timeFormat(unhealthySince), + sorter: (a: IContainerResponse, b: IContainerResponse) => a.unhealthySince - b.unhealthySince } ]; @@ -171,7 +215,10 @@ interface IExpandedRowState { interface IMissingContainersState { loading: boolean; - dataSource: IMissingContainerResponse[]; + missingDataSource: IContainerResponse[]; + underReplicatedDataSource: IContainerResponse[]; + overReplicatedDataSource: IContainerResponse[]; + misReplicatedDataSource: IContainerResponse[]; totalCount: number; expandedRowData: IExpandedRow; } @@ -181,7 +228,10 @@ export class MissingContainers extends React.Component, I super(props); this.state = { loading: false, - dataSource: [], + missingDataSource: [], + underReplicatedDataSource: [], + overReplicatedDataSource: [], + misReplicatedDataSource: [], totalCount: 0, expandedRowData: {} }; @@ -192,16 +242,36 @@ export class MissingContainers extends React.Component, I this.setState({ loading: true }); - axios.get('/api/v1/containers/missing').then(response => { - const missingContainersResponse: IMissingContainersResponse = response.data; - const totalCount = missingContainersResponse.totalCount; - const missingContainers: IMissingContainerResponse[] = missingContainersResponse.containers; + + axios.all([ + axios.get('/api/v1/containers/unhealthy/MISSING'), + axios.get('/api/v1/containers/unhealthy/UNDER_REPLICATED'), + axios.get('/api/v1/containers/unhealthy/OVER_REPLICATED'), + axios.get('/api/v1/containers/unhealthy/MIS_REPLICATED') + ]).then(axios.spread((missingContainersResponse, underReplicatedResponse, overReplicatedResponse, misReplicatedResponse, allReplicatedResponse) => { + + const missingContainersResponseData: IUnhealthyContainersResponse = missingContainersResponse.data; + const totalCount = missingContainersResponseData.missingCount; + const missingContainers: IContainerResponse[] = missingContainersResponseData.containers; + + const underReplicatedResponseData: IUnhealthyContainersResponse = underReplicatedResponse.data; + const uContainers: IContainerResponse[] = underReplicatedResponseData.containers; + + const overReplicatedResponseData: IUnhealthyContainersResponse = overReplicatedResponse.data; + const oContainers: IContainerResponse[] = overReplicatedResponseData.containers; + + const misReplicatedResponseData: IUnhealthyContainersResponse = misReplicatedResponse.data; + const mContainers: IContainerResponse[] = misReplicatedResponseData.containers; + this.setState({ loading: false, - dataSource: missingContainers, + missingDataSource: missingContainers, + underReplicatedDataSource: uContainers, + overReplicatedDataSource: oContainers, + misReplicatedDataSource: mContainers, totalCount }); - }).catch(error => { + })).catch(error => { this.setState({ loading: false }); @@ -212,7 +282,7 @@ export class MissingContainers extends React.Component, I onShowSizeChange = (current: number, pageSize: number) => { console.log(current, pageSize); }; - + onRowExpandClick = (expanded: boolean, record: IMissingContainerResponse) => { if (expanded) { this.setState(({expandedRowData}) => { @@ -270,7 +340,7 @@ export class MissingContainers extends React.Component, I }; render() { - const {dataSource, loading, totalCount} = this.state; + const {missingDataSource, loading, underReplicatedDataSource, overReplicatedDataSource, misReplicatedDataSource} = this.state; const paginationConfig: PaginationConfig = { showTotal: (total: number, range) => `${range[0]}-${range[1]} of ${total} missing containers`, showSizeChanger: true, @@ -279,14 +349,39 @@ export class MissingContainers extends React.Component, I return (
- Missing Containers ({totalCount}) + Containers
- + + +
+ + +
+ + +
+ + +
+ + ); From ecdfc207562e352bd1bcff52ca55b5c25456907f Mon Sep 17 00:00:00 2001 From: DaveTeng0 <109315747+DaveTeng0@users.noreply.github.com> Date: Sat, 22 Oct 2022 09:12:05 -0700 Subject: [PATCH 17/48] HDDS-7199. Implement new mix workload Read/Write Freon command (#3872) --- .../main/smoketest/freon/read-write-key.robot | 53 ++++ .../org/apache/hadoop/ozone/freon/Freon.java | 2 + .../hadoop/ozone/freon/KeyGeneratorUtil.java | 50 ++++ .../freon/OzoneClientKeyReadWriteOps.java | 245 ++++++++++++++++++ .../ozone/freon/RangeKeysGenerator.java | 164 ++++++++++++ 5 files changed, 514 insertions(+) create mode 100644 hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java diff --git a/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot new file mode 100644 index 000000000000..0995f757f35d --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test freon read/write key commands +Resource ../ozone-lib/freon.robot +Test Timeout 5 minutes + +*** Variables *** +${PREFIX} ${EMPTY} + + +*** Test Cases *** +Pre-generate 100 keys of size 1 byte each to Ozone + ${result} = Execute ozone freon ork -n 1 -t 10 -r 100 --size 1 -v voltest -b buckettest -p performanceTest + +Read 10 keys from pre-generated keys + ${keysCount} = BuiltIn.Set Variable 10 + ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 -r 100 -v voltest -b buckettest -p performanceTest + Should contain ${result} Successful executions: ${keysCount} + +Read 10 keys' metadata from pre-generated keys + ${keysCount} = BuiltIn.Set Variable 10 + ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 -m -r 100 -v voltest -b buckettest -p performanceTest + Should contain ${result} Successful executions: ${keysCount} + +Write 10 keys of size 1 byte each from key index 0 to 99 + ${keysCount} = BuiltIn.Set Variable 10 + ${size} = BuiltIn.Set Variable 1 + ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 --percentage-read 0 --size ${size} -r 100 -v voltest -b buckettest -p performanceTest + Should contain ${result} Successful executions: ${keysCount} + ${keyName} = Execute echo -n '1' | md5sum | head -c 7 + ${result} = Execute ozone sh key info /voltest/buckettest/performanceTest/${keyName} + Should contain ${result} \"dataSize\" : 1 + + +Run 90 % of read-key tasks and 10 % of write-key tasks for 10 keys from pre-generated keys + ${keysCount} = BuiltIn.Set Variable 10 + ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 --percentage-read 90 -r 100 -v voltest -b buckettest -p performanceTest + Should contain ${result} Successful executions: ${keysCount} + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java index b6bb6055eefe..dfb52ca9add6 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java @@ -68,6 +68,8 @@ OmBucketReadWriteFileOps.class, OmBucketReadWriteKeyOps.class, OmRPCLoadGenerator.class, + OzoneClientKeyReadWriteOps.class, + RangeKeysGenerator.class }, versionProvider = HddsVersionProvider.class, mixinStandardHelpOptions = true) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java new file mode 100644 index 000000000000..9773caa44070 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.freon; + +import org.apache.commons.codec.digest.DigestUtils; + +import java.util.function.Function; + +/** + * Utility class to generate key name from a given key index. + */ +public class KeyGeneratorUtil { + public static final String PURE_INDEX = "pureIndex"; + public static final String MD5 = "md5"; + public static final String FILE_DIR_SEPARATOR = "/"; + + public String generatePureIndexKeyName(int number) { + return String.valueOf(number); + } + public Function pureIndexKeyNameFunc() { + return number -> String.valueOf(number); + } + + public String generateMd5KeyName(int number) { + String encodedStr = DigestUtils.md5Hex(String.valueOf(number)); + return encodedStr.substring(0, 7); + } + + public Function md5KeyNameFunc() { + return number -> DigestUtils.md5Hex(String.valueOf(number)).substring(0, 7); + } + +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java new file mode 100644 index 000000000000..88aec0763ce6 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.freon; + + +import com.codahale.metrics.Timer; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneKeyDetails; +import org.apache.hadoop.ozone.client.io.OzoneInputStream; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; + +import java.io.IOException; +import java.util.concurrent.Callable; +import java.util.concurrent.ThreadLocalRandom; +import java.util.HashMap; + +import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR; + +/** + * Ozone key generator/reader for performance test. + */ + +@CommandLine.Command(name = "ockrw", + aliases = "ozone-client-key-read-write-ops", + description = "Generate keys with a fixed name and ranges that can" + + " be written and read as sub-ranges from multiple clients.", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true, + showDefaultValues = true) +public class OzoneClientKeyReadWriteOps extends BaseFreonGenerator + implements Callable { + + @CommandLine.Option(names = {"-v", "--volume"}, + description = "Name of the volume which contains the test data. " + + "Will be created if missing.", + defaultValue = "ockrwvolume") + private String volumeName; + + @CommandLine.Option(names = {"-b", "--bucket"}, + description = "Name of the bucket which contains the test data.", + defaultValue = "ockrwbucket") + private String bucketName; + + @CommandLine.Option(names = {"-m", "--read-metadata-only"}, + description = "If only read key's metadata.", + defaultValue = "false") + private boolean readMetadataOnly; + + @CommandLine.Option(names = {"-s", "--start-index"}, + description = "Start index of keys of read/write operation." + + "This can allow adding keys incrementally or parallel from multiple" + + " clients. Example: Write keys 0-1000000 followed by " + + "keys 1000001-2000000.", + defaultValue = "0") + private int startIndex; + + @CommandLine.Option(names = {"-r", "--range"}, + description = "Range of read/write operations. This in co-ordination" + + " with --start-index can specify the range to read. " + + "Example: Read from --start-index 1000 and read --range 1000 keys.", + defaultValue = "1") + private int range; + + @CommandLine.Option(names = {"--size"}, + description = "Object size (in bytes) " + + "to read/write. If user sets a read size which is larger" + + " than the key size, it only reads bytes up to key size.", + defaultValue = "1") + private int objectSizeInBytes; + + @CommandLine.Option(names = {"--contiguous"}, + description = "By default, the keys are randomized lexically" + + " by calculating the md5 of the key name. If this option is set," + + " the keys are written lexically contiguously.", + defaultValue = "false") + private boolean keySorted; + + @CommandLine.Option(names = {"--percentage-read"}, + description = "Percentage of read tasks in mix workload." + + " The remainder of the percentage will writes to keys." + + " Example --percentage-read 90 will result in 10% writes.", + defaultValue = "100") + private int percentageRead; + + @CommandLine.Option( + names = "--om-service-id", + description = "OM Service ID" + ) + private String omServiceID = null; + + private Timer timer; + + private OzoneClient[] ozoneClients; + + private int clientCount; + + private byte[] keyContent; + + private static final Logger LOG = + LoggerFactory.getLogger(OzoneClientKeyReadWriteOps.class); + + /** + * Task type of read task, or write task. + */ + public enum TaskType { + READ_TASK, + WRITE_TASK + } + private KeyGeneratorUtil kg; + + + @Override + public Void call() throws Exception { + init(); + OzoneConfiguration ozoneConfiguration = createOzoneConfiguration(); + clientCount = getThreadNo(); + ozoneClients = new OzoneClient[clientCount]; + for (int i = 0; i < clientCount; i++) { + ozoneClients[i] = createOzoneClient(omServiceID, ozoneConfiguration); + } + + ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName); + + timer = getMetrics().timer("key-read-write"); + if (objectSizeInBytes >= 0) { + keyContent = RandomUtils.nextBytes(objectSizeInBytes); + } + if (kg == null) { + kg = new KeyGeneratorUtil(); + } + runTests(this::readWriteKeys); + + for (int i = 0; i < clientCount; i++) { + if (ozoneClients[i] != null) { + ozoneClients[i].close(); + } + } + return null; + } + + public void readWriteKeys(long counter) throws RuntimeException, IOException { + int clientIndex = (int)((counter) % clientCount); + TaskType taskType = decideReadOrWriteTask(); + String keyName = getKeyName(); + + timer.time(() -> { + try { + switch (taskType) { + case READ_TASK: + processReadTasks(keyName, ozoneClients[clientIndex]); + break; + case WRITE_TASK: + processWriteTasks(keyName, ozoneClients[clientIndex]); + break; + default: + break; + } + } catch (RuntimeException ex) { + LOG.error(ex.getMessage()); + throw ex; + } catch (IOException ex) { + LOG.error(ex.getMessage()); + throw new RuntimeException(ex.getMessage()); + } + + }); + } + @SuppressFBWarnings + public void processReadTasks(String keyName, OzoneClient client) + throws RuntimeException, IOException { + OzoneKeyDetails keyDetails = client.getProxy(). + getKeyDetails(volumeName, bucketName, keyName); + if (!readMetadataOnly) { + byte[] data = new byte[objectSizeInBytes]; + try (OzoneInputStream introStream = keyDetails.getContent()) { + introStream.read(data); + } catch (Exception ex) { + throw ex; + } + } + } + + public void processWriteTasks(String keyName, OzoneClient ozoneClient) + throws RuntimeException, IOException { + try (OzoneOutputStream out = + ozoneClient.getProxy().createKey(volumeName, bucketName, + keyName, objectSizeInBytes, null, new HashMap())) { + out.write(keyContent); + } catch (Exception ex) { + throw ex; + } + } + + public TaskType decideReadOrWriteTask() { + if (percentageRead == 100) { + return TaskType.READ_TASK; + } else if (percentageRead == 0) { + return TaskType.WRITE_TASK; + } + //mix workload + int tmp = ThreadLocalRandom.current().nextInt(1, 101); + if (tmp <= percentageRead) { + return TaskType.READ_TASK; + } else { + return TaskType.WRITE_TASK; + } + } + + public String getKeyName() { + StringBuilder keyNameSb = new StringBuilder(); + int randomIdxWithinRange = ThreadLocalRandom.current(). + nextInt(startIndex, startIndex + range); + + if (keySorted) { + keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR). + append(randomIdxWithinRange); + } else { + keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR). + append(kg.generateMd5KeyName(randomIdxWithinRange)); + } + return keyNameSb.toString(); + } + +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java new file mode 100644 index 000000000000..e14941028110 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.freon; + +import com.codahale.metrics.Timer; +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; + +import java.util.concurrent.Callable; +import java.util.function.Function; +import java.util.HashMap; + +import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.PURE_INDEX; +import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.MD5; +import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR; + +/** + * Ozone range keys generator for performance test. + */ +@CommandLine.Command(name = "ork", + description = "write range keys with the help of the ozone clients.", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true, + showDefaultValues = true) +public class RangeKeysGenerator extends BaseFreonGenerator + implements Callable { + + private static final Logger LOG = + LoggerFactory.getLogger(RangeKeysGenerator.class); + + @CommandLine.Option(names = {"-v", "--volume"}, + description = "Name of the volume which contains the test data. " + + "Will be created if missing.", + defaultValue = "ockrwvolume") + private String volumeName; + + @CommandLine.Option(names = {"-b", "--bucket"}, + description = "Name of the bucket which contains the test data.", + defaultValue = "ockrwbucket") + private String bucketName; + + @CommandLine.Option(names = {"-r", "--range-each-client-write"}, + description = "Write range for each client.", + defaultValue = "0") + private int range; + + @CommandLine.Option(names = {"-s", "--key-start-index"}, + description = "Start index of key.", + defaultValue = "0") + private int startIndex; + + + @CommandLine.Option(names = {"-k", "--key-encode"}, + description = "The algorithm to generate key names. " + + "Options are pureIndex, md5", + defaultValue = "md5") + private String encodeFormat; + + @CommandLine.Option(names = {"-g", "--size"}, + description = "Generated object size (in bytes) " + + "to be written.", + defaultValue = "1") + private int objectSizeInBytes; + + @CommandLine.Option( + names = "--om-service-id", + description = "OM Service ID" + ) + private String omServiceID = null; + private KeyGeneratorUtil kg; + private int clientCount; + private OzoneClient[] ozoneClients; + private byte[] keyContent; + private Timer timer; + + + @Override + public Void call() throws Exception { + init(); + OzoneConfiguration ozoneConfiguration = createOzoneConfiguration(); + clientCount = getThreadNo(); + ozoneClients = new OzoneClient[clientCount]; + for (int i = 0; i < clientCount; i++) { + ozoneClients[i] = createOzoneClient(omServiceID, ozoneConfiguration); + } + + ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName); + if (objectSizeInBytes >= 0) { + keyContent = RandomUtils.nextBytes(objectSizeInBytes); + } + timer = getMetrics().timer("key-read-write"); + + kg = new KeyGeneratorUtil(); + runTests(this::generateRangeKeys); + for (int i = 0; i < clientCount; i++) { + if (ozoneClients[i] != null) { + ozoneClients[i].close(); + } + } + + return null; + } + + public void generateRangeKeys(long count) throws Exception { + int clientIndex = (int)(count % clientCount); + OzoneClient client = ozoneClients[clientIndex]; + int start = startIndex + (int)count * range; + int end = start + range; + + timer.time(() -> { + switch (encodeFormat) { + case PURE_INDEX: + loopRunner(kg.pureIndexKeyNameFunc(), client, start, end); + break; + case MD5: + loopRunner(kg.md5KeyNameFunc(), client, start, end); + break; + default: + loopRunner(kg.md5KeyNameFunc(), client, start, end); + break; + } + return null; + }); + } + + + public void loopRunner(Function keyNameGeneratorfunc, + OzoneClient client, int start, int end) + throws Exception { + String keyName; + for (int i = start; i < end + 1; i++) { + keyName = getPrefix() + FILE_DIR_SEPARATOR + + keyNameGeneratorfunc.apply(i); + try (OzoneOutputStream out = client.getProxy(). + createKey(volumeName, bucketName, keyName, + objectSizeInBytes, null, new HashMap())) { + out.write(keyContent); + } + } + } +} From fdc57a93cb107c619659d1ba208fe795c78dd4b2 Mon Sep 17 00:00:00 2001 From: SaketaChalamchala Date: Mon, 24 Oct 2022 09:22:08 -0700 Subject: [PATCH 18/48] HDDS-7403. README Security Improvement (#3879) Co-authored-by: SaketaChalamchala --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 88b0bc122a73..6a7226fc36fc 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Ozone is a scalable, redundant, and distributed object store for Hadoop and Clou * SCALABLE: Ozone is designed to scale to tens of billions of files and blocks and, in the future, even more. * CONSISTENT: Ozone is a strongly consistent object store. This consistency is achieved by using protocols like RAFT. * CLOUD-NATIVE: Ozone is designed to work well in containerized environments like YARN and Kubernetes. - * SECURE: Ozone integrates with Kerberos infrastructure for access control and supports TDE and on-wire encryption. + * SECURE: Ozone integrates with Kerberos infrastructure for authentication, supports native ACLs and integrates with Ranger for access control and supports TDE and on-wire encryption. * HIGHLY AVAILABLE: Ozone is a fully replicated system that is designed to survive multiple failures. ## Documentation From df0d1e81e3d5298ea30c8b37a939c2a9b0f8ae2d Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Mon, 24 Oct 2022 11:07:45 -0700 Subject: [PATCH 19/48] HDDS-7368. [Multi-Tenant] Add Volume Existence check in preExecute for OMTenantCreateRequest (#3869) --- .../om/request/s3/tenant/OMTenantCreateRequest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tenant/OMTenantCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tenant/OMTenantCreateRequest.java index d678b8f1d1e4..885f45beb310 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tenant/OMTenantCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tenant/OMTenantCreateRequest.java @@ -152,6 +152,17 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { final String volumeName = request.getVolumeName(); // Validate volume name OmUtils.validateVolumeName(volumeName); + + final String dbVolumeKey = ozoneManager.getMetadataManager() + .getVolumeKey(volumeName); + + // Check volume existence + if (ozoneManager.getMetadataManager().getVolumeTable() + .isExist(dbVolumeKey)) { + LOG.debug("volume: '{}' already exists", volumeName); + throw new OMException("Volume already exists", VOLUME_ALREADY_EXISTS); + } + // TODO: Refactor this and OMVolumeCreateRequest to improve maintainability. final VolumeInfo volumeInfo = VolumeInfo.newBuilder() .setVolume(volumeName) From 2f1117539c762ff9d9a040b22d2f3ad33ad94420 Mon Sep 17 00:00:00 2001 From: Sumit Agrawal Date: Tue, 25 Oct 2022 13:24:21 +0530 Subject: [PATCH 20/48] HDDS-7284. JVM crash for rocksdb for read/write after close (#3801) --- .../TestSchemaOneBackwardsCompatibility.java | 2 +- .../hdds/scm/metadata/SCMMetadataStore.java | 4 +- .../apache/hadoop/hdds/utils/db/RDBTable.java | 6 +- .../hadoop/hdds/utils/db/RocksDatabase.java | 205 +++++++++++++++--- .../apache/hadoop/hdds/utils/db/Table.java | 4 +- .../hadoop/hdds/utils/db/TypedTable.java | 2 +- .../block/DeletedBlockLogStateManager.java | 3 +- .../DeletedBlockLogStateManagerImpl.java | 2 +- .../scm/metadata/SCMMetadataStoreImpl.java | 3 +- .../ozone/om/TestObjectStoreWithLegacyFS.java | 4 +- .../om/TestOzoneManagerHAKeyDeletion.java | 10 +- .../hadoop/ozone/om/OMMetadataManager.java | 2 +- .../hadoop/ozone/om/KeyManagerImpl.java | 3 +- .../ozone/om/OmMetadataManagerImpl.java | 2 +- .../ReconContainerMetadataManagerImpl.java | 4 +- .../ozone/recon/tasks/TestTableCountTask.java | 2 +- 16 files changed, 208 insertions(+), 50 deletions(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java index 1323a98b1e01..7aab0af64ea8 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java @@ -160,7 +160,7 @@ private void assertTableIteratorUnsupported(Table table) { table.iterator(); Assert.fail("Table iterator should have thrown " + "UnsupportedOperationException."); - } catch (UnsupportedOperationException ex) { + } catch (IOException | UnsupportedOperationException ex) { // Exception thrown as expected. } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java index a1c9c6bdc1ff..46b19aa09040 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java @@ -127,8 +127,10 @@ public interface SCMMetadataStore extends DBStoreHAManager { * * @param certType - CertType. * @return Iterator + * @throws IOException on failure. */ - TableIterator getAllCerts(CertificateStore.CertType certType); + TableIterator getAllCerts(CertificateStore.CertType certType) + throws IOException; /** * A Table that maintains all the pipeline information. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java index d2bb381ed823..09f58a9335f0 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java @@ -157,12 +157,14 @@ public void deleteWithBatch(BatchOperation batch, byte[] key) } @Override - public TableIterator iterator() { + public TableIterator iterator() + throws IOException { return new RDBStoreIterator(db.newIterator(family, false), this); } @Override - public TableIterator iterator(byte[] prefix) { + public TableIterator iterator(byte[] prefix) + throws IOException { return new RDBStoreIterator(db.newIterator(family, false), this, prefix); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java index 6f20dcee7b12..5fb7d51f6db0 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java @@ -48,6 +48,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -137,12 +138,13 @@ static RocksDatabase open(File dbFile, ManagedDBOptions dbOptions, descriptors, handles); } // init a column family map. + AtomicLong counter = new AtomicLong(0); for (ColumnFamilyHandle h : handles) { - final ColumnFamily f = new ColumnFamily(h); + final ColumnFamily f = new ColumnFamily(h, counter); columnFamilies.put(f.getName(), f); } return new RocksDatabase(dbFile, db, dbOptions, writeOptions, - descriptors, Collections.unmodifiableMap(columnFamilies)); + descriptors, Collections.unmodifiableMap(columnFamilies), counter); } catch (RocksDBException e) { close(columnFamilies, db, descriptors, writeOptions, dbOptions); throw toIOException(RocksDatabase.class, "open " + dbFile, e); @@ -208,11 +210,15 @@ private RocksCheckpoint() { } public void createCheckpoint(Path path) throws IOException { + assertClose(); try { + counter.incrementAndGet(); checkpoint.get().createCheckpoint(path.toString()); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); throw toIOException(this, "createCheckpoint " + path, e); + } finally { + counter.decrementAndGet(); } } @@ -233,11 +239,15 @@ public void close() throws IOException { */ public static final class ColumnFamily { private final byte[] nameBytes; + private AtomicLong counter; private final String name; private final ColumnFamilyHandle handle; + private AtomicBoolean isClosed = new AtomicBoolean(false); - public ColumnFamily(ColumnFamilyHandle handle) throws RocksDBException { + public ColumnFamily(ColumnFamilyHandle handle, AtomicLong counter) + throws RocksDBException { this.nameBytes = handle.getName(); + this.counter = counter; this.name = bytes2String(nameBytes); this.handle = handle; LOG.debug("new ColumnFamily for {}", name); @@ -261,19 +271,37 @@ public int getID() { public void batchDelete(ManagedWriteBatch writeBatch, byte[] key) throws IOException { + assertClosed(); try { + counter.incrementAndGet(); writeBatch.delete(getHandle(), key); } catch (RocksDBException e) { throw toIOException(this, "batchDelete key " + bytes2String(key), e); + } finally { + counter.decrementAndGet(); } } public void batchPut(ManagedWriteBatch writeBatch, byte[] key, byte[] value) throws IOException { + assertClosed(); try { + counter.incrementAndGet(); writeBatch.put(getHandle(), key, value); } catch (RocksDBException e) { throw toIOException(this, "batchPut key " + bytes2String(key), e); + } finally { + counter.decrementAndGet(); + } + } + + public void markClosed() { + isClosed.set(true); + } + + private void assertClosed() throws IOException { + if (isClosed.get()) { + throw new IOException("Rocks Database is closed"); } } @@ -291,28 +319,58 @@ public String toString() { private final Map columnFamilies; private final AtomicBoolean isClosed = new AtomicBoolean(); + + private final AtomicLong counter; private RocksDatabase(File dbFile, ManagedRocksDB db, ManagedDBOptions dbOptions, ManagedWriteOptions writeOptions, List descriptors, - Map columnFamilies) { + Map columnFamilies, AtomicLong counter) { this.name = getClass().getSimpleName() + "[" + dbFile + "]"; this.db = db; this.dbOptions = dbOptions; this.writeOptions = writeOptions; this.descriptors = descriptors; this.columnFamilies = columnFamilies; + this.counter = counter; } public void close() { if (isClosed.compareAndSet(false, true)) { + if (columnFamilies != null) { + columnFamilies.values().stream().forEach(f -> f.markClosed()); + } + // wait till all access to rocks db is process to avoid crash while close + while (true) { + if (counter.get() == 0) { + break; + } + try { + Thread.currentThread().sleep(1); + } catch (InterruptedException e) { + close(columnFamilies, db, descriptors, writeOptions, dbOptions); + Thread.currentThread().interrupt(); + return; + } + } + + // close when counter is 0, no more operation close(columnFamilies, db, descriptors, writeOptions, dbOptions); } } - private void closeOnError(RocksDBException e) { + private void closeOnError(RocksDBException e, boolean isCounted) { if (shouldClose(e)) { - close(); + try { + if (isCounted) { + counter.decrementAndGet(); + } + close(); + } finally { + if (isCounted) { + counter.incrementAndGet(); + } + } } } @@ -325,55 +383,81 @@ private boolean shouldClose(RocksDBException e) { return false; } } + + private void assertClose() throws IOException { + if (isClosed()) { + throw new IOException("Rocks Database is closed"); + } + } public void ingestExternalFile(ColumnFamily family, List files, ManagedIngestExternalFileOptions ingestOptions) throws IOException { + assertClose(); try { + counter.incrementAndGet(); db.get().ingestExternalFile(family.getHandle(), files, ingestOptions); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); String msg = "Failed to ingest external files " + files.stream().collect(Collectors.joining(", ")) + " of " + family.getName(); throw toIOException(this, msg, e); + } finally { + counter.decrementAndGet(); } } public void put(ColumnFamily family, byte[] key, byte[] value) throws IOException { + assertClose(); try { + counter.incrementAndGet(); db.get().put(family.getHandle(), writeOptions, key, value); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); throw toIOException(this, "put " + bytes2String(key), e); + } finally { + counter.decrementAndGet(); } } public void flush() throws IOException { + assertClose(); try (ManagedFlushOptions options = new ManagedFlushOptions()) { + counter.incrementAndGet(); options.setWaitForFlush(true); db.get().flush(options); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); throw toIOException(this, "flush", e); + } finally { + counter.decrementAndGet(); } } public void flushWal(boolean sync) throws IOException { + assertClose(); try { + counter.incrementAndGet(); db.get().flushWal(sync); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); throw toIOException(this, "flushWal with sync=" + sync, e); + } finally { + counter.decrementAndGet(); } } public void compactRange() throws IOException { + assertClose(); try { + counter.incrementAndGet(); db.get().compactRange(); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); throw toIOException(this, "compactRange", e); + } finally { + counter.decrementAndGet(); } } @@ -386,8 +470,15 @@ RocksCheckpoint createCheckpoint() { * otherwise, return true. * @see org.rocksdb.RocksDB#keyMayExist(ColumnFamilyHandle, byte[], Holder) */ - public boolean keyMayExist(ColumnFamily family, byte[] key) { - return db.get().keyMayExist(family.getHandle(), key, null); + public boolean keyMayExist(ColumnFamily family, byte[] key) + throws IOException { + assertClose(); + try { + counter.incrementAndGet(); + return db.get().keyMayExist(family.getHandle(), key, null); + } finally { + counter.decrementAndGet(); + } } /** @@ -396,10 +487,16 @@ public boolean keyMayExist(ColumnFamily family, byte[] key) { * @see org.rocksdb.RocksDB#keyMayExist(ColumnFamilyHandle, byte[], Holder) */ public Supplier keyMayExistHolder(ColumnFamily family, - byte[] key) { - final Holder out = new Holder<>(); - return db.get().keyMayExist(family.getHandle(), key, out) ? - out::getValue : null; + byte[] key) throws IOException { + assertClose(); + try { + counter.incrementAndGet(); + final Holder out = new Holder<>(); + return db.get().keyMayExist(family.getHandle(), key, out) ? + out::getValue : null; + } finally { + counter.decrementAndGet(); + } } public ColumnFamily getColumnFamily(String key) { @@ -411,12 +508,16 @@ public Collection getExtraColumnFamilies() { } public byte[] get(ColumnFamily family, byte[] key) throws IOException { + assertClose(); try { + counter.incrementAndGet(); return db.get().get(family.getHandle(), key); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); final String message = "get " + bytes2String(key) + " from " + family; throw toIOException(this, message, e); + } finally { + counter.decrementAndGet(); } } @@ -429,78 +530,118 @@ public long estimateNumKeys(ColumnFamily family) throws IOException { } private long getLongProperty(String key) throws IOException { + assertClose(); try { + counter.incrementAndGet(); return db.get().getLongProperty(key); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); throw toIOException(this, "getLongProperty " + key, e); + } finally { + counter.decrementAndGet(); } } private long getLongProperty(ColumnFamily family, String key) throws IOException { + assertClose(); try { + counter.incrementAndGet(); return db.get().getLongProperty(family.getHandle(), key); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); final String message = "getLongProperty " + key + " from " + family; throw toIOException(this, message, e); + } finally { + counter.decrementAndGet(); } } public String getProperty(String key) throws IOException { + assertClose(); try { + counter.incrementAndGet(); return db.get().getProperty(key); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); throw toIOException(this, "getProperty " + key, e); + } finally { + counter.decrementAndGet(); } } public String getProperty(ColumnFamily family, String key) throws IOException { + assertClose(); try { + counter.incrementAndGet(); return db.get().getProperty(family.getHandle(), key); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); throw toIOException(this, "getProperty " + key + " from " + family, e); + } finally { + counter.decrementAndGet(); } } public ManagedTransactionLogIterator getUpdatesSince(long sequenceNumber) throws IOException { + assertClose(); try { + counter.incrementAndGet(); return managed(db.get().getUpdatesSince(sequenceNumber)); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); throw toIOException(this, "getUpdatesSince " + sequenceNumber, e); + } finally { + counter.decrementAndGet(); } } public long getLatestSequenceNumber() { - return db.get().getLatestSequenceNumber(); + try { + counter.incrementAndGet(); + return db.get().getLatestSequenceNumber(); + } finally { + counter.decrementAndGet(); + } } - public ManagedRocksIterator newIterator(ColumnFamily family) { - return managed(db.get().newIterator(family.getHandle())); + public ManagedRocksIterator newIterator(ColumnFamily family) + throws IOException { + assertClose(); + try { + counter.incrementAndGet(); + return managed(db.get().newIterator(family.getHandle())); + } finally { + counter.decrementAndGet(); + } } public ManagedRocksIterator newIterator(ColumnFamily family, - boolean fillCache) { + boolean fillCache) throws IOException { + assertClose(); try (ManagedReadOptions readOptions = new ManagedReadOptions()) { + counter.incrementAndGet(); readOptions.setFillCache(fillCache); return managed(db.get().newIterator(family.getHandle(), readOptions)); + } finally { + counter.decrementAndGet(); } } public void batchWrite(ManagedWriteBatch writeBatch, ManagedWriteOptions options) throws IOException { + assertClose(); try { + counter.incrementAndGet(); db.get().write(options, writeBatch); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); throw toIOException(this, "batchWrite", e); + } finally { + counter.decrementAndGet(); } } @@ -509,12 +650,16 @@ public void batchWrite(ManagedWriteBatch writeBatch) throws IOException { } public void delete(ColumnFamily family, byte[] key) throws IOException { + assertClose(); try { + counter.incrementAndGet(); db.get().delete(family.getHandle(), key); } catch (RocksDBException e) { - closeOnError(e); + closeOnError(e, true); final String message = "delete " + bytes2String(key) + " from " + family; throw toIOException(this, message, e); + } finally { + counter.decrementAndGet(); } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java index 3202431474f3..98fbde6417ad 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java @@ -151,8 +151,10 @@ default VALUE getReadCopy(KEY key) throws IOException { * Returns the iterator for this metadata store. * * @return MetaStoreIterator + * @throws IOException on failure. */ - TableIterator> iterator(); + TableIterator> iterator() + throws IOException; /** * Returns a prefixed iterator for this metadata store. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java index c43855a06565..cc2f1345e272 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java @@ -275,7 +275,7 @@ public void deleteWithBatch(BatchOperation batch, KEY key) } @Override - public TableIterator iterator() { + public TableIterator iterator() throws IOException { TableIterator> iterator = rawTable.iterator(); return new TypedTableIterator(iterator, keyType, valueType); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManager.java index 6d35d4c8077a..0c03152f9da8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManager.java @@ -49,7 +49,8 @@ int resetRetryCountOfTransactionInDB(ArrayList txIDs) throws IOException, TimeoutException; TableIterator> getReadOnlyIterator(); + KeyValue> getReadOnlyIterator() + throws IOException; void onFlush(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java index c4e0953fa35b..ceeb27861353 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java @@ -72,7 +72,7 @@ public DeletedBlockLogStateManagerImpl(ConfigurationSource conf, } public TableIterator> getReadOnlyIterator() { + DeletedBlocksTransaction>> getReadOnlyIterator() throws IOException { return new TableIterator>() { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java index e71f6d725888..3d57b0b145d6 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java @@ -267,7 +267,8 @@ public Table getCRLSequenceIdTable() { } @Override - public TableIterator getAllCerts(CertificateStore.CertType certType) { + public TableIterator getAllCerts(CertificateStore.CertType certType) + throws IOException { if (certType == CertificateStore.CertType.VALID_CERTS) { return validCertsTable.iterator(); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java index 152e502b70a8..fb10a346e93f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java @@ -145,10 +145,10 @@ public void testFlatKeyStructureWithOBS() throws Exception { private boolean assertKeyCount( Table keyTable, String dbKey, int expectedCnt, String keyName) { - TableIterator> - itr = keyTable.iterator(); int countKeys = 0; try { + TableIterator> + itr = keyTable.iterator(); itr.seek(dbKey); while (itr.hasNext()) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAKeyDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAKeyDeletion.java index 3e0bda9db176..ca24d5dce754 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAKeyDeletion.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAKeyDeletion.java @@ -70,8 +70,14 @@ public void testKeyDeletion() throws Exception { // Check delete table is empty or not on all OMs. getCluster().getOzoneManagersList().forEach((om) -> { try { - GenericTestUtils.waitFor(() -> - !om.getMetadataManager().getDeletedTable().iterator().hasNext(), + GenericTestUtils.waitFor(() -> { + try { + return !om.getMetadataManager().getDeletedTable().iterator() + .hasNext(); + } catch (Exception ex) { + return false; + } + }, 10000, 120000); } catch (Exception ex) { fail("TestOzoneManagerHAKeyDeletion failed"); diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 67356a88f355..469170475db3 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -433,7 +433,7 @@ Set getMultipartUploadKeys(String volumeName, getBucketIterator(); TableIterator> - getKeyIterator(); + getKeyIterator() throws IOException; /** * Given parent object id and path component name, return the corresponding diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 12be1cdad891..dd66c33a1e3b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -39,7 +39,6 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; @@ -1571,7 +1570,7 @@ public List listStatus(OmKeyArgs args, boolean recursive, getIteratorForKeyInTableCache( boolean recursive, String startKey, String volumeName, String bucketName, TreeMap cacheKeyMap, String keyArgs, - Table keyTable) { + Table keyTable) throws IOException { TableIterator> iterator; try { Iterator, CacheValue>> diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 1db265ed587e..1aae8447dc79 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -969,7 +969,7 @@ public List listBuckets(final String volumeName, @Override public TableIterator> - getKeyIterator() { + getKeyIterator() throws IOException { return keyTable.iterator(); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java index 8e9931798c0e..53aca7ade575 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java @@ -506,12 +506,12 @@ public long getCountForContainers() { } @Override - public TableIterator getContainerTableIterator() { + public TableIterator getContainerTableIterator() throws IOException { return containerKeyTable.iterator(); } @Override - public TableIterator getKeyContainerTableIterator() { + public TableIterator getKeyContainerTableIterator() throws IOException { return keyContainerTable.iterator(); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestTableCountTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestTableCountTask.java index 8151b2385b98..fb400f6417b8 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestTableCountTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestTableCountTask.java @@ -83,7 +83,7 @@ public void setUp() throws IOException { } @Test - public void testReprocess() { + public void testReprocess() throws Exception { OMMetadataManager omMetadataManager = mock(OmMetadataManagerImpl.class); // Mock 5 rows in each table and test the count for (String tableName: tableCountTask.getTaskTables()) { From 74aef20c9e536e65bce05815aed5ae59edd80ba1 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Tue, 25 Oct 2022 17:01:09 +0800 Subject: [PATCH 21/48] HDDS-7182. Add property to control RocksDB max open files (#3843) --- .../statemachine/DatanodeConfiguration.java | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java index 6167d71efef4..75c34477de5b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java @@ -75,8 +75,10 @@ public class DatanodeConfiguration { static final boolean CONTAINER_SCHEMA_V3_ENABLED_DEFAULT = false; static final long ROCKSDB_LOG_MAX_FILE_SIZE_BYTES_DEFAULT = 32 * 1024 * 1024; static final int ROCKSDB_LOG_MAX_FILE_NUM_DEFAULT = 64; + // one hour static final long ROCKSDB_DELETE_OBSOLETE_FILES_PERIOD_MICRO_SECONDS_DEFAULT = - 6 * 60 * 60 * 1000 * 1000; + 1L * 60 * 60 * 1000 * 1000; + static final int ROCKSDB_MAX_OPEN_FILES_DEFAULT = 1024; public static final String ROCKSDB_LOG_MAX_FILE_SIZE_BYTES_KEY = "hdds.datanode.rocksdb.log.max-file-size"; public static final String ROCKSDB_LOG_MAX_FILE_NUM_KEY = @@ -338,16 +340,24 @@ public void setWaitOnAllFollowers(boolean val) { ) private int rocksdbMaxFileNum = ROCKSDB_LOG_MAX_FILE_NUM_DEFAULT; - @Config(key = "rocksdb.delete_obsolete_files_period", - defaultValue = "6h", timeUnit = MICROSECONDS, + @Config(key = "rocksdb.delete-obsolete-files-period", + defaultValue = "1h", timeUnit = MICROSECONDS, type = ConfigType.TIME, tags = { DATANODE }, description = "Periodicity when obsolete files get deleted. " + - "Default is 6h." + "Default is 1h." ) private long rocksdbDeleteObsoleteFilesPeriod = ROCKSDB_DELETE_OBSOLETE_FILES_PERIOD_MICRO_SECONDS_DEFAULT; + @Config(key = "rocksdb.max-open-files", + defaultValue = "1024", + type = ConfigType.INT, + tags = { DATANODE }, + description = "The total number of files that a RocksDB can open. " + ) + private int rocksdbMaxOpenFiles = ROCKSDB_MAX_OPEN_FILES_DEFAULT; + @PostConstruct public void validate() { if (containerDeleteThreads < 1) { @@ -562,4 +572,12 @@ public long getRocksdbDeleteObsoleteFilesPeriod() { public void setRocksdbDeleteObsoleteFilesPeriod(long period) { this.rocksdbDeleteObsoleteFilesPeriod = period; } -} + + public void setRocksdbMaxOpenFiles(int count) { + this.rocksdbMaxOpenFiles = count; + } + + public int getRocksdbMaxOpenFiles() { + return this.rocksdbMaxOpenFiles; + } +} \ No newline at end of file From b9a47f6bbf72605d5391cafc8d382190f822b375 Mon Sep 17 00:00:00 2001 From: XiChen <32928346+xichen01@users.noreply.github.com> Date: Tue, 25 Oct 2022 20:09:50 +0800 Subject: [PATCH 22/48] HDDS-7253. Fix exception when '/' in key name (#3774) --- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 81 ++++++++++++++++++- .../hadoop/ozone/om/TestKeyManagerImpl.java | 70 +++++++++++++++- .../hadoop/ozone/om/KeyManagerImpl.java | 68 +++++++++++++--- 3 files changed, 199 insertions(+), 20 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index f3e8cf10bee3..1f688029b6d1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -153,6 +153,7 @@ public TestOzoneFileSystem(boolean setDefaultFs, boolean enableOMRatis) { private static OzoneManagerProtocol writeClient; private static FileSystem fs; private static OzoneFileSystem o3fs; + private static OzoneBucket ozoneBucket; private static String volumeName; private static String bucketName; private static Trash trash; @@ -179,10 +180,9 @@ private void init() throws Exception { writeClient = cluster.getRpcClient().getObjectStore() .getClientProxy().getOzoneManagerClient(); // create a volume and a bucket to be used by OzoneFileSystem - OzoneBucket bucket = - TestDataUtil.createVolumeAndBucket(cluster, bucketLayout); - volumeName = bucket.getVolumeName(); - bucketName = bucket.getName(); + ozoneBucket = TestDataUtil.createVolumeAndBucket(cluster, bucketLayout); + volumeName = ozoneBucket.getVolumeName(); + bucketName = ozoneBucket.getName(); String rootPath = String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName); @@ -334,6 +334,30 @@ public void testMakeDirsWithAnExistingDirectoryPath() throws Exception { assertTrue("Shouldn't send error if dir exists", status); } + @Test + public void testMakeDirsWithAnFakeDirectory() throws Exception { + /* + * Op 1. commit a key -> "dir1/dir2/key1" + * Op 2. create dir -> "dir1/testDir", the dir1 is a fake dir, + * "dir1/testDir" can be created normal + */ + + String fakeGrandpaKey = "dir1"; + String fakeParentKey = fakeGrandpaKey + "/dir2"; + String fullKeyName = fakeParentKey + "/key1"; + TestDataUtil.createKey(ozoneBucket, fullKeyName, ""); + + // /dir1/dir2 should not exist + assertFalse(fs.exists(new Path(fakeParentKey))); + + // /dir1/dir2/key2 should be created because has a fake parent directory + Path subdir = new Path(fakeParentKey, "key2"); + assertTrue(fs.mkdirs(subdir)); + // the intermediate directories /dir1 and /dir1/dir2 will be created too + assertTrue(fs.exists(new Path(fakeGrandpaKey))); + assertTrue(fs.exists(new Path(fakeParentKey))); + } + @Test public void testCreateWithInvalidPaths() throws Exception { // Test for path with .. @@ -727,6 +751,37 @@ public void testListStatusOnLargeDirectory() throws Exception { } } + @Test + public void testListStatusOnKeyNameContainDelimiter() throws Exception { + /* + * op1: create a key -> "dir1/dir2/key1" + * op2: `ls /` child dir "/dir1/" will be return + * op2: `ls /dir1` child dir "/dir1/dir2/" will be return + * op3: `ls /dir1/dir2` file "/dir1/dir2/key" will be return + * + * the "/dir1", "/dir1/dir2/" are fake directory + * */ + String keyName = "dir1/dir2/key1"; + TestDataUtil.createKey(ozoneBucket, keyName, ""); + FileStatus[] fileStatuses; + + fileStatuses = fs.listStatus(new Path("/")); + assertEquals(1, fileStatuses.length); + assertEquals("/dir1", fileStatuses[0].getPath().toUri().getPath()); + assertTrue(fileStatuses[0].isDirectory()); + + fileStatuses = fs.listStatus(new Path("/dir1")); + assertEquals(1, fileStatuses.length); + assertEquals("/dir1/dir2", fileStatuses[0].getPath().toUri().getPath()); + assertTrue(fileStatuses[0].isDirectory()); + + fileStatuses = fs.listStatus(new Path("/dir1/dir2")); + assertEquals(1, fileStatuses.length); + assertEquals("/dir1/dir2/key1", + fileStatuses[0].getPath().toUri().getPath()); + assertTrue(fileStatuses[0].isFile()); + } + /** * Cleanup files and directories. * @@ -1273,6 +1328,24 @@ public void testRenameFileToDir() throws Exception { "file1"))); } + @Test + public void testRenameContainDelimiterFile() throws Exception { + String fakeGrandpaKey = "dir1"; + String fakeParentKey = fakeGrandpaKey + "/dir2"; + String sourceKeyName = fakeParentKey + "/key1"; + String targetKeyName = fakeParentKey + "/key2"; + TestDataUtil.createKey(ozoneBucket, sourceKeyName, ""); + + Path sourcePath = new Path(fs.getUri().toString() + "/" + sourceKeyName); + Path targetPath = new Path(fs.getUri().toString() + "/" + targetKeyName); + assertTrue(fs.rename(sourcePath, targetPath)); + assertFalse(fs.exists(sourcePath)); + assertTrue(fs.exists(targetPath)); + // intermediate directories will not be created + assertFalse(fs.exists(new Path(fakeGrandpaKey))); + assertFalse(fs.exists(new Path(fakeParentKey))); + } + /** * Fails if the (a) parent of dst does not exist or (b) parent is a file. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index 492173b71cf9..e30d27fc7460 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -800,19 +800,20 @@ public void testLookupKeyWithLocation() throws IOException { List locationList = keySession.getKeyInfo().getLatestVersionLocations().getLocationList(); Assert.assertEquals(1, locationList.size()); + long containerID = locationList.get(0).getContainerID(); locationInfoList.add( new OmKeyLocationInfo.Builder().setPipeline(pipeline) - .setBlockID(new BlockID(locationList.get(0).getContainerID(), + .setBlockID(new BlockID(containerID, locationList.get(0).getLocalID())).build()); keyArgs.setLocationInfoList(locationInfoList); writeClient.commitKey(keyArgs, keySession.getId()); - ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(1L) - .setPipelineID(pipeline.getId()).build(); + ContainerInfo containerInfo = new ContainerInfo.Builder() + .setContainerID(containerID).setPipelineID(pipeline.getId()).build(); List containerWithPipelines = Arrays.asList( new ContainerWithPipeline(containerInfo, pipeline)); when(mockScmContainerClient.getContainerWithPipelineBatch( - Arrays.asList(1L))).thenReturn(containerWithPipelines); + Arrays.asList(containerID))).thenReturn(containerWithPipelines); OmKeyInfo key = keyManager.lookupKey(keyArgs, null); Assert.assertEquals(key.getKeyName(), keyName); @@ -1273,6 +1274,67 @@ public void testListStatus() throws IOException { } } + @Test + public void testGetFileStatus() throws IOException { + // create a key + String keyName = RandomStringUtils.randomAlphabetic(5); + OmKeyArgs keyArgs = createBuilder() + .setKeyName(keyName) + .setLatestVersionLocation(true) + .build(); + writeClient.createFile(keyArgs, false, false); + OpenKeySession keySession = writeClient.createFile(keyArgs, true, true); + keyArgs.setLocationInfoList( + keySession.getKeyInfo().getLatestVersionLocations().getLocationList()); + writeClient.commitKey(keyArgs, keySession.getId()); + OzoneFileStatus ozoneFileStatus = keyManager.getFileStatus(keyArgs); + Assert.assertEquals(keyName, ozoneFileStatus.getKeyInfo().getFileName()); + } + + @Test + public void testGetFileStatusWithFakeDir() throws IOException { + String parentDir = "dir1"; + String key = "key1"; + String fullKeyName = parentDir + OZONE_URI_DELIMITER + key; + OzoneFileStatus ozoneFileStatus; + + // create a key "dir1/key1" + OmKeyArgs keyArgs = createBuilder().setKeyName(fullKeyName).build(); + OpenKeySession keySession = writeClient.openKey(keyArgs); + keyArgs.setLocationInfoList( + keySession.getKeyInfo().getLatestVersionLocations().getLocationList()); + writeClient.commitKey(keyArgs, keySession.getId()); + + // verify + String keyArg; + keyArg = metadataManager.getOzoneKey(VOLUME_NAME, BUCKET_NAME, parentDir); + Assert.assertNull( + metadataManager.getKeyTable(getDefaultBucketLayout()).get(keyArg)); + keyArg = metadataManager.getOzoneKey(VOLUME_NAME, BUCKET_NAME, fullKeyName); + Assert.assertNotNull(metadataManager.getKeyTable(getDefaultBucketLayout()) + .get(keyArg)); + + // get a non-existing "dir1", since the key is prefixed "dir1/key1", + // a fake "/dir1" will be returned + keyArgs = createBuilder().setKeyName(parentDir).build(); + ozoneFileStatus = keyManager.getFileStatus(keyArgs); + Assert.assertEquals(parentDir, ozoneFileStatus.getKeyInfo().getFileName()); + Assert.assertTrue(ozoneFileStatus.isDirectory()); + + // get a non-existing "dir", since the key is not prefixed "dir1/key1", + // a `OMException` will be thrown + keyArgs = createBuilder().setKeyName("dir").build(); + OmKeyArgs finalKeyArgs = keyArgs; + Assert.assertThrows(OMException.class, () -> keyManager.getFileStatus( + finalKeyArgs)); + + // get a file "dir1/key1" + keyArgs = createBuilder().setKeyName(fullKeyName).build(); + ozoneFileStatus = keyManager.getFileStatus(keyArgs); + Assert.assertEquals(key, ozoneFileStatus.getKeyInfo().getFileName()); + Assert.assertTrue(ozoneFileStatus.isFile()); + } + @Test public void testRefreshPipeline() throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index dd66c33a1e3b..483a2e88b045 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1209,6 +1209,8 @@ private OzoneFileStatus getOzoneFileStatus(OmKeyArgs args, final String keyName = args.getKeyName(); OmKeyInfo fileKeyInfo = null; + OmKeyInfo dirKeyInfo = null; + OmKeyInfo fakeDirKeyInfo = null; metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, bucketName); try { @@ -1221,28 +1223,27 @@ private OzoneFileStatus getOzoneFileStatus(OmKeyArgs args, // Check if the key is a file. String fileKeyBytes = metadataManager.getOzoneKey( volumeName, bucketName, keyName); - fileKeyInfo = metadataManager - .getKeyTable(getBucketLayout(metadataManager, volumeName, bucketName)) - .get(fileKeyBytes); + BucketLayout layout = + getBucketLayout(metadataManager, volumeName, bucketName); + fileKeyInfo = metadataManager.getKeyTable(layout).get(fileKeyBytes); + String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); // Check if the key is a directory. if (fileKeyInfo == null) { - String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); String dirKeyBytes = metadataManager.getOzoneKey( volumeName, bucketName, dirKey); - OmKeyInfo dirKeyInfo = metadataManager.getKeyTable( - getBucketLayout(metadataManager, volumeName, bucketName)) - .get(dirKeyBytes); - if (dirKeyInfo != null) { - return new OzoneFileStatus(dirKeyInfo, scmBlockSize, true); + dirKeyInfo = metadataManager.getKeyTable(layout).get(dirKeyBytes); + if (dirKeyInfo == null) { + fakeDirKeyInfo = + createFakeDirIfShould(volumeName, bucketName, keyName, layout); } } } finally { metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, bucketName); - - // if the key is a file then do refresh pipeline info in OM by asking SCM if (fileKeyInfo != null) { + // if the key is a file + // then do refresh pipeline info in OM by asking SCM if (args.getLatestVersionLocation()) { slimLocationVersion(fileKeyInfo); } @@ -1257,10 +1258,21 @@ private OzoneFileStatus getOzoneFileStatus(OmKeyArgs args, sortDatanodes(clientAddress, fileKeyInfo); } } - return new OzoneFileStatus(fileKeyInfo, scmBlockSize, false); } } + if (fileKeyInfo != null) { + return new OzoneFileStatus(fileKeyInfo, scmBlockSize, false); + } + + if (dirKeyInfo != null) { + return new OzoneFileStatus(dirKeyInfo, scmBlockSize, true); + } + + if (fakeDirKeyInfo != null) { + return new OzoneFileStatus(fakeDirKeyInfo, scmBlockSize, true); + } + // Key is not found, throws exception if (LOG.isDebugEnabled()) { LOG.debug("Unable to get file status for the key: volume: {}, bucket:" + @@ -1272,6 +1284,37 @@ private OzoneFileStatus getOzoneFileStatus(OmKeyArgs args, FILE_NOT_FOUND); } + /** + * Create a fake directory if the key is a path prefix, + * otherwise returns null. + * Some keys may contain '/' Ozone will treat '/' as directory separator + * such as : key name is 'a/b/c', 'a' and 'b' may not really exist, + * but Ozone treats 'a' and 'b' as a directory. + * we need create a fake directory 'a' or 'a/b' + * + * @return OmKeyInfo if the key is a path prefix, otherwise returns null. + */ + private OmKeyInfo createFakeDirIfShould(String volume, String bucket, + String keyName, BucketLayout layout) throws IOException { + OmKeyInfo fakeDirKeyInfo = null; + String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); + String fileKeyBytes = metadataManager.getOzoneKey(volume, bucket, keyName); + Table.KeyValue keyValue = + metadataManager.getKeyTable(layout).iterator().seek(fileKeyBytes); + + if (keyValue != null) { + Path fullPath = Paths.get(keyValue.getValue().getKeyName()); + Path subPath = Paths.get(dirKey); + OmKeyInfo omKeyInfo = keyValue.getValue(); + if (fullPath.startsWith(subPath)) { + // create fake directory + fakeDirKeyInfo = createDirectoryKey(omKeyInfo, dirKey); + } + } + + return fakeDirKeyInfo; + } + private OzoneFileStatus getOzoneFileStatusFSO(OmKeyArgs args, String clientAddress, boolean skipFileNotFoundError) throws IOException { @@ -1349,6 +1392,7 @@ private OmKeyInfo createDirectoryKey(OmKeyInfo keyInfo, String keyName) .setVolumeName(keyInfo.getVolumeName()) .setBucketName(keyInfo.getBucketName()) .setKeyName(dir) + .setFileName(OzoneFSUtils.getFileName(keyName)) .setOmKeyLocationInfos(Collections.singletonList( new OmKeyLocationInfoGroup(0, new ArrayList<>()))) .setCreationTime(Time.now()) From 340f3a787627a86783dfe22b026c074a72d0d78f Mon Sep 17 00:00:00 2001 From: Maxim Myskov Date: Tue, 25 Oct 2022 17:07:05 +0300 Subject: [PATCH 23/48] HDDS-7381. Cleanup of VolumeManagerImpl (#3873) --- .../apache/hadoop/ozone/om/OzoneManager.java | 2 +- .../apache/hadoop/ozone/om/VolumeManager.java | 2 +- .../hadoop/ozone/om/VolumeManagerImpl.java | 59 +------------------ 3 files changed, 4 insertions(+), 59 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index b5be6c02d769..24a81d677966 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -715,7 +715,7 @@ private void instantiateServices(boolean withNewSnapshot) throws IOException { multiTenantManager = new OMMultiTenantManagerImpl(this, configuration); OzoneAclUtils.setOMMultiTenantManager(multiTenantManager); } - volumeManager = new VolumeManagerImpl(metadataManager, configuration); + volumeManager = new VolumeManagerImpl(metadataManager); bucketManager = new BucketManagerImpl(metadataManager); if (secConfig.isSecurityEnabled() || testSecureOmFlag) { s3SecretManager = new S3SecretManagerImpl(configuration, metadataManager); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManager.java index 7375cabc2854..49c33f566bf3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManager.java @@ -22,7 +22,7 @@ import java.util.List; /** - * OM volume manager interface for read operations on a volume. + * VolumeManager is responsible for read operations on a volume. */ public interface VolumeManager extends IOzoneAcl { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java index 7041d7b9694b..7d8c8fe51721 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java @@ -20,7 +20,6 @@ import java.util.List; import java.util.Objects; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; @@ -37,7 +36,7 @@ import org.slf4j.LoggerFactory; /** - * OM volume management code. + * Volume Manager implementation. */ public class VolumeManagerImpl implements VolumeManager { private static final Logger LOG = @@ -45,22 +44,10 @@ public class VolumeManagerImpl implements VolumeManager { private final OMMetadataManager metadataManager; - /** - * Constructor. - * @param conf - Ozone configuration. - * @throws IOException - */ - public VolumeManagerImpl(OMMetadataManager metadataManager, - OzoneConfiguration conf) { + public VolumeManagerImpl(OMMetadataManager metadataManager) { this.metadataManager = metadataManager; } - /** - * Gets the volume information. - * @param volume - Volume name. - * @return VolumeArgs or exception is thrown. - * @throws IOException - */ @Override public OmVolumeArgs getVolumeInfo(String volume) throws IOException { Preconditions.checkNotNull(volume); @@ -86,9 +73,6 @@ public OmVolumeArgs getVolumeInfo(String volume) throws IOException { } } - /** - * {@inheritDoc} - */ @Override public List listVolumes(String userName, String prefix, String startKey, int maxKeys) throws IOException { @@ -100,14 +84,6 @@ public List listVolumes(String userName, } } - /** - * Add acl for Ozone object. Return true if acl is added successfully else - * false. - * - * @param obj Ozone object for which acl should be added. - * @param acl ozone acl to be added. - * @throws IOException if there is error. - */ @Override public boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException { Objects.requireNonNull(obj); @@ -144,14 +120,6 @@ public boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException { return false; } - /** - * Remove acl for Ozone object. Return true if acl is removed successfully - * else false. - * - * @param obj Ozone object. - * @param acl Ozone acl to be removed. - * @throws IOException if there is error. - */ @Override public boolean removeAcl(OzoneObj obj, OzoneAcl acl) throws IOException { Objects.requireNonNull(obj); @@ -177,7 +145,6 @@ public boolean removeAcl(OzoneObj obj, OzoneAcl acl) throws IOException { } Preconditions.checkState(volume.equals(volumeArgs.getVolume())); - //return volumeArgs.getAclMap().hasAccess(userAcl); } catch (IOException ex) { if (!(ex instanceof OMException)) { LOG.error("Remove acl operation failed for volume:{} acl:{}", @@ -191,14 +158,6 @@ public boolean removeAcl(OzoneObj obj, OzoneAcl acl) throws IOException { return false; } - /** - * Acls to be set for given Ozone object. This operations reset ACL for given - * object to list of ACLs provided in argument. - * - * @param obj Ozone object. - * @param acls List of acls. - * @throws IOException if there is error. - */ @Override public boolean setAcl(OzoneObj obj, List acls) throws IOException { Objects.requireNonNull(obj); @@ -223,7 +182,6 @@ public boolean setAcl(OzoneObj obj, List acls) throws IOException { metadataManager.getVolumeTable().put(dbVolumeKey, volumeArgs); Preconditions.checkState(volume.equals(volumeArgs.getVolume())); - //return volumeArgs.getAclMap().hasAccess(userAcl); } catch (IOException ex) { if (!(ex instanceof OMException)) { LOG.error("Set acl operation failed for volume:{} acls:{}", @@ -237,12 +195,6 @@ public boolean setAcl(OzoneObj obj, List acls) throws IOException { return true; } - /** - * Returns list of ACLs for given Ozone object. - * - * @param obj Ozone object. - * @throws IOException if there is error. - */ @Override public List getAcl(OzoneObj obj) throws IOException { Objects.requireNonNull(obj); @@ -275,13 +227,6 @@ public List getAcl(OzoneObj obj) throws IOException { } } - /** - * Check access for given ozoneObject. - * - * @param ozObject object for which access needs to be checked. - * @param context Context object encapsulating all user related information. - * @return true if user has access else false. - */ @Override public boolean checkAccess(OzoneObj ozObject, RequestContext context) throws OMException { From dfc13a0506726c51a343051a7a371ff1d5cf0ef2 Mon Sep 17 00:00:00 2001 From: Nibiru Date: Wed, 26 Oct 2022 01:00:02 +0800 Subject: [PATCH 24/48] HDDS-7258. Cleanup the allocated but uncommitted blocks (#3778) --- .../hadoop/ozone/om/helpers/OmKeyInfo.java | 76 ++++++---- .../om/request/key/OMKeyCommitRequest.java | 19 ++- .../key/OMKeyCommitRequestWithFSO.java | 23 ++- .../ozone/om/request/key/OMKeyRequest.java | 24 ++++ .../om/response/key/OMKeyCommitResponse.java | 6 + .../request/key/TestOMKeyCommitRequest.java | 96 ++++++++++++- .../om/service/TestKeyDeletingService.java | 136 ++++++++++++++++-- 7 files changed, 333 insertions(+), 47 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index 9b2014dd7b84..f8f589af2efc 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -26,9 +26,9 @@ import java.util.Objects; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileEncryptionInfo; -import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.ContainerBlockID; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; @@ -180,25 +180,30 @@ public void updateModifcationTime() { /** * updates the length of the each block in the list given. * This will be called when the key is being committed to OzoneManager. + * Return the uncommitted locationInfo to be deleted. * * @param locationInfoList list of locationInfo + * @return allocated but uncommitted locationInfos */ - public void updateLocationInfoList(List locationInfoList, - boolean isMpu) { - updateLocationInfoList(locationInfoList, isMpu, false); + public List updateLocationInfoList( + List locationInfoList, boolean isMpu) { + return updateLocationInfoList(locationInfoList, isMpu, false); } /** * updates the length of the each block in the list given. * This will be called when the key is being committed to OzoneManager. + * Return the uncommitted locationInfo to be deleted. * * @param locationInfoList list of locationInfo * @param isMpu a true represents multi part key, false otherwise * @param skipBlockIDCheck a true represents that the blockId verification * check should be skipped, false represents that * the blockId verification will be required + * @return allocated but uncommitted locationInfos */ - public void updateLocationInfoList(List locationInfoList, + public List updateLocationInfoList( + List locationInfoList, boolean isMpu, boolean skipBlockIDCheck) { long latestVersion = getLatestVersionLocations().getVersion(); OmKeyLocationInfoGroup keyLocationInfoGroup = getLatestVersionLocations(); @@ -207,51 +212,68 @@ public void updateLocationInfoList(List locationInfoList, // Compare user given block location against allocatedBlockLocations // present in OmKeyInfo. + List uncommittedBlocks; List updatedBlockLocations; if (skipBlockIDCheck) { updatedBlockLocations = locationInfoList; + uncommittedBlocks = new ArrayList<>(); } else { - updatedBlockLocations = + Pair, List> verifiedResult = verifyAndGetKeyLocations(locationInfoList, keyLocationInfoGroup); + updatedBlockLocations = verifiedResult.getLeft(); + uncommittedBlocks = verifiedResult.getRight(); } - // Updates the latest locationList in the latest version only with - // given locationInfoList here. - // TODO : The original allocated list and the updated list here may vary - // as the containers on the Datanode on which the blocks were pre allocated - // might get closed. The diff of blocks between these two lists here - // need to be garbage collected in case the ozone client dies. + keyLocationInfoGroup.removeBlocks(latestVersion); // set each of the locationInfo object to the latest version updatedBlockLocations.forEach(omKeyLocationInfo -> omKeyLocationInfo .setCreateVersion(latestVersion)); keyLocationInfoGroup.addAll(latestVersion, updatedBlockLocations); - } - private List verifyAndGetKeyLocations( - List locationInfoList, - OmKeyLocationInfoGroup keyLocationInfoGroup) { - - List allocatedBlockLocations = - keyLocationInfoGroup.getBlocksLatestVersionOnly(); - List updatedBlockLocations = new ArrayList<>(); + return uncommittedBlocks; + } - List existingBlockIDs = new ArrayList<>(); - for (OmKeyLocationInfo existingLocationInfo : allocatedBlockLocations) { - BlockID existingBlockID = existingLocationInfo.getBlockID(); - existingBlockIDs.add(existingBlockID.getContainerBlockID()); + /** + * 1. Verify committed KeyLocationInfos + * 2. Find out the allocated but uncommitted KeyLocationInfos. + * + * @param locationInfoList committed KeyLocationInfos + * @param keyLocationInfoGroup allocated KeyLocationInfoGroup + * @return Pair of updatedOmKeyLocationInfo and uncommittedOmKeyLocationInfo + */ + private Pair, List> + verifyAndGetKeyLocations( + List locationInfoList, + OmKeyLocationInfoGroup keyLocationInfoGroup) { + // Only check ContainerBlockID here to avoid the mismatch of the pipeline + // field and BcsId in the OmKeyLocationInfo, as the OmKeyInfoCodec ignores + // the pipeline field by default and bcsId would be updated in Ratis mode. + Map allocatedBlockLocations = + new HashMap<>(); + for (OmKeyLocationInfo existingLocationInfo : keyLocationInfoGroup. + getLocationList()) { + ContainerBlockID existingBlockID = existingLocationInfo.getBlockID(). + getContainerBlockID(); + // The case of overwriting value should never happen + allocatedBlockLocations.put(existingBlockID, existingLocationInfo); } + List updatedBlockLocations = new ArrayList<>(); for (OmKeyLocationInfo modifiedLocationInfo : locationInfoList) { - BlockID modifiedBlockID = modifiedLocationInfo.getBlockID(); - if (existingBlockIDs.contains(modifiedBlockID.getContainerBlockID())) { + ContainerBlockID modifiedContainerBlockId = + modifiedLocationInfo.getBlockID().getContainerBlockID(); + if (allocatedBlockLocations.containsKey(modifiedContainerBlockId)) { updatedBlockLocations.add(modifiedLocationInfo); + allocatedBlockLocations.remove(modifiedContainerBlockId); } else { LOG.warn("Unknown BlockLocation:{}, where the blockID of given " + "location doesn't match with the stored/allocated block of" + " keyName:{}", modifiedLocationInfo, keyName); } } - return updatedBlockLocations; + List uncommittedLocationInfos = new ArrayList<>( + allocatedBlockLocations.values()); + return Pair.of(updatedBlockLocations, uncommittedLocationInfos); } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index de504ea4e52f..02cbf83f78b0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -218,8 +218,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); omKeyInfo.setDataSize(commitKeyArgs.getDataSize()); omKeyInfo.setModificationTime(commitKeyArgs.getModificationTime()); - // Update the block length for each block - omKeyInfo.updateLocationInfoList(locationInfoList, false); + // Update the block length for each block, return the allocated but + // uncommitted blocks + List uncommitted = omKeyInfo.updateLocationInfoList( + locationInfoList, false); + // Set the UpdateID to current transactionLogIndex omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); @@ -239,6 +242,18 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, omBucketInfo.incrUsedNamespace(1L); } + // let the uncommitted blocks pretend as key's old version blocks + // which will be deleted as RepeatedOmKeyInfo + OmKeyInfo pseudoKeyInfo = wrapUncommittedBlocksAsPseudoKey(uncommitted, + omKeyInfo); + if (pseudoKeyInfo != null) { + if (oldKeyVersionsToDelete != null) { + oldKeyVersionsToDelete.addOmKeyInfo(pseudoKeyInfo); + } else { + oldKeyVersionsToDelete = new RepeatedOmKeyInfo(pseudoKeyInfo); + } + } + // Add to cache of open key table and key table. omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry( new CacheKey<>(dbOpenKey), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java index b9239c9d8648..5c7ac450b8f3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java @@ -41,6 +41,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Path; @@ -58,6 +60,9 @@ */ public class OMKeyCommitRequestWithFSO extends OMKeyCommitRequest { + private static final Logger LOG = + LoggerFactory.getLogger(OMKeyCommitRequestWithFSO.class); + public OMKeyCommitRequestWithFSO(OMRequest omRequest, BucketLayout bucketLayout) { super(omRequest, bucketLayout); @@ -144,10 +149,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, omKeyInfo.setModificationTime(commitKeyArgs.getModificationTime()); - // Update the block length for each block - List allocatedLocationInfoList = - omKeyInfo.getLatestVersionLocations().getLocationList(); - omKeyInfo.updateLocationInfoList(locationInfoList, false); + List uncommitted = omKeyInfo.updateLocationInfoList( + locationInfoList, false); // Set the UpdateID to current transactionLogIndex omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); @@ -177,6 +180,18 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, omBucketInfo.incrUsedNamespace(1L); } + // let the uncommitted blocks pretend as key's old version blocks + // which will be deleted as RepeatedOmKeyInfo + OmKeyInfo pseudoKeyInfo = wrapUncommittedBlocksAsPseudoKey(uncommitted, + omKeyInfo); + if (pseudoKeyInfo != null) { + if (oldKeyVersionsToDelete != null) { + oldKeyVersionsToDelete.addOmKeyInfo(pseudoKeyInfo); + } else { + oldKeyVersionsToDelete = new RepeatedOmKeyInfo(pseudoKeyInfo); + } + } + // Add to cache of open key table and key table. OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, dbFileKey, null, fileName, trxnLogIndex); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 904b5c000592..8c79e16dcd94 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -816,4 +816,28 @@ protected OzoneLockStrategy getOzoneLockStrategy(OzoneManager ozoneManager) { return ozoneManager.getOzoneLockProvider() .createLockStrategy(getBucketLayout()); } + + /** + * Wrap the uncommitted blocks as pseudoKeyInfo. + * + * @param uncommitted Uncommitted OmKeyLocationInfo + * @param omKeyInfo Args for key block + * @return pseudoKeyInfo + */ + protected OmKeyInfo wrapUncommittedBlocksAsPseudoKey( + List uncommitted, OmKeyInfo omKeyInfo) { + if (uncommitted.isEmpty()) { + return null; + } + LOG.info("Detect allocated but uncommitted blocks {} in key {}.", + uncommitted, omKeyInfo.getKeyName()); + OmKeyInfo pseudoKeyInfo = omKeyInfo.copyObject(); + // TODO dataSize of pseudoKey is not real here + List uncommittedGroups = new ArrayList<>(); + // version not matters in the current logic of keyDeletingService, + // all versions of blocks will be deleted. + uncommittedGroups.add(new OmKeyLocationInfoGroup(0, uncommitted)); + pseudoKeyInfo.setKeyLocationVersions(uncommittedGroups); + return pseudoKeyInfo; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java index 513e10cba46a..911a61cce63c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponse.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.response.key; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -104,6 +105,11 @@ protected String getOzoneKeyName() { return ozoneKeyName; } + @VisibleForTesting + public RepeatedOmKeyInfo getKeysToDelete() { + return keysToDelete; + } + protected void updateDeletedTable(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { if (this.keysToDelete != null) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index bdf16fef42a4..10552e380d18 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.ozone.om.response.key.OMKeyCommitResponse; import org.apache.hadoop.util.Time; import org.jetbrains.annotations.NotNull; import org.junit.Assert; @@ -190,6 +191,93 @@ public void testValidateAndUpdateCache() throws Exception { omKeyInfo.getLatestVersionLocations().getLocationList()); } + @Test + public void testValidateAndUpdateCacheWithUncommittedBlocks() + throws Exception { + + // allocated block list + List allocatedKeyLocationList = getKeyLocation(5); + + List allocatedBlockList = allocatedKeyLocationList + .stream().map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + // committed block list, with three blocks different with the allocated + List committedKeyLocationList = getKeyLocation(3); + + OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest( + committedKeyLocationList)); + + OMKeyCommitRequest omKeyCommitRequest = + getOmKeyCommitRequest(modifiedOmRequest); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + String ozoneKey = addKeyToOpenKeyTable(allocatedBlockList); + + // Key should not be there in key table, as validateAndUpdateCache is + // still not called. + OmKeyInfo omKeyInfo = + omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()) + .get(ozoneKey); + + Assert.assertNull(omKeyInfo); + + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, + 100L, ozoneManagerDoubleBufferHelper); + + Assert.assertEquals(OzoneManagerProtocolProtos.Status.OK, + omClientResponse.getOMResponse().getStatus()); + + List toDeleteKeyList = ((OMKeyCommitResponse) omClientResponse). + getKeysToDelete().cloneOmKeyInfoList(); + + // This is the first time to commit key, only the allocated but uncommitted + // blocks should be deleted. + Assert.assertEquals(1, toDeleteKeyList.size()); + Assert.assertEquals(2, toDeleteKeyList.get(0). + getKeyLocationVersions().get(0).getLocationList().size()); + + // Entry should be deleted from openKey Table. + omKeyInfo = + omMetadataManager.getOpenKeyTable(omKeyCommitRequest.getBucketLayout()) + .get(ozoneKey); + Assert.assertNull(omKeyInfo); + + // Now entry should be created in key Table. + omKeyInfo = + omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()) + .get(ozoneKey); + + Assert.assertNotNull(omKeyInfo); + + // DB keyInfo format + verifyKeyName(omKeyInfo); + + // Check modification time + CommitKeyRequest commitKeyRequest = modifiedOmRequest.getCommitKeyRequest(); + Assert.assertEquals(commitKeyRequest.getKeyArgs().getModificationTime(), + omKeyInfo.getModificationTime()); + + // Check block location. + List locationInfoListFromCommitKeyRequest = + commitKeyRequest.getKeyArgs() + .getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + List intersection = new ArrayList<>(allocatedBlockList); + intersection.retainAll(locationInfoListFromCommitKeyRequest); + + // Key table should have three blocks. + Assert.assertEquals(intersection, + omKeyInfo.getLatestVersionLocations().getLocationList()); + Assert.assertEquals(3, intersection.size()); + + } + @Test public void testValidateAndUpdateCacheWithSubDirs() throws Exception { parentDir = "dir1/dir2/dir3/"; @@ -466,15 +554,19 @@ private void verifyKeyArgs(KeyArgs originalKeyArgs, KeyArgs modifiedKeyArgs) { modifiedKeyArgs.getFactor()); } + private OMRequest createCommitKeyRequest() { + return createCommitKeyRequest(getKeyLocation(5)); + } + /** * Create OMRequest which encapsulates CommitKeyRequest. */ - private OMRequest createCommitKeyRequest() { + private OMRequest createCommitKeyRequest(List keyLocations) { KeyArgs keyArgs = KeyArgs.newBuilder().setDataSize(dataSize).setVolumeName(volumeName) .setKeyName(keyName).setBucketName(bucketName) .setType(replicationType).setFactor(replicationFactor) - .addAllKeyLocations(getKeyLocation(5)).build(); + .addAllKeyLocations(keyLocations).build(); CommitKeyRequest commitKeyRequest = CommitKeyRequest.newBuilder().setKeyArgs(keyArgs) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java index a24a72ac80ba..3bc35c8b3b82 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java @@ -22,14 +22,21 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OmTestManagers; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ScmBlockLocationTestingClient; +import org.apache.hadoop.ozone.common.BlockGroup; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.ratis.util.ExitUtils; import org.junit.BeforeClass; @@ -133,8 +140,8 @@ public void checkIfDeleteServiceIsDeletingKeys() () -> keyDeletingService.getDeletedKeyCount().get() >= keyCount, 1000, 10000); Assert.assertTrue(keyDeletingService.getRunCount().get() > 1); - Assert.assertEquals( - keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size(), 0); + Assert.assertEquals(0, + keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size()); } @Test(timeout = 40000) @@ -176,9 +183,9 @@ public void checkIfDeleteServiceWithFailingSCM() () -> keyDeletingService.getRunCount().get() >= 5, 100, 1000); // Since SCM calls are failing, deletedKeyCount should be zero. - Assert.assertEquals(keyDeletingService.getDeletedKeyCount().get(), 0); - Assert.assertEquals( - keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size(), keyCount); + Assert.assertEquals(0, keyDeletingService.getDeletedKeyCount().get()); + Assert.assertEquals(keyCount, + keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size()); } @Test(timeout = 30000) @@ -200,15 +207,86 @@ public void checkDeletionForEmptyKey() KeyDeletingService keyDeletingService = (KeyDeletingService) keyManager.getDeletingService(); - // Since empty keys are directly deleted from db there should be no - // pending deletion keys. Also deletedKeyCount should be zero. - Assert.assertEquals( - keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size(), 0); + // the pre-allocated blocks are not committed, hence they will be deleted. + Assert.assertEquals(100, + keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size()); // Make sure that we have run the background thread 2 times or more GenericTestUtils.waitFor( () -> keyDeletingService.getRunCount().get() >= 2, 100, 1000); - Assert.assertEquals(keyDeletingService.getDeletedKeyCount().get(), 0); + // the blockClient is set to fail the deletion of key blocks, hence no keys + // will be deleted + Assert.assertEquals(0, keyDeletingService.getDeletedKeyCount().get()); + } + + @Test(timeout = 30000) + public void checkDeletionForPartiallyCommitKey() + throws IOException, TimeoutException, InterruptedException, + AuthenticationException { + OzoneConfiguration conf = createConfAndInitValues(); + ScmBlockLocationProtocol blockClient = + //failCallsFrequency = 1 , means all calls fail. + new ScmBlockLocationTestingClient(null, null, 1); + OmTestManagers omTestManagers + = new OmTestManagers(conf, blockClient, null); + KeyManager keyManager = omTestManagers.getKeyManager(); + writeClient = omTestManagers.getWriteClient(); + om = omTestManagers.getOzoneManager(); + + String volumeName = String.format("volume%s", + RandomStringUtils.randomAlphanumeric(5)); + String bucketName = String.format("bucket%s", + RandomStringUtils.randomAlphanumeric(5)); + String keyName = String.format("key%s", + RandomStringUtils.randomAlphanumeric(5)); + + // Create Volume and Bucket + createVolumeAndBucket(keyManager, volumeName, bucketName, false); + + OmKeyArgs keyArg = createAndCommitKey(keyManager, volumeName, bucketName, + keyName, 3, 1); + + // Only the uncommitted block should be pending to be deleted. + GenericTestUtils.waitFor( + () -> { + try { + return keyManager.getPendingDeletionKeys(Integer.MAX_VALUE) + .stream() + .map(BlockGroup::getBlockIDList) + .flatMap(Collection::stream) + .collect(Collectors.toList()).size() == 1; + } catch (IOException e) { + e.printStackTrace(); + } + return false; + }, + 500, 3000); + + // Delete the key + writeClient.deleteKey(keyArg); + + KeyDeletingService keyDeletingService = + (KeyDeletingService) keyManager.getDeletingService(); + + // All blocks should be pending to be deleted. + GenericTestUtils.waitFor( + () -> { + try { + return keyManager.getPendingDeletionKeys(Integer.MAX_VALUE) + .stream() + .map(BlockGroup::getBlockIDList) + .flatMap(Collection::stream) + .collect(Collectors.toList()).size() == 3; + } catch (IOException e) { + e.printStackTrace(); + } + return false; + }, + 500, 3000); + + // the blockClient is set to fail the deletion of key blocks, hence no keys + // will be deleted + Assert.assertEquals(0, keyDeletingService.getDeletedKeyCount().get()); } @Test(timeout = 30000) @@ -299,6 +377,15 @@ private void createVolumeAndBucket(KeyManager keyManager, String volumeName, private OmKeyArgs createAndCommitKey(KeyManager keyManager, String volumeName, String bucketName, String keyName, int numBlocks) throws IOException { + return createAndCommitKey(keyManager, volumeName, bucketName, keyName, + numBlocks, 0); + } + + private OmKeyArgs createAndCommitKey(KeyManager keyManager, String volumeName, + String bucketName, String keyName, int numBlocks, int numUncommitted) + throws IOException { + // Even if no key size is appointed, there will be at least one + // block pre-allocated when key is created OmKeyArgs keyArg = new OmKeyArgs.Builder() .setVolumeName(volumeName) @@ -311,10 +398,35 @@ private OmKeyArgs createAndCommitKey(KeyManager keyManager, String volumeName, .build(); //Open and Commit the Key in the Key Manager. OpenKeySession session = writeClient.openKey(keyArg); - for (int i = 0; i < numBlocks; i++) { - keyArg.addLocationInfo(writeClient.allocateBlock(keyArg, session.getId(), + + // add pre-allocated blocks into args and avoid creating excessive block + OmKeyLocationInfoGroup keyLocationVersions = session.getKeyInfo(). + getLatestVersionLocations(); + assert keyLocationVersions != null; + List latestBlocks = keyLocationVersions. + getBlocksLatestVersionOnly(); + int preAllocatedSize = latestBlocks.size(); + for (OmKeyLocationInfo block : latestBlocks) { + keyArg.addLocationInfo(block); + } + + // allocate blocks until the blocks num equal to numBlocks + LinkedList allocated = new LinkedList<>(); + for (int i = 0; i < numBlocks - preAllocatedSize; i++) { + allocated.add(writeClient.allocateBlock(keyArg, session.getId(), new ExcludeList())); } + + // remove the blocks not to be committed + for (int i = 0; i < numUncommitted; i++) { + allocated.removeFirst(); + } + + // add the blocks to be committed + for (OmKeyLocationInfo block: allocated) { + keyArg.addLocationInfo(block); + } + writeClient.commitKey(keyArg, session.getId()); return keyArg; } From 5b7f448a5353db7d7840f7a503a8e8a4908c759f Mon Sep 17 00:00:00 2001 From: Christos Bisias Date: Tue, 25 Oct 2022 21:47:56 +0300 Subject: [PATCH 25/48] HDDS-7121. Support namespace summaries (du, dist & counts) for legacy FS buckets (#3746) --- .../hadoop/ozone/om/helpers/BucketLayout.java | 13 + .../src/main/compose/ozone-legacy-bucket/.env | 20 + .../compose/ozone-legacy-bucket/README.md | 21 + .../ozone-legacy-bucket/docker-compose.yaml | 78 + .../compose/ozone-legacy-bucket/docker-config | 52 + .../main/compose/ozone-legacy-bucket/test.sh | 35 + ...-nssummary.robot => recon-nssummary.robot} | 12 +- .../ozone/shell/TestNSSummaryAdmin.java | 6 +- .../ozone/recon/ReconControllerModule.java | 4 +- .../recon/api/handlers/BucketHandler.java | 30 +- .../api/handlers/DirectoryEntityHandler.java | 20 +- .../api/handlers/LegacyBucketHandler.java | 325 +++++ .../ozone/recon/api/types/NSSummary.java | 4 +- .../ozone/recon/tasks/NSSummaryTask.java | 232 ++- .../tasks/NSSummaryTaskDbEventHandler.java | 197 +++ .../recon/tasks/NSSummaryTaskWithFSO.java | 42 +- .../recon/tasks/NSSummaryTaskWithLegacy.java | 307 ++++ .../recon/OMMetadataManagerTestUtils.java | 24 +- .../api/TestNSSummaryEndpointWithFSO.java | 10 +- .../api/TestNSSummaryEndpointWithLegacy.java | 1292 +++++++++++++++++ .../ozone/recon/tasks/TestNSSummaryTask.java | 492 +++++++ .../recon/tasks/TestNSSummaryTaskWithFSO.java | 13 +- .../tasks/TestNSSummaryTaskWithLegacy.java | 740 ++++++++++ .../admin/nssummary/DiskUsageSubCommand.java | 7 +- .../nssummary/FileSizeDistSubCommand.java | 7 +- .../ozone/admin/nssummary/NSSummaryAdmin.java | 57 + .../admin/nssummary/NSSummaryCLIUtils.java | 9 +- .../admin/nssummary/QuotaUsageSubCommand.java | 7 +- .../admin/nssummary/SummarySubCommand.java | 7 +- 29 files changed, 3843 insertions(+), 220 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/.env create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/README.md create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-compose.yaml create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-config create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/test.sh rename hadoop-ozone/dist/src/main/smoketest/recon/{recon-fso-nssummary.robot => recon-nssummary.robot} (92%) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java index 8a17777b1101..68147fab4465 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java @@ -70,6 +70,19 @@ public boolean isLegacy() { return this.equals(LEGACY); } + public boolean isObjectStore(boolean enableFileSystemPaths) { + if (this.equals(OBJECT_STORE)) { + return true; + } else { + // If bucket layout is Legacy and FileSystemPaths + // are disabled, then the bucket operates as OBS. + if (this.equals(LEGACY) && !enableFileSystemPaths) { + return true; + } + return false; + } + } + public boolean shouldNormalizePaths(boolean enableFileSystemPaths) { switch (this) { case OBJECT_STORE: diff --git a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/.env b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/.env new file mode 100644 index 000000000000..2de359fc5dbf --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/.env @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HDDS_VERSION=${hdds.version} +OZONE_RUNNER_VERSION=${docker.ozone-runner.version} +OZONE_RUNNER_IMAGE=apache/ozone-runner +OZONE_OPTS= diff --git a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/README.md b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/README.md new file mode 100644 index 000000000000..d31d8f20fb2f --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/README.md @@ -0,0 +1,21 @@ + + +# For Legacy Bucket Operations + +For Legacy buckets, set `ozone.om.enable.filesystem.paths` to `true` for them to behave like FSO buckets, +otherwise Legacy buckets act like OBS buckets. + +This is the same as `compose/ozone` but for testing operations that need `ozone.om.enable.filesystem.paths` +flag enabled. \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-compose.yaml new file mode 100644 index 000000000000..72303abaf63b --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-compose.yaml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.4" + +# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields) +x-common-config: + &common-config + image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + env_file: + - docker-config + +x-replication: + &replication + OZONE-SITE.XML_ozone.replication: ${OZONE_REPLICATION_FACTOR:-1} + +services: + datanode: + <<: *common-config + ports: + - 9864 + - 9882 + environment: + <<: *replication + OZONE_OPTS: + command: ["ozone","datanode"] + om: + <<: *common-config + environment: + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + OZONE_OPTS: + <<: *replication + ports: + - 9874:9874 + - 9862:9862 + command: ["ozone","om"] + scm: + <<: *common-config + ports: + - 9876:9876 + - 9860:9860 + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} + OZONE_OPTS: + <<: *replication + command: ["ozone","scm"] + s3g: + <<: *common-config + environment: + OZONE_OPTS: + <<: *replication + ports: + - 9878:9878 + command: ["ozone","s3g"] + recon: + <<: *common-config + ports: + - 9888:9888 + environment: + OZONE_OPTS: + <<: *replication + command: ["ozone","recon"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-config new file mode 100644 index 000000000000..90d62dcd0047 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/docker-config @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.XML_fs.defaultFS=ofs://om +CORE-SITE.XML_fs.trash.interval=1 + +OZONE-SITE.XML_ozone.om.address=om +OZONE-SITE.XML_ozone.om.enable.filesystem.paths=true +OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY +OZONE-SITE.XML_ozone.om.http-address=om:9874 +OZONE-SITE.XML_ozone.scm.http-address=scm:9876 +OZONE-SITE.XML_ozone.scm.container.size=1GB +OZONE-SITE.XML_ozone.scm.block.size=1MB +OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB +OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s +OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_hdds.datanode.dir=/data/hdds +OZONE-SITE.XML_ozone.recon.address=recon:9891 +OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888 +OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889 +OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m +OZONE-SITE.XML_ozone.datanode.pipeline.limit=1 +OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s +OZONE-SITE.XML_hdds.container.report.interval=60s +OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true +OZONE-SITE.XML_ozone.scm.stale.node.interval=30s +OZONE-SITE.XML_ozone.scm.dead.node.interval=45s +OZONE-SITE.XML_hdds.heartbeat.interval=5s + +OZONE_CONF_DIR=/etc/hadoop +OZONE_LOG_DIR=/var/log/hadoop + +no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/test.sh new file mode 100644 index 000000000000..4f776686a549 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/test.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#suite:unsecure + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +export SECURITY_ENABLED=false +export OZONE_REPLICATION_FACTOR=3 + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +start_docker_env 5 + +execute_robot_test scm -v BUCKET_LAYOUT:LEGACY recon/recon-nssummary.robot + +stop_docker_env + +generate_report diff --git a/hadoop-ozone/dist/src/main/smoketest/recon/recon-fso-nssummary.robot b/hadoop-ozone/dist/src/main/smoketest/recon/recon-nssummary.robot similarity index 92% rename from hadoop-ozone/dist/src/main/smoketest/recon/recon-fso-nssummary.robot rename to hadoop-ozone/dist/src/main/smoketest/recon/recon-nssummary.robot index 5994e487f90a..86ca25e219ff 100644 --- a/hadoop-ozone/dist/src/main/smoketest/recon/recon-fso-nssummary.robot +++ b/hadoop-ozone/dist/src/main/smoketest/recon/recon-nssummary.robot @@ -14,7 +14,7 @@ # limitations under the License. *** Settings *** -Documentation Smoke test for Recon Namespace Summary Endpoint for FSO buckets. +Documentation Smoke test for Recon Namespace Summary Endpoint for ${BUCKET_LAYOUT} buckets. Library OperatingSystem Library String Library BuiltIn @@ -29,6 +29,7 @@ ${SUMMARY_URL} ${ADMIN_NAMESPACE_URL}/summary ${DISK_USAGE_URL} ${ADMIN_NAMESPACE_URL}/du ${QUOTA_USAGE_URL} ${ADMIN_NAMESPACE_URL}/quota ${FILE_SIZE_DIST_URL} ${ADMIN_NAMESPACE_URL}/dist +${BUCKET_LAYOUT} FILE_SYSTEM_OPTIMIZED ${VOLUME} ${BUCKET} @@ -42,7 +43,7 @@ Create volume Create bucket ${random} = Generate Random String 5 [LOWER] Set Suite Variable ${BUCKET} buc-${random} - ${result} = Execute ozone sh bucket create -l FILE_SYSTEM_OPTIMIZED /${VOLUME}/${BUCKET} + ${result} = Execute ozone sh bucket create -l ${BUCKET_LAYOUT} /${VOLUME}/${BUCKET} Should not contain ${result} Failed Create keys @@ -83,7 +84,7 @@ Check Access kinit as recon admin Check http return code ${url} 200 -Test Summary +Test Summary [Arguments] ${url} ${expected} ${result} = Execute curl --negotiate -u : -LSs ${url} Should contain ${result} \"status\":\"OK\" @@ -131,7 +132,8 @@ Check Recon Namespace Summary Key Wait For Summary ${SUMMARY_URL}?path=/${VOLUME}/${BUCKET}/file1 KEY Check Recon Namespace Summary Directory - Wait For Summary ${SUMMARY_URL}?path=/${VOLUME}/${BUCKET}/dir1/dir2 DIRECTORY + Run Keyword If '${BUCKET_LAYOUT}' == 'LEGACY' Wait For Summary ${SUMMARY_URL}?path=/${VOLUME}/${BUCKET}/dir1/dir2/ DIRECTORY + Run Keyword If '${BUCKET_LAYOUT}' == 'FILE_SYSTEM_OPTIMIZED' Wait For Summary ${SUMMARY_URL}?path=/${VOLUME}/${BUCKET}/dir1/dir2 DIRECTORY Check Recon Namespace Disk Usage Wait For Summary ${DISK_USAGE_URL}?path=/${VOLUME}/${BUCKET}&files=true&replica=true \"sizeWithReplica\" @@ -143,4 +145,4 @@ Check Recon Namespace Bucket Quota Usage Wait For Summary ${QUOTA_USAGE_URL}?path=/${VOLUME}/${BUCKET} \"used\" Check Recon Namespace File Size Distribution Root - Wait For Summary ${FILE_SIZE_DIST_URL}?path=/ \"dist\" + Wait For Summary ${FILE_SIZE_DIST_URL}?path=/ \"dist\" \ No newline at end of file diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java index 69c4a762e6c0..c0b5e82a0b61 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java @@ -110,7 +110,7 @@ public void testNSSummaryCLIRoot() throws UnsupportedEncodingException { // Should throw warning - only buckets can have bucket layout. Assert.assertTrue( getOutContentString().contains( - "[Warning] Namespace CLI is only designed for FSO mode.")); + "[Warning] Namespace CLI is not designed for OBS bucket layout.")); Assert.assertTrue(getOutContentString() .contains("Put more files into it to visualize DU")); Assert.assertTrue(getOutContentString().contains( @@ -128,7 +128,7 @@ public void testNSSummaryCLIFSO() throws UnsupportedEncodingException { // Should not throw warning, since bucket is in FSO bucket layout. Assert.assertFalse( getOutContentString().contains( - "[Warning] Namespace CLI is only designed for FSO mode.")); + "[Warning] Namespace CLI is not designed for OBS bucket layout.")); Assert.assertTrue(getOutContentString() .contains("Put more files into it to visualize DU")); Assert.assertTrue(getOutContentString().contains( @@ -146,7 +146,7 @@ public void testNSSummaryCLIOBS() throws UnsupportedEncodingException { // Should throw warning, since bucket is in OBS bucket layout. Assert.assertTrue( getOutContentString().contains( - "[Warning] Namespace CLI is only designed for FSO mode.")); + "[Warning] Namespace CLI is not designed for OBS bucket layout.")); Assert.assertTrue(getOutContentString() .contains("Put more files into it to visualize DU")); Assert.assertTrue(getOutContentString().contains( diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java index 6892524e829a..61fefabee50d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java @@ -47,7 +47,7 @@ import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider; import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTask; import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask; import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTask; import org.apache.hadoop.ozone.recon.tasks.TableCountTask; @@ -126,7 +126,7 @@ protected void configure() { taskBinder.addBinding().to(ContainerKeyMapperTask.class); taskBinder.addBinding().to(FileSizeCountTask.class); taskBinder.addBinding().to(TableCountTask.class); - taskBinder.addBinding().to(NSSummaryTaskWithFSO.class); + taskBinder.addBinding().to(NSSummaryTask.class); } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java index 2815fb6d1332..d12c3cdb6ad3 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/BucketHandler.java @@ -38,6 +38,8 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Objects; + import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.removeTrailingSlashIfNeeded; @@ -183,8 +185,24 @@ public static BucketHandler getBucketHandler( OzoneStorageContainerManager reconSCM, OmBucketInfo bucketInfo) throws IOException { - return new FSOBucketHandler(reconNamespaceSummaryManager, - omMetadataManager, reconSCM, bucketInfo); + // If bucketInfo is null then entity type is UNKNOWN + if (Objects.isNull(bucketInfo)) { + return null; + } else { + if (bucketInfo.getBucketLayout() + .equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) { + return new FSOBucketHandler(reconNamespaceSummaryManager, + omMetadataManager, reconSCM, bucketInfo); + } else if (bucketInfo.getBucketLayout() + .equals(BucketLayout.LEGACY)) { + return new LegacyBucketHandler(reconNamespaceSummaryManager, + omMetadataManager, reconSCM, bucketInfo); + } else { + LOG.error("Unsupported bucket layout: " + + bucketInfo.getBucketLayout()); + return null; + } + } } public static BucketHandler getBucketHandler( @@ -197,11 +215,7 @@ public static BucketHandler getBucketHandler( OmBucketInfo bucketInfo = omMetadataManager .getBucketTable().getSkipCache(bucketKey); - if (bucketInfo == null) { - return null; - } else { - return getBucketHandler(reconNamespaceSummaryManager, - omMetadataManager, reconSCM, bucketInfo); - } + return getBucketHandler(reconNamespaceSummaryManager, + omMetadataManager, reconSCM, bucketInfo); } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java index f1058ddf3db1..0cfa6f1b47fc 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/DirectoryEntityHandler.java @@ -30,6 +30,8 @@ import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -89,7 +91,23 @@ public DUResponse getDuResponse( for (long subdirObjectId: subdirs) { NSSummary subdirNSSummary = getReconNamespaceSummaryManager().getNSSummary(subdirObjectId); - String subdirName = subdirNSSummary.getDirName(); + // for the subdirName we need the subdir filename, not the key name + // Eg. /vol/bucket1/dir1/dir2, + // key name is /dir1/dir2 + // we need to get dir2 + Path subdirPath = Paths.get(subdirNSSummary.getDirName()); + Path subdirFileName = subdirPath.getFileName(); + String subdirName; + // Checking for null to get rid of a findbugs error and + // then throwing the NPException to avoid swallowing it. + // Error: Possible null pointer dereference in + // ...DirectoryEntityHandler.getDuResponse(boolean, boolean) due to + // return value of called method Dereferenced at DirectoryEntityHandler + if (subdirFileName != null) { + subdirName = subdirFileName.toString(); + } else { + throw new NullPointerException("Subdirectory file name is null."); + } // build the path for subdirectory String subpath = BucketHandler .buildSubpath(getNormalizedPath(), subdirName); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java new file mode 100644 index 000000000000..e4d218fed9b3 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.recon.api.handlers; + +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; +import org.apache.hadoop.ozone.recon.api.types.EntityType; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; + +/** + * Class for handling Legacy buckets. + */ +public class LegacyBucketHandler extends BucketHandler { + + private static final Logger LOG = LoggerFactory.getLogger( + LegacyBucketHandler.class); + + private final String vol; + private final String bucket; + private final OmBucketInfo omBucketInfo; + + public LegacyBucketHandler( + ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager omMetadataManager, + OzoneStorageContainerManager reconSCM, + OmBucketInfo bucketInfo) { + super(reconNamespaceSummaryManager, omMetadataManager, + reconSCM); + this.omBucketInfo = bucketInfo; + this.vol = omBucketInfo.getVolumeName(); + this.bucket = omBucketInfo.getBucketName(); + } + + /** + * Helper function to check if a path is a directory, key, or invalid. + * @param keyName key name + * @return DIRECTORY, KEY, or UNKNOWN + * @throws IOException + */ + @Override + public EntityType determineKeyPath(String keyName) + throws IOException { + + String filename = OzoneFSUtils.removeTrailingSlashIfNeeded(keyName); + // For example, /vol1/buck1/a/b/c/d/e/file1.txt + // Look in the KeyTable for the key path, + // if the first one we seek to is the same as the seek key, + // it is a key; + // if it is the seekKey with a trailing slash, it is a directory + // else it is unknown + String key = OM_KEY_PREFIX + vol + + OM_KEY_PREFIX + bucket + + OM_KEY_PREFIX + filename; + + Table keyTable = getKeyTable(); + + TableIterator> + iterator = keyTable.iterator(); + + iterator.seek(key); + if (iterator.hasNext()) { + Table.KeyValue kv = iterator.next(); + String dbKey = kv.getKey(); + if (dbKey.equals(key)) { + return EntityType.KEY; + } + if (dbKey.equals(key + OM_KEY_PREFIX)) { + return EntityType.DIRECTORY; + } + } + return EntityType.UNKNOWN; + } + + /** + * KeyTable's key is in the format of "vol/bucket/keyName". + * Make use of RocksDB's order to seek to the prefix and avoid full iteration. + * Calculating DU only for keys. Skipping any directories and + * handling only direct keys. + * @param parentId + * @return total DU of direct keys under object + * @throws IOException + */ + @Override + public long calculateDUUnderObject(long parentId) + throws IOException { + Table keyTable = getKeyTable(); + + long totalDU = 0L; + TableIterator> + iterator = keyTable.iterator(); + + String seekPrefix = OM_KEY_PREFIX + + vol + + OM_KEY_PREFIX + + bucket + + OM_KEY_PREFIX; + + NSSummary nsSummary = getReconNamespaceSummaryManager() + .getNSSummary(parentId); + // empty bucket + if (nsSummary == null) { + return 0; + } + + if (omBucketInfo.getObjectID() != parentId) { + String dirName = nsSummary.getDirName(); + seekPrefix += dirName; + } + + String[] seekKeys = seekPrefix.split(OM_KEY_PREFIX); + iterator.seek(seekPrefix); + // handle direct keys + while (iterator.hasNext()) { + Table.KeyValue kv = iterator.next(); + String dbKey = kv.getKey(); + // since the RocksDB is ordered, seek until the prefix isn't matched + if (!dbKey.startsWith(seekPrefix)) { + break; + } + + String[] keys = dbKey.split(OM_KEY_PREFIX); + + // iteration moved to the next level + // and not handling direct keys + if (keys.length - seekKeys.length > 1) { + continue; + } + + OmKeyInfo keyInfo = kv.getValue(); + if (keyInfo != null) { + // skip directory markers, just include directKeys + if (keyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) { + continue; + } + totalDU += getKeySizeWithReplication(keyInfo); + } + } + + // handle nested keys (DFS) + Set subDirIds = nsSummary.getChildDir(); + for (long subDirId: subDirIds) { + totalDU += calculateDUUnderObject(subDirId); + } + return totalDU; + } + + /** + * This method handles disk usage of direct keys. + * @param parentId parent directory/bucket + * @param withReplica if withReplica is enabled, set sizeWithReplica + * for each direct key's DU + * @param listFile if listFile is enabled, append key DU as a subpath + * @param duData the current DU data + * @param normalizedPath the normalized path request + * @return the total DU of all direct keys + * @throws IOException IOE + */ + @Override + public long handleDirectKeys(long parentId, boolean withReplica, + boolean listFile, + List duData, + String normalizedPath) throws IOException { + + Table keyTable = getKeyTable(); + long keyDataSizeWithReplica = 0L; + + TableIterator> + iterator = keyTable.iterator(); + + String seekPrefix = OM_KEY_PREFIX + + vol + + OM_KEY_PREFIX + + bucket + + OM_KEY_PREFIX; + + NSSummary nsSummary = getReconNamespaceSummaryManager() + .getNSSummary(parentId); + // empty bucket + if (nsSummary == null) { + return 0; + } + + if (omBucketInfo.getObjectID() != parentId) { + String dirName = nsSummary.getDirName(); + seekPrefix += dirName; + } + String[] seekKeys = seekPrefix.split(OM_KEY_PREFIX); + iterator.seek(seekPrefix); + + while (iterator.hasNext()) { + Table.KeyValue kv = iterator.next(); + String dbKey = kv.getKey(); + + if (!dbKey.startsWith(seekPrefix)) { + break; + } + + String[] keys = dbKey.split(OM_KEY_PREFIX); + + // iteration moved to the next level + // and not handling direct keys + if (keys.length - seekKeys.length > 1) { + continue; + } + + OmKeyInfo keyInfo = kv.getValue(); + if (keyInfo != null) { + // skip directory markers, just include directKeys + if (keyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) { + continue; + } + DUResponse.DiskUsage diskUsage = new DUResponse.DiskUsage(); + String subpath = buildSubpath(normalizedPath, + keyInfo.getFileName()); + diskUsage.setSubpath(subpath); + diskUsage.setKey(true); + diskUsage.setSize(keyInfo.getDataSize()); + + if (withReplica) { + long keyDU = getKeySizeWithReplication(keyInfo); + keyDataSizeWithReplica += keyDU; + diskUsage.setSizeWithReplica(keyDU); + } + // list the key as a subpath + if (listFile) { + duData.add(diskUsage); + } + } + } + + return keyDataSizeWithReplica; + } + + /** + * Given a valid path request for a directory, + * return the directory object ID. + * @param names parsed path request in a list of names + * @return directory object ID + */ + @Override + public long getDirObjectId(String[] names) throws IOException { + return getDirObjectId(names, names.length); + } + + /** + * Given a valid path request and a cutoff length where should be iterated + * up to. + * return the directory object ID for the object at the cutoff length + * @param names parsed path request in a list of names + * @param cutoff cannot be larger than the names' length. If equals, + * return the directory object id for the whole path + * @return directory object ID + */ + @Override + public long getDirObjectId(String[] names, int cutoff) throws IOException { + long dirObjectId = getBucketObjectId(names); + StringBuilder bld = new StringBuilder(); + for (int i = 0; i < cutoff; ++i) { + bld.append(OM_KEY_PREFIX) + .append(names[i]); + } + bld.append(OM_KEY_PREFIX); + String dirKey = bld.toString(); + OmKeyInfo dirInfo = getKeyTable().getSkipCache(dirKey); + + if (dirInfo != null) { + dirObjectId = dirInfo.getObjectID(); + } else { + throw new IOException("OmKeyInfo for the directory is null"); + } + + return dirObjectId; + } + + @Override + public BucketLayout getBucketLayout() { + return BucketLayout.LEGACY; + } + + @Override + public OmKeyInfo getKeyInfo(String[] names) throws IOException { + String ozoneKey = OM_KEY_PREFIX; + ozoneKey += String.join(OM_KEY_PREFIX, names); + + OmKeyInfo keyInfo = getKeyTable().getSkipCache(ozoneKey); + return keyInfo; + } + + public Table getKeyTable() { + Table keyTable = + getOmMetadataManager().getKeyTable(getBucketLayout()); + return keyTable; + } +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java index 1b22081f5244..eeb501499131 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java @@ -24,6 +24,8 @@ import java.util.HashSet; import java.util.Arrays; +import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.removeTrailingSlashIfNeeded; + /** * Class to encapsulate namespace metadata summaries from OM. */ @@ -90,7 +92,7 @@ public void setChildDir(Set childDir) { } public void setDirName(String dirName) { - this.dirName = dirName; + this.dirName = removeTrailingSlashIfNeeded(dirName); } public void addChildDir(long childId) { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java index 7baeefdbe4c6..63b6ee375c35 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java @@ -17,25 +17,30 @@ */ package org.apache.hadoop.ozone.recon.tasks; +import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.recon.ReconUtils; -import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.inject.Inject; import java.io.IOException; -import java.util.Map; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Callable; +import java.util.concurrent.Future; /** * Task to query data from OMDB and write into Recon RocksDB. - * Reprocess() will take a snapshots on OMDB, and iterate the keyTable and - * dirTable to write all information to RocksDB. + * Reprocess() will take a snapshots on OMDB, and iterate the keyTable, + * the fileTable and the dirTable to write all information to RocksDB. * * For FSO-enabled keyTable (fileTable), we need to fetch the parent object * (bucket or directory), increment its numOfKeys by 1, increase its sizeOfKeys @@ -44,166 +49,99 @@ * For dirTable, we need to fetch the parent object (bucket or directory), * add the current directory's objectID to the parent object's childDir field. * + * For keyTable, the parent object is not available. Get the parent object, + * add it to the current object and reuse the existing methods for FSO. + * Only processing entries that belong to Legacy buckets. If the entry + * refers to a directory then build directory info object from it. + * * Process() will write all OMDB updates to RocksDB. - * The write logic is the same as above. For update action, we will treat it as + * Write logic is the same as above. For update action, we will treat it as * delete old value first, and write updated value then. */ -public abstract class NSSummaryTask implements ReconOmTask { +public class NSSummaryTask implements ReconOmTask { private static final Logger LOG = LoggerFactory.getLogger(NSSummaryTask.class); private final ReconNamespaceSummaryManager reconNamespaceSummaryManager; + private final ReconOMMetadataManager reconOMMetadataManager; + private final NSSummaryTaskWithFSO nsSummaryTaskWithFSO; + private final NSSummaryTaskWithLegacy nsSummaryTaskWithLegacy; + private final OzoneConfiguration ozoneConfiguration; @Inject public NSSummaryTask(ReconNamespaceSummaryManager - reconNamespaceSummaryManager) { + reconNamespaceSummaryManager, + ReconOMMetadataManager + reconOMMetadataManager, + OzoneConfiguration + ozoneConfiguration) { this.reconNamespaceSummaryManager = reconNamespaceSummaryManager; + this.reconOMMetadataManager = reconOMMetadataManager; + this.ozoneConfiguration = ozoneConfiguration; + this.nsSummaryTaskWithFSO = new NSSummaryTaskWithFSO( + reconNamespaceSummaryManager, reconOMMetadataManager); + this.nsSummaryTaskWithLegacy = new NSSummaryTaskWithLegacy( + reconNamespaceSummaryManager, + reconOMMetadataManager, ozoneConfiguration); } - public ReconNamespaceSummaryManager getReconNamespaceSummaryManager() { - return reconNamespaceSummaryManager; - } - - public abstract String getTaskName(); - - public abstract Pair process(OMUpdateEventBatch events); - - public abstract Pair reprocess( - OMMetadataManager omMetadataManager); - - protected void writeNSSummariesToDB(Map nsSummaryMap) - throws IOException { - try (RDBBatchOperation rdbBatchOperation = new RDBBatchOperation()) { - nsSummaryMap.keySet().forEach((Long key) -> { - try { - reconNamespaceSummaryManager.batchStoreNSSummaries(rdbBatchOperation, - key, nsSummaryMap.get(key)); - } catch (IOException e) { - LOG.error("Unable to write Namespace Summary data in Recon DB.", - e); - } - }); - reconNamespaceSummaryManager.commitBatchOperation(rdbBatchOperation); - } - } - - protected void handlePutKeyEvent(OmKeyInfo keyInfo, Map nsSummaryMap) throws IOException { - long parentObjectId = keyInfo.getParentObjectID(); - // Try to get the NSSummary from our local map that maps NSSummaries to IDs - NSSummary nsSummary = nsSummaryMap.get(parentObjectId); - if (nsSummary == null) { - // If we don't have it in this batch we try to get it from the DB - nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); - } - if (nsSummary == null) { - // If we don't have it locally and in the DB we create a new instance - // as this is a new ID - nsSummary = new NSSummary(); - } - int numOfFile = nsSummary.getNumOfFiles(); - long sizeOfFile = nsSummary.getSizeOfFiles(); - int[] fileBucket = nsSummary.getFileSizeBucket(); - nsSummary.setNumOfFiles(numOfFile + 1); - long dataSize = keyInfo.getDataSize(); - nsSummary.setSizeOfFiles(sizeOfFile + dataSize); - int binIndex = ReconUtils.getBinIndex(dataSize); - - ++fileBucket[binIndex]; - nsSummary.setFileSizeBucket(fileBucket); - nsSummaryMap.put(parentObjectId, nsSummary); - } - - protected void handlePutDirEvent(OmDirectoryInfo directoryInfo, - Map nsSummaryMap) - throws IOException { - long parentObjectId = directoryInfo.getParentObjectID(); - long objectId = directoryInfo.getObjectID(); - // write the dir name to the current directory - String dirName = directoryInfo.getName(); - // Try to get the NSSummary from our local map that maps NSSummaries to IDs - NSSummary curNSSummary = nsSummaryMap.get(objectId); - if (curNSSummary == null) { - // If we don't have it in this batch we try to get it from the DB - curNSSummary = reconNamespaceSummaryManager.getNSSummary(objectId); - } - if (curNSSummary == null) { - // If we don't have it locally and in the DB we create a new instance - // as this is a new ID - curNSSummary = new NSSummary(); - } - curNSSummary.setDirName(dirName); - nsSummaryMap.put(objectId, curNSSummary); - - // Write the child dir list to the parent directory - // Try to get the NSSummary from our local map that maps NSSummaries to IDs - NSSummary nsSummary = nsSummaryMap.get(parentObjectId); - if (nsSummary == null) { - // If we don't have it in this batch we try to get it from the DB - nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); - } - if (nsSummary == null) { - // If we don't have it locally and in the DB we create a new instance - // as this is a new ID - nsSummary = new NSSummary(); - } - nsSummary.addChildDir(objectId); - nsSummaryMap.put(parentObjectId, nsSummary); + @Override + public String getTaskName() { + return "NSSummaryTask"; } - protected void handleDeleteKeyEvent(OmKeyInfo keyInfo, - Map nsSummaryMap) - throws IOException { - long parentObjectId = keyInfo.getParentObjectID(); - // Try to get the NSSummary from our local map that maps NSSummaries to IDs - NSSummary nsSummary = nsSummaryMap.get(parentObjectId); - if (nsSummary == null) { - // If we don't have it in this batch we try to get it from the DB - nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); + @Override + public Pair process(OMUpdateEventBatch events) { + boolean success; + success = nsSummaryTaskWithFSO.processWithFSO(events); + if (success) { + success = nsSummaryTaskWithLegacy.processWithLegacy(events); + } else { + LOG.error("processWithFSO failed."); } - - // Just in case the OmKeyInfo isn't correctly written. - if (nsSummary == null) { - LOG.error("The namespace table is not correctly populated."); - return; - } - int numOfFile = nsSummary.getNumOfFiles(); - long sizeOfFile = nsSummary.getSizeOfFiles(); - int[] fileBucket = nsSummary.getFileSizeBucket(); - - long dataSize = keyInfo.getDataSize(); - int binIndex = ReconUtils.getBinIndex(dataSize); - - // decrement count, data size, and bucket count - // even if there's no direct key, we still keep the entry because - // we still need children dir IDs info - nsSummary.setNumOfFiles(numOfFile - 1); - nsSummary.setSizeOfFiles(sizeOfFile - dataSize); - --fileBucket[binIndex]; - nsSummary.setFileSizeBucket(fileBucket); - nsSummaryMap.put(parentObjectId, nsSummary); + return new ImmutablePair<>(getTaskName(), success); } - protected void handleDeleteDirEvent(OmDirectoryInfo directoryInfo, - Map nsSummaryMap) - throws IOException { - long parentObjectId = directoryInfo.getParentObjectID(); - long objectId = directoryInfo.getObjectID(); - // Try to get the NSSummary from our local map that maps NSSummaries to IDs - NSSummary nsSummary = nsSummaryMap.get(parentObjectId); - if (nsSummary == null) { - // If we don't have it in this batch we try to get it from the DB - nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); + @Override + public Pair reprocess(OMMetadataManager omMetadataManager) { + Collection> tasks = new ArrayList<>(); + + try { + // reinit Recon RocksDB's namespace CF. + reconNamespaceSummaryManager.clearNSSummaryTable(); + } catch (IOException ioEx) { + LOG.error("Unable to clear NSSummary table in Recon DB. ", + ioEx); + return new ImmutablePair<>(getTaskName(), false); } - // Just in case the OmDirectoryInfo isn't correctly written. - if (nsSummary == null) { - LOG.error("The namespace table is not correctly populated."); - return; + tasks.add(() -> nsSummaryTaskWithFSO + .reprocessWithFSO(omMetadataManager)); + tasks.add(() -> nsSummaryTaskWithLegacy + .reprocessWithLegacy(reconOMMetadataManager)); + + List> results; + ExecutorService executorService = Executors + .newFixedThreadPool(2); + try { + results = executorService.invokeAll(tasks); + for (int i = 0; i < results.size(); i++) { + if (results.get(i).get().equals(false)) { + return new ImmutablePair<>(getTaskName(), false); + } + } + } catch (InterruptedException ex) { + LOG.error("Error while reprocessing NSSummary " + + "table in Recon DB. ", ex); + return new ImmutablePair<>(getTaskName(), false); + } catch (ExecutionException ex2) { + LOG.error("Error while reprocessing NSSummary " + + "table in Recon DB. ", ex2); + return new ImmutablePair<>(getTaskName(), false); + } finally { + executorService.shutdown(); } - - nsSummary.removeChildDir(objectId); - nsSummaryMap.put(parentObjectId, nsSummary); + return new ImmutablePair<>(getTaskName(), true); } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java new file mode 100644 index 000000000000..4cadbf273a15 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.tasks; + +import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +/** + * Class for holding all NSSummaryTask methods + * related to DB operations so that they can commonly be + * used in NSSummaryTaskWithFSO and NSSummaryTaskWithLegacy. + */ +public class NSSummaryTaskDbEventHandler { + + private static final Logger LOG = + LoggerFactory.getLogger(NSSummaryTaskDbEventHandler.class); + + private ReconNamespaceSummaryManager reconNamespaceSummaryManager; + private ReconOMMetadataManager reconOMMetadataManager; + + public NSSummaryTaskDbEventHandler(ReconNamespaceSummaryManager + reconNamespaceSummaryManager, + ReconOMMetadataManager + reconOMMetadataManager) { + this.reconNamespaceSummaryManager = reconNamespaceSummaryManager; + this.reconOMMetadataManager = reconOMMetadataManager; + } + + public ReconNamespaceSummaryManager getReconNamespaceSummaryManager() { + return reconNamespaceSummaryManager; + } + + public ReconOMMetadataManager getReconOMMetadataManager() { + return reconOMMetadataManager; + } + + protected void writeNSSummariesToDB(Map nsSummaryMap) + throws IOException { + try (RDBBatchOperation rdbBatchOperation = new RDBBatchOperation()) { + nsSummaryMap.keySet().forEach((Long key) -> { + try { + reconNamespaceSummaryManager.batchStoreNSSummaries(rdbBatchOperation, + key, nsSummaryMap.get(key)); + } catch (IOException e) { + LOG.error("Unable to write Namespace Summary data in Recon DB.", + e); + } + }); + reconNamespaceSummaryManager.commitBatchOperation(rdbBatchOperation); + } + } + + protected void handlePutKeyEvent(OmKeyInfo keyInfo, Map nsSummaryMap) throws IOException { + long parentObjectId = keyInfo.getParentObjectID(); + // Try to get the NSSummary from our local map that maps NSSummaries to IDs + NSSummary nsSummary = nsSummaryMap.get(parentObjectId); + if (nsSummary == null) { + // If we don't have it in this batch we try to get it from the DB + nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); + } + if (nsSummary == null) { + // If we don't have it locally and in the DB we create a new instance + // as this is a new ID + nsSummary = new NSSummary(); + } + int numOfFile = nsSummary.getNumOfFiles(); + long sizeOfFile = nsSummary.getSizeOfFiles(); + int[] fileBucket = nsSummary.getFileSizeBucket(); + nsSummary.setNumOfFiles(numOfFile + 1); + long dataSize = keyInfo.getDataSize(); + nsSummary.setSizeOfFiles(sizeOfFile + dataSize); + int binIndex = ReconUtils.getBinIndex(dataSize); + + ++fileBucket[binIndex]; + nsSummary.setFileSizeBucket(fileBucket); + nsSummaryMap.put(parentObjectId, nsSummary); + } + + protected void handlePutDirEvent(OmDirectoryInfo directoryInfo, + Map nsSummaryMap) + throws IOException { + long parentObjectId = directoryInfo.getParentObjectID(); + long objectId = directoryInfo.getObjectID(); + // write the dir name to the current directory + String dirName = directoryInfo.getName(); + // Try to get the NSSummary from our local map that maps NSSummaries to IDs + NSSummary curNSSummary = nsSummaryMap.get(objectId); + if (curNSSummary == null) { + // If we don't have it in this batch we try to get it from the DB + curNSSummary = reconNamespaceSummaryManager.getNSSummary(objectId); + } + if (curNSSummary == null) { + // If we don't have it locally and in the DB we create a new instance + // as this is a new ID + curNSSummary = new NSSummary(); + } + curNSSummary.setDirName(dirName); + nsSummaryMap.put(objectId, curNSSummary); + + // Write the child dir list to the parent directory + // Try to get the NSSummary from our local map that maps NSSummaries to IDs + NSSummary nsSummary = nsSummaryMap.get(parentObjectId); + if (nsSummary == null) { + // If we don't have it in this batch we try to get it from the DB + nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); + } + if (nsSummary == null) { + // If we don't have it locally and in the DB we create a new instance + // as this is a new ID + nsSummary = new NSSummary(); + } + nsSummary.addChildDir(objectId); + nsSummaryMap.put(parentObjectId, nsSummary); + } + + protected void handleDeleteKeyEvent(OmKeyInfo keyInfo, + Map nsSummaryMap) + throws IOException { + long parentObjectId = keyInfo.getParentObjectID(); + // Try to get the NSSummary from our local map that maps NSSummaries to IDs + NSSummary nsSummary = nsSummaryMap.get(parentObjectId); + if (nsSummary == null) { + // If we don't have it in this batch we try to get it from the DB + nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); + } + + // Just in case the OmKeyInfo isn't correctly written. + if (nsSummary == null) { + LOG.error("The namespace table is not correctly populated."); + return; + } + int numOfFile = nsSummary.getNumOfFiles(); + long sizeOfFile = nsSummary.getSizeOfFiles(); + int[] fileBucket = nsSummary.getFileSizeBucket(); + + long dataSize = keyInfo.getDataSize(); + int binIndex = ReconUtils.getBinIndex(dataSize); + + // decrement count, data size, and bucket count + // even if there's no direct key, we still keep the entry because + // we still need children dir IDs info + nsSummary.setNumOfFiles(numOfFile - 1); + nsSummary.setSizeOfFiles(sizeOfFile - dataSize); + --fileBucket[binIndex]; + nsSummary.setFileSizeBucket(fileBucket); + nsSummaryMap.put(parentObjectId, nsSummary); + } + + protected void handleDeleteDirEvent(OmDirectoryInfo directoryInfo, + Map nsSummaryMap) + throws IOException { + long parentObjectId = directoryInfo.getParentObjectID(); + long objectId = directoryInfo.getObjectID(); + // Try to get the NSSummary from our local map that maps NSSummaries to IDs + NSSummary nsSummary = nsSummaryMap.get(parentObjectId); + if (nsSummary == null) { + // If we don't have it in this batch we try to get it from the DB + nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); + } + + // Just in case the OmDirectoryInfo isn't correctly written. + if (nsSummary == null) { + LOG.error("The namespace table is not correctly populated."); + return; + } + + nsSummary.removeChildDir(objectId); + nsSummaryMap.put(parentObjectId, nsSummary); + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithFSO.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithFSO.java index 1b8a0ce5a54a..0f80927d8377 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithFSO.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithFSO.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.ozone.recon.tasks; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -26,11 +24,11 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; import java.io.IOException; import java.util.Arrays; import java.util.Collection; @@ -44,20 +42,16 @@ /** * Class for handling FSO specific tasks. */ -public class NSSummaryTaskWithFSO extends NSSummaryTask { +public class NSSummaryTaskWithFSO extends NSSummaryTaskDbEventHandler { private static final Logger LOG = LoggerFactory.getLogger(NSSummaryTaskWithFSO.class); - @Inject public NSSummaryTaskWithFSO(ReconNamespaceSummaryManager - reconNamespaceSummaryManager) { - super(reconNamespaceSummaryManager); - } - - @Override - public String getTaskName() { - return "NSSummaryTaskWithFSO"; + reconNamespaceSummaryManager, + ReconOMMetadataManager + reconOMMetadataManager) { + super(reconNamespaceSummaryManager, reconOMMetadataManager); } // We only listen to updates from FSO-enabled KeyTable(FileTable) and DirTable @@ -65,8 +59,7 @@ public Collection getTaskTables() { return Arrays.asList(FILE_TABLE, DIRECTORY_TABLE); } - @Override - public Pair process(OMUpdateEventBatch events) { + public boolean processWithFSO(OMUpdateEventBatch events) { Iterator eventIterator = events.getIterator(); final Collection taskTables = getTaskTables(); Map nsSummaryMap = new HashMap<>(); @@ -152,7 +145,7 @@ public Pair process(OMUpdateEventBatch events) { } catch (IOException ioEx) { LOG.error("Unable to process Namespace Summary data in Recon DB. ", ioEx); - return new ImmutablePair<>(getTaskName(), false); + return false; } } @@ -160,21 +153,17 @@ public Pair process(OMUpdateEventBatch events) { writeNSSummariesToDB(nsSummaryMap); } catch (IOException e) { LOG.error("Unable to write Namespace Summary data in Recon DB.", e); - return new ImmutablePair<>(getTaskName(), false); + return false; } LOG.info("Completed a process run of NSSummaryTaskWithFSO"); - return new ImmutablePair<>(getTaskName(), true); + return true; } - @Override - public Pair reprocess(OMMetadataManager omMetadataManager) { + public boolean reprocessWithFSO(OMMetadataManager omMetadataManager) { Map nsSummaryMap = new HashMap<>(); try { - // reinit Recon RocksDB's namespace CF. - getReconNamespaceSummaryManager().clearNSSummaryTable(); - Table dirTable = omMetadataManager.getDirectoryTable(); try (TableIterator reprocess(OMMetadataManager omMetadataManager) { } // Get fileTable used by FSO - Table keyTable = omMetadataManager.getFileTable(); + Table keyTable = + omMetadataManager.getFileTable(); try (TableIterator> keyTableIter = keyTable.iterator()) { @@ -202,16 +192,16 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { } catch (IOException ioEx) { LOG.error("Unable to reprocess Namespace Summary data in Recon DB. ", ioEx); - return new ImmutablePair<>(getTaskName(), false); + return false; } try { writeNSSummariesToDB(nsSummaryMap); } catch (IOException e) { LOG.error("Unable to write Namespace Summary data in Recon DB.", e); - return new ImmutablePair<>(getTaskName(), false); + return false; } LOG.info("Completed a reprocess run of NSSummaryTaskWithFSO"); - return new ImmutablePair<>(getTaskName(), true); + return true; } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java new file mode 100644 index 000000000000..6e414a3b4e01 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.tasks; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; + +/** + * Class for handling Legacy specific tasks. + */ +public class NSSummaryTaskWithLegacy extends NSSummaryTaskDbEventHandler { + + private static final BucketLayout BUCKET_LAYOUT = BucketLayout.LEGACY; + + private static final Logger LOG = + LoggerFactory.getLogger(NSSummaryTaskWithLegacy.class); + + private boolean enableFileSystemPaths; + + public NSSummaryTaskWithLegacy(ReconNamespaceSummaryManager + reconNamespaceSummaryManager, + ReconOMMetadataManager + reconOMMetadataManager, + OzoneConfiguration + ozoneConfiguration) { + super(reconNamespaceSummaryManager, reconOMMetadataManager); + // true if FileSystemPaths enabled + enableFileSystemPaths = ozoneConfiguration + .getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, + OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT); + } + + public boolean processWithLegacy(OMUpdateEventBatch events) { + Iterator eventIterator = events.getIterator(); + Map nsSummaryMap = new HashMap<>(); + + while (eventIterator.hasNext()) { + OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); + OMDBUpdateEvent.OMDBUpdateAction action = omdbUpdateEvent.getAction(); + + // we only process updates on OM's KeyTable + String table = omdbUpdateEvent.getTable(); + boolean updateOnKeyTable = table.equals(KEY_TABLE); + if (!updateOnKeyTable) { + continue; + } + + String updatedKey = omdbUpdateEvent.getKey(); + + try { + OMDBUpdateEvent keyTableUpdateEvent = + (OMDBUpdateEvent) omdbUpdateEvent; + OmKeyInfo updatedKeyInfo = keyTableUpdateEvent.getValue(); + OmKeyInfo oldKeyInfo = keyTableUpdateEvent.getOldValue(); + + // KeyTable entries belong to both Legacy and OBS buckets. + // Check bucket layout and if it's OBS + // continue to the next iteration. + // Check just for the current KeyInfo. + String volumeName = updatedKeyInfo.getVolumeName(); + String bucketName = updatedKeyInfo.getBucketName(); + String bucketDBKey = getReconOMMetadataManager() + .getBucketKey(volumeName, bucketName); + // Get bucket info from bucket table + OmBucketInfo omBucketInfo = getReconOMMetadataManager() + .getBucketTable().getSkipCache(bucketDBKey); + + if (omBucketInfo.getBucketLayout() + .isObjectStore(enableFileSystemPaths)) { + continue; + } + + setKeyParentID(updatedKeyInfo); + + if (!updatedKeyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) { + switch (action) { + case PUT: + handlePutKeyEvent(updatedKeyInfo, nsSummaryMap); + break; + + case DELETE: + handleDeleteKeyEvent(updatedKeyInfo, nsSummaryMap); + break; + + case UPDATE: + if (oldKeyInfo != null) { + // delete first, then put + setKeyParentID(oldKeyInfo); + handleDeleteKeyEvent(oldKeyInfo, nsSummaryMap); + } else { + LOG.warn("Update event does not have the old keyInfo for {}.", + updatedKey); + } + handlePutKeyEvent(updatedKeyInfo, nsSummaryMap); + break; + + default: + LOG.debug("Skipping DB update event : {}", + omdbUpdateEvent.getAction()); + } + } else { + OmDirectoryInfo updatedDirectoryInfo = + new OmDirectoryInfo.Builder() + .setName(updatedKeyInfo.getKeyName()) + .setObjectID(updatedKeyInfo.getObjectID()) + .setParentObjectID(updatedKeyInfo.getParentObjectID()) + .build(); + + OmDirectoryInfo oldDirectoryInfo = null; + + if (oldKeyInfo != null) { + oldDirectoryInfo = + new OmDirectoryInfo.Builder() + .setName(oldKeyInfo.getKeyName()) + .setObjectID(oldKeyInfo.getObjectID()) + .setParentObjectID(oldKeyInfo.getParentObjectID()) + .build(); + } + + switch (action) { + case PUT: + handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap); + break; + + case DELETE: + handleDeleteDirEvent(updatedDirectoryInfo, nsSummaryMap); + break; + + case UPDATE: + if (oldDirectoryInfo != null) { + // delete first, then put + handleDeleteDirEvent(oldDirectoryInfo, nsSummaryMap); + } else { + LOG.warn("Update event does not have the old dirInfo for {}.", + updatedKey); + } + handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap); + break; + + default: + LOG.debug("Skipping DB update event : {}", + omdbUpdateEvent.getAction()); + } + } + } catch (IOException ioEx) { + LOG.error("Unable to process Namespace Summary data in Recon DB. ", + ioEx); + return false; + } + } + + try { + writeNSSummariesToDB(nsSummaryMap); + } catch (IOException e) { + LOG.error("Unable to write Namespace Summary data in Recon DB.", e); + return false; + } + + LOG.info("Completed a process run of NSSummaryTaskWithLegacy"); + return true; + } + + public boolean reprocessWithLegacy(OMMetadataManager omMetadataManager) { + Map nsSummaryMap = new HashMap<>(); + + try { + Table keyTable = + omMetadataManager.getKeyTable(BUCKET_LAYOUT); + + try (TableIterator> + keyTableIter = keyTable.iterator()) { + + while (keyTableIter.hasNext()) { + Table.KeyValue kv = keyTableIter.next(); + OmKeyInfo keyInfo = kv.getValue(); + + // KeyTable entries belong to both Legacy and OBS buckets. + // Check bucket layout and if it's OBS + // continue to the next iteration. + String volumeName = keyInfo.getVolumeName(); + String bucketName = keyInfo.getBucketName(); + String bucketDBKey = omMetadataManager + .getBucketKey(volumeName, bucketName); + // Get bucket info from bucket table + OmBucketInfo omBucketInfo = omMetadataManager + .getBucketTable().getSkipCache(bucketDBKey); + + if (omBucketInfo.getBucketLayout() + .isObjectStore(enableFileSystemPaths)) { + continue; + } + + setKeyParentID(keyInfo); + + if (keyInfo.getKeyName().endsWith(OM_KEY_PREFIX)) { + OmDirectoryInfo directoryInfo = + new OmDirectoryInfo.Builder() + .setName(keyInfo.getKeyName()) + .setObjectID(keyInfo.getObjectID()) + .setParentObjectID(keyInfo.getParentObjectID()) + .build(); + handlePutDirEvent(directoryInfo, nsSummaryMap); + } else { + handlePutKeyEvent(keyInfo, nsSummaryMap); + } + } + } + } catch (IOException ioEx) { + LOG.error("Unable to reprocess Namespace Summary data in Recon DB. ", + ioEx); + return false; + } + + try { + writeNSSummariesToDB(nsSummaryMap); + } catch (IOException e) { + LOG.error("Unable to write Namespace Summary data in Recon DB.", e); + return false; + } + LOG.info("Completed a reprocess run of NSSummaryTaskWithLegacy"); + return true; + } + + /** + * KeyTable entries don't have the parentId set. + * In order to reuse the existing FSO methods that rely on + * the parentId, we have to set it explicitly. + * @param keyInfo + * @throws IOException + */ + private void setKeyParentID(OmKeyInfo keyInfo) throws IOException { + String[] keyPath = keyInfo.getKeyName().split(OM_KEY_PREFIX); + + // If the path contains only one key then keyPath.length + // will be 1 and the parent will be a bucket. + // If the keyPath.length is greater than 1 then + // there is at least one directory. + if (keyPath.length > 1) { + String[] dirs = Arrays.copyOf(keyPath, keyPath.length - 1); + String parentKeyName = String.join(OM_KEY_PREFIX, dirs); + parentKeyName += OM_KEY_PREFIX; + String fullParentKeyName = + getReconOMMetadataManager().getOzoneKey(keyInfo.getVolumeName(), + keyInfo.getBucketName(), parentKeyName); + OmKeyInfo parentKeyInfo = getReconOMMetadataManager() + .getKeyTable(BUCKET_LAYOUT) + .getSkipCache(fullParentKeyName); + + if (parentKeyInfo != null) { + keyInfo.setParentObjectID(parentKeyInfo.getObjectID()); + } else { + throw new IOException("ParentKeyInfo for " + + "NSSummaryTaskWithLegacy is null"); + } + } else { + String bucketKey = getReconOMMetadataManager() + .getBucketKey(keyInfo.getVolumeName(), keyInfo.getBucketName()); + OmBucketInfo parentBucketInfo = + getReconOMMetadataManager().getBucketTable().getSkipCache(bucketKey); + + if (parentBucketInfo != null) { + keyInfo.setParentObjectID(parentBucketInfo.getObjectID()); + } else { + throw new IOException("ParentKeyInfo for " + + "NSSummaryTaskWithLegacy is null"); + } + } + } +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java index 8be665a1dfa2..ee51c318b8b1 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java @@ -183,11 +183,11 @@ public static void writeDataToOm(OMMetadataManager omMetadataManager, .build()); } - @SuppressWarnings("checkstyle:parameternumber") /** * Write a key on OM instance. * @throw IOException while writing. */ + @SuppressWarnings("checkstyle:parameternumber") public static void writeKeyToOm(OMMetadataManager omMetadataManager, String key, String bucket, @@ -256,6 +256,28 @@ public static void writeKeyToOm(OMMetadataManager omMetadataManager, .build()); } + /** + * Write a directory as key on OM instance. + * We don't need to set size. + * @throws IOException + */ + @SuppressWarnings("checkstyle:parameternumber") + public static void writeDirToOm(OMMetadataManager omMetadataManager, + String key, + String bucket, + String volume, + String fileName, + long objectID, + long parentObjectId, + long bucketObjectId, + long volumeObjectId, + BucketLayout bucketLayout) + throws IOException { + writeKeyToOm(omMetadataManager, key, bucket, volume, + fileName, objectID, parentObjectId, bucketObjectId, + volumeObjectId, 0, bucketLayout); + } + public static void writeDirToOm(OMMetadataManager omMetadataManager, long objectId, long parentObjectId, diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java index 20e423e89626..c68bab873526 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java @@ -1,4 +1,4 @@ -/*' +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -367,8 +367,9 @@ public void setUp() throws Exception { // populate OM DB and reprocess into Recon RocksDB populateOMDB(); NSSummaryTaskWithFSO nSSummaryTaskWithFso = - new NSSummaryTaskWithFSO(reconNamespaceSummaryManager); - nSSummaryTaskWithFso.reprocess(reconOMMetadataManager); + new NSSummaryTaskWithFSO(reconNamespaceSummaryManager, + reconOMMetadataManager); + nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); } @Test @@ -841,7 +842,6 @@ private void populateOMDB() throws Exception { getBucketLayout()); } - /** * Create a new OM Metadata manager instance with one user, one vol, and two * buckets. @@ -1246,4 +1246,4 @@ private static ReconStorageContainerManagerFacade getMockReconSCM() private static BucketLayout getBucketLayout() { return BucketLayout.FILE_SYSTEM_OPTIMIZED; } -} +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java new file mode 100644 index 000000000000..ccbdd3619577 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java @@ -0,0 +1,1292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.recon.ReconConstants; +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; +import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; +import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; +import org.apache.hadoop.ozone.recon.api.types.EntityType; +import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; +import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithLegacy; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import javax.ws.rs.core.Response; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.ArrayList; +import java.util.Set; +import java.util.HashSet; + +import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test for NSSummary REST APIs with Legacy. + * We tested on a mini file system with the following setting: + * vol + * / \ + * bucket1 bucket2 + * / \ / \ + * file1 dir1 file4 file5 + * / \ \ + * dir2 dir3 dir4 + * / \ \ + * file2 file3 file6 + * ---------------------------------------- + * vol2 + * / \ + * bucket3 bucket4 + * / \ / + * file8 dir5 file11 + * / \ + * file9 file10 + * This is a test for the Rest APIs only. We have tested NSSummaryTask before, + * so there is no need to test process() on DB's updates + */ +public class TestNSSummaryEndpointWithLegacy { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private ReconOMMetadataManager reconOMMetadataManager; + private NSSummaryEndpoint nsSummaryEndpoint; + private OzoneConfiguration conf; + + private static final String TEST_PATH_UTILITY = + "/vol1/buck1/a/b/c/d/e/file1.txt"; + private static final String PARENT_DIR = "vol1/buck1/a/b/c/d/e"; + private static final String[] TEST_NAMES = + new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; + private static final String TEST_KEY_NAMES = "a/b/c/d/e/file1.txt"; + + // Object names + private static final String VOL = "vol"; + private static final String VOL_TWO = "vol2"; + private static final String BUCKET_ONE = "bucket1"; + private static final String BUCKET_TWO = "bucket2"; + private static final String BUCKET_THREE = "bucket3"; + private static final String BUCKET_FOUR = "bucket4"; + private static final String KEY_ONE = "file1"; + private static final String KEY_TWO = "dir1/dir2/file2"; + private static final String KEY_THREE = "dir1/dir3/file3"; + private static final String KEY_FOUR = "file4"; + private static final String KEY_FIVE = "file5"; + private static final String KEY_SIX = "dir1/dir4/file6"; + private static final String KEY_SEVEN = "dir1/file7"; + private static final String KEY_EIGHT = "file8"; + private static final String KEY_NINE = "dir5/file9"; + private static final String KEY_TEN = "dir5/file10"; + private static final String KEY_ELEVEN = "file11"; + private static final String MULTI_BLOCK_KEY = "dir1/file7"; + private static final String MULTI_BLOCK_FILE = "file7"; + + private static final String FILE_ONE = "file1"; + private static final String FILE_TWO = "file2"; + private static final String FILE_THREE = "file3"; + private static final String FILE_FOUR = "file4"; + private static final String FILE_FIVE = "file5"; + private static final String FILE_SIX = "file6"; + private static final String FILE_SEVEN = "file7"; + private static final String FILE_EIGHT = "file8"; + private static final String FILE_NINE = "file9"; + private static final String FILE_TEN = "file10"; + private static final String FILE_ELEVEN = "file11"; + + private static final String DIR_ONE = "dir1"; + private static final String DIR_TWO = "dir2"; + private static final String DIR_THREE = "dir3"; + private static final String DIR_FOUR = "dir4"; + private static final String DIR_FIVE = "dir5"; + // objects IDs + private static final long PARENT_OBJECT_ID_ZERO = 0L; + private static final long VOL_OBJECT_ID = 0L; + private static final long BUCKET_ONE_OBJECT_ID = 1L; + private static final long BUCKET_TWO_OBJECT_ID = 2L; + private static final long KEY_ONE_OBJECT_ID = 3L; + private static final long DIR_ONE_OBJECT_ID = 4L; + private static final long KEY_TWO_OBJECT_ID = 5L; + private static final long KEY_FOUR_OBJECT_ID = 6L; + private static final long DIR_TWO_OBJECT_ID = 7L; + private static final long KEY_THREE_OBJECT_ID = 8L; + private static final long KEY_FIVE_OBJECT_ID = 9L; + private static final long KEY_SIX_OBJECT_ID = 10L; + private static final long DIR_THREE_OBJECT_ID = 11L; + private static final long DIR_FOUR_OBJECT_ID = 12L; + private static final long MULTI_BLOCK_KEY_OBJECT_ID = 13L; + private static final long KEY_SEVEN_OBJECT_ID = 13L; + private static final long VOL_TWO_OBJECT_ID = 14L; + private static final long BUCKET_THREE_OBJECT_ID = 15L; + private static final long BUCKET_FOUR_OBJECT_ID = 16L; + private static final long KEY_EIGHT_OBJECT_ID = 17L; + private static final long DIR_FIVE_OBJECT_ID = 18L; + private static final long KEY_NINE_OBJECT_ID = 19L; + private static final long KEY_TEN_OBJECT_ID = 20L; + private static final long KEY_ELEVEN_OBJECT_ID = 21L; + + // container IDs + private static final long CONTAINER_ONE_ID = 1L; + private static final long CONTAINER_TWO_ID = 2L; + private static final long CONTAINER_THREE_ID = 3L; + private static final long CONTAINER_FOUR_ID = 4L; + private static final long CONTAINER_FIVE_ID = 5L; + private static final long CONTAINER_SIX_ID = 6L; + + // replication factors + private static final int CONTAINER_ONE_REPLICA_COUNT = 3; + private static final int CONTAINER_TWO_REPLICA_COUNT = 2; + private static final int CONTAINER_THREE_REPLICA_COUNT = 4; + private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; + private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; + private static final int CONTAINER_SIX_REPLICA_COUNT = 3; + + // block lengths + private static final long BLOCK_ONE_LENGTH = 1000L; + private static final long BLOCK_TWO_LENGTH = 2000L; + private static final long BLOCK_THREE_LENGTH = 3000L; + private static final long BLOCK_FOUR_LENGTH = 4000L; + private static final long BLOCK_FIVE_LENGTH = 5000L; + private static final long BLOCK_SIX_LENGTH = 6000L; + + // data size in bytes + private static final long KEY_ONE_SIZE = 500L; // bin 0 + private static final long KEY_TWO_SIZE = OzoneConsts.KB + 1; // bin 1 + private static final long KEY_THREE_SIZE = 4 * OzoneConsts.KB + 1; // bin 3 + private static final long KEY_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + private static final long KEY_FIVE_SIZE = 100L; // bin 0 + private static final long KEY_SIX_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + private static final long KEY_EIGHT_SIZE = OzoneConsts.KB + 1; // bin 1 + private static final long KEY_NINE_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + private static final long KEY_TEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + private static final long KEY_ELEVEN_SIZE = OzoneConsts.KB + 1; // bin 1 + private static final long LOCATION_INFO_GROUP_ONE_SIZE + = CONTAINER_ONE_REPLICA_COUNT * BLOCK_ONE_LENGTH + + CONTAINER_TWO_REPLICA_COUNT * BLOCK_TWO_LENGTH + + CONTAINER_THREE_REPLICA_COUNT * BLOCK_THREE_LENGTH; + + private static final long MULTI_BLOCK_KEY_SIZE_WITH_REPLICA + = LOCATION_INFO_GROUP_ONE_SIZE; + + private static final long LOCATION_INFO_GROUP_TWO_SIZE + = CONTAINER_FOUR_REPLICA_COUNT * BLOCK_FOUR_LENGTH + + CONTAINER_FIVE_REPLICA_COUNT * BLOCK_FIVE_LENGTH + + CONTAINER_SIX_REPLICA_COUNT * BLOCK_SIX_LENGTH; + + private static final long FILE1_SIZE_WITH_REPLICA = + LOCATION_INFO_GROUP_ONE_SIZE; + private static final long FILE2_SIZE_WITH_REPLICA = + LOCATION_INFO_GROUP_TWO_SIZE; + private static final long FILE3_SIZE_WITH_REPLICA = + LOCATION_INFO_GROUP_ONE_SIZE; + private static final long FILE4_SIZE_WITH_REPLICA = + LOCATION_INFO_GROUP_TWO_SIZE; + private static final long FILE5_SIZE_WITH_REPLICA = + LOCATION_INFO_GROUP_ONE_SIZE; + private static final long FILE6_SIZE_WITH_REPLICA = + LOCATION_INFO_GROUP_TWO_SIZE; + private static final long FILE7_SIZE_WITH_REPLICA = + LOCATION_INFO_GROUP_ONE_SIZE; + private static final long FILE8_SIZE_WITH_REPLICA = + LOCATION_INFO_GROUP_TWO_SIZE; + private static final long FILE9_SIZE_WITH_REPLICA = + LOCATION_INFO_GROUP_ONE_SIZE; + private static final long FILE10_SIZE_WITH_REPLICA = + LOCATION_INFO_GROUP_TWO_SIZE; + private static final long FILE11_SIZE_WITH_REPLICA = + LOCATION_INFO_GROUP_ONE_SIZE; + + private static final long + MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT + = FILE1_SIZE_WITH_REPLICA + + FILE2_SIZE_WITH_REPLICA + + FILE3_SIZE_WITH_REPLICA + + FILE4_SIZE_WITH_REPLICA + + FILE5_SIZE_WITH_REPLICA + + FILE6_SIZE_WITH_REPLICA + + FILE7_SIZE_WITH_REPLICA + + FILE8_SIZE_WITH_REPLICA + + FILE9_SIZE_WITH_REPLICA + + FILE10_SIZE_WITH_REPLICA + + FILE11_SIZE_WITH_REPLICA; + + private static final long + MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL + = FILE1_SIZE_WITH_REPLICA + + FILE2_SIZE_WITH_REPLICA + + FILE3_SIZE_WITH_REPLICA + + FILE4_SIZE_WITH_REPLICA + + FILE5_SIZE_WITH_REPLICA + + FILE6_SIZE_WITH_REPLICA + + FILE7_SIZE_WITH_REPLICA; + + private static final long + MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1 + = FILE1_SIZE_WITH_REPLICA + + FILE2_SIZE_WITH_REPLICA + + FILE3_SIZE_WITH_REPLICA + + FILE6_SIZE_WITH_REPLICA + + FILE7_SIZE_WITH_REPLICA; + + private static final long + MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1 + = FILE2_SIZE_WITH_REPLICA + + FILE3_SIZE_WITH_REPLICA + + FILE6_SIZE_WITH_REPLICA + + FILE7_SIZE_WITH_REPLICA; + + private static final long + MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR2 + = FILE2_SIZE_WITH_REPLICA; + + private static final long + MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY + = FILE4_SIZE_WITH_REPLICA; + + // quota in bytes + private static final long ROOT_QUOTA = 2 * (2 * OzoneConsts.MB); + private static final long VOL_QUOTA = 2 * OzoneConsts.MB; + private static final long VOL_TWO_QUOTA = 2 * OzoneConsts.MB; + private static final long BUCKET_ONE_QUOTA = OzoneConsts.MB; + private static final long BUCKET_TWO_QUOTA = OzoneConsts.MB; + private static final long BUCKET_THREE_QUOTA = OzoneConsts.MB; + private static final long BUCKET_FOUR_QUOTA = OzoneConsts.MB; + + // mock client's path requests + private static final String TEST_USER = "TestUser"; + private static final String ROOT_PATH = "/"; + private static final String VOL_PATH = "/vol"; + private static final String VOL_TWO_PATH = "/vol2"; + private static final String BUCKET_ONE_PATH = "/vol/bucket1"; + private static final String BUCKET_TWO_PATH = "/vol/bucket2"; + private static final String DIR_ONE_PATH = "/vol/bucket1/dir1"; + private static final String DIR_TWO_PATH = "/vol/bucket1/dir1/dir2"; + private static final String DIR_THREE_PATH = "/vol/bucket1/dir1/dir3"; + private static final String DIR_FOUR_PATH = "/vol/bucket1/dir1/dir4"; + private static final String KEY_PATH = "/vol/bucket2/file4"; + private static final String MULTI_BLOCK_KEY_PATH = "/vol/bucket1/dir1/file7"; + private static final String INVALID_PATH = "/vol/path/not/found"; + + // some expected answers + private static final long ROOT_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + + KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + + KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE; + private static final long VOL_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + + KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE; + + private static final long VOL_TWO_DATA_SIZE = + KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE; + + private static final long BUCKET_ONE_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + + KEY_THREE_SIZE + KEY_SIX_SIZE; + + private static final long BUCKET_TWO_DATA_SIZE = + KEY_FOUR_SIZE + KEY_FIVE_SIZE; + + private static final long DIR_ONE_DATA_SIZE = KEY_TWO_SIZE + + KEY_THREE_SIZE + KEY_SIX_SIZE; + + @Before + public void setUp() throws Exception { + conf = new OzoneConfiguration(); + OMMetadataManager omMetadataManager = initializeNewOmMetadataManager( + temporaryFolder.newFolder(), conf); + OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = + getMockOzoneManagerServiceProvider(); + reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, + temporaryFolder.newFolder()); + + ReconTestInjector reconTestInjector = + new ReconTestInjector.Builder(temporaryFolder) + .withReconOm(reconOMMetadataManager) + .withOmServiceProvider(ozoneManagerServiceProvider) + .withReconSqlDb() + .withContainerDB() + .addBinding(OzoneStorageContainerManager.class, + getMockReconSCM()) + .addBinding(StorageContainerServiceProvider.class, + mock(StorageContainerServiceProviderImpl.class)) + .addBinding(NSSummaryEndpoint.class) + .build(); + ReconNamespaceSummaryManager reconNamespaceSummaryManager = + reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); + nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); + + // populate OM DB and reprocess into Recon RocksDB + populateOMDB(); + NSSummaryTaskWithLegacy nsSummaryTaskWithLegacy = + new NSSummaryTaskWithLegacy(reconNamespaceSummaryManager, + reconOMMetadataManager, conf); + nsSummaryTaskWithLegacy.reprocessWithLegacy(reconOMMetadataManager); + } + + @Test + public void testUtility() { + String[] names = EntityHandler.parseRequestPath(TEST_PATH_UTILITY); + Assert.assertArrayEquals(TEST_NAMES, names); + String keyName = BucketHandler.getKeyName(names); + Assert.assertEquals(TEST_KEY_NAMES, keyName); + String subpath = BucketHandler.buildSubpath(PARENT_DIR, "file1.txt"); + Assert.assertEquals(TEST_PATH_UTILITY, subpath); + } + + @Test + public void testGetBasicInfoRoot() throws Exception { + // Test root basics + Response rootResponse = nsSummaryEndpoint.getBasicInfo(ROOT_PATH); + NamespaceSummaryResponse rootResponseObj = + (NamespaceSummaryResponse) rootResponse.getEntity(); + Assert.assertEquals(EntityType.ROOT, rootResponseObj.getEntityType()); + Assert.assertEquals(2, rootResponseObj.getNumVolume()); + Assert.assertEquals(4, rootResponseObj.getNumBucket()); + Assert.assertEquals(5, rootResponseObj.getNumTotalDir()); + Assert.assertEquals(10, rootResponseObj.getNumTotalKey()); + } + + @Test + public void testGetBasicInfoVol() throws Exception { + // Test volume basics + Response volResponse = nsSummaryEndpoint.getBasicInfo(VOL_PATH); + NamespaceSummaryResponse volResponseObj = + (NamespaceSummaryResponse) volResponse.getEntity(); + Assert.assertEquals(EntityType.VOLUME, volResponseObj.getEntityType()); + Assert.assertEquals(2, volResponseObj.getNumBucket()); + Assert.assertEquals(4, volResponseObj.getNumTotalDir()); + Assert.assertEquals(6, volResponseObj.getNumTotalKey()); + } + + @Test + public void testGetBasicInfoBucketOne() throws Exception { + // Test bucket 1's basics + Response bucketOneResponse = + nsSummaryEndpoint.getBasicInfo(BUCKET_ONE_PATH); + NamespaceSummaryResponse bucketOneObj = + (NamespaceSummaryResponse) bucketOneResponse.getEntity(); + Assert.assertEquals(EntityType.BUCKET, bucketOneObj.getEntityType()); + Assert.assertEquals(4, bucketOneObj.getNumTotalDir()); + Assert.assertEquals(4, bucketOneObj.getNumTotalKey()); + } + + @Test + public void testGetBasicInfoBucketTwo() throws Exception { + // Test bucket 2's basics + Response bucketTwoResponse = + nsSummaryEndpoint.getBasicInfo(BUCKET_TWO_PATH); + NamespaceSummaryResponse bucketTwoObj = + (NamespaceSummaryResponse) bucketTwoResponse.getEntity(); + Assert.assertEquals(EntityType.BUCKET, bucketTwoObj.getEntityType()); + Assert.assertEquals(0, bucketTwoObj.getNumTotalDir()); + Assert.assertEquals(2, bucketTwoObj.getNumTotalKey()); + } + + @Test + public void testGetBasicInfoDir() throws Exception { + // Test intermediate directory basics + Response dirOneResponse = nsSummaryEndpoint.getBasicInfo(DIR_ONE_PATH); + NamespaceSummaryResponse dirOneObj = + (NamespaceSummaryResponse) dirOneResponse.getEntity(); + Assert.assertEquals(EntityType.DIRECTORY, dirOneObj.getEntityType()); + Assert.assertEquals(3, dirOneObj.getNumTotalDir()); + Assert.assertEquals(3, dirOneObj.getNumTotalKey()); + } + + @Test + public void testGetBasicInfoNoPath() throws Exception { + // Test invalid path + Response invalidResponse = nsSummaryEndpoint.getBasicInfo(INVALID_PATH); + NamespaceSummaryResponse invalidObj = + (NamespaceSummaryResponse) invalidResponse.getEntity(); + Assert.assertEquals(ResponseStatus.PATH_NOT_FOUND, + invalidObj.getStatus()); + } + + @Test + public void testGetBasicInfoKey() throws Exception { + // Test key + Response keyResponse = nsSummaryEndpoint.getBasicInfo(KEY_PATH); + NamespaceSummaryResponse keyResObj = + (NamespaceSummaryResponse) keyResponse.getEntity(); + Assert.assertEquals(EntityType.KEY, keyResObj.getEntityType()); + } + + @Test + public void testDiskUsageRoot() throws Exception { + // root level DU + Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, + false, false); + DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); + Assert.assertEquals(2, duRootRes.getCount()); + List duRootData = duRootRes.getDuData(); + // sort based on subpath + Collections.sort(duRootData, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + DUResponse.DiskUsage duVol1 = duRootData.get(0); + DUResponse.DiskUsage duVol2 = duRootData.get(1); + Assert.assertEquals(VOL_PATH, duVol1.getSubpath()); + Assert.assertEquals(VOL_TWO_PATH, duVol2.getSubpath()); + Assert.assertEquals(VOL_DATA_SIZE, duVol1.getSize()); + Assert.assertEquals(VOL_TWO_DATA_SIZE, duVol2.getSize()); + } + + @Test + public void testDiskUsageVolume() throws Exception { + // volume level DU + Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, + false, false); + DUResponse duVolRes = (DUResponse) volResponse.getEntity(); + Assert.assertEquals(2, duVolRes.getCount()); + List duData = duVolRes.getDuData(); + // sort based on subpath + Collections.sort(duData, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + DUResponse.DiskUsage duBucket1 = duData.get(0); + DUResponse.DiskUsage duBucket2 = duData.get(1); + Assert.assertEquals(BUCKET_ONE_PATH, duBucket1.getSubpath()); + Assert.assertEquals(BUCKET_TWO_PATH, duBucket2.getSubpath()); + Assert.assertEquals(BUCKET_ONE_DATA_SIZE, duBucket1.getSize()); + Assert.assertEquals(BUCKET_TWO_DATA_SIZE, duBucket2.getSize()); + } + + @Test + public void testDiskUsageBucket() throws Exception { + // bucket level DU + Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, + false, false); + DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); + Assert.assertEquals(1, duBucketResponse.getCount()); + DUResponse.DiskUsage duDir1 = duBucketResponse.getDuData().get(0); + Assert.assertEquals(DIR_ONE_PATH, duDir1.getSubpath()); + Assert.assertEquals(DIR_ONE_DATA_SIZE, duDir1.getSize()); + } + + @Test + public void testDiskUsageDir() throws Exception { + // dir level DU + Response dirResponse = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, + false, false); + DUResponse duDirReponse = (DUResponse) dirResponse.getEntity(); + Assert.assertEquals(3, duDirReponse.getCount()); + List duSubDir = duDirReponse.getDuData(); + Collections.sort(duSubDir, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + DUResponse.DiskUsage duDir2 = duSubDir.get(0); + DUResponse.DiskUsage duDir3 = duSubDir.get(1); + DUResponse.DiskUsage duDir4 = duSubDir.get(2); + Assert.assertEquals(DIR_TWO_PATH, duDir2.getSubpath()); + Assert.assertEquals(KEY_TWO_SIZE, duDir2.getSize()); + + Assert.assertEquals(DIR_THREE_PATH, duDir3.getSubpath()); + Assert.assertEquals(KEY_THREE_SIZE, duDir3.getSize()); + + Assert.assertEquals(DIR_FOUR_PATH, duDir4.getSubpath()); + Assert.assertEquals(KEY_SIX_SIZE, duDir4.getSize()); + } + + @Test + public void testDiskUsageKey() throws Exception { + // key level DU + Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, + false, false); + DUResponse keyObj = (DUResponse) keyResponse.getEntity(); + Assert.assertEquals(0, keyObj.getCount()); + Assert.assertEquals(KEY_FOUR_SIZE, keyObj.getSize()); + } + + @Test + public void testDiskUsageUnknown() throws Exception { + // invalid path check + Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, + false, false); + DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); + Assert.assertEquals(ResponseStatus.PATH_NOT_FOUND, + invalidObj.getStatus()); + } + + @Test + public void testDiskUsageWithReplication() throws Exception { + setUpMultiBlockKey(); + Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, + false, true); + DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); + Assert.assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); + Assert.assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, + replicaDUResponse.getSizeWithReplica()); + } + + @Test + public void testDataSizeUnderRootWithReplication() throws IOException { + setUpMultiBlockReplicatedKeys(); + // withReplica is true + Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, + false, true); + DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); + Assert.assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); + Assert.assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, + replicaDUResponse.getSizeWithReplica()); + Assert.assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, + replicaDUResponse.getDuData().get(0).getSizeWithReplica()); + + } + + @Test + public void testDataSizeUnderVolWithReplication() throws IOException { + setUpMultiBlockReplicatedKeys(); + Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, + false, true); + DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); + Assert.assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); + Assert.assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, + replicaDUResponse.getSizeWithReplica()); + Assert.assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, + replicaDUResponse.getDuData().get(0).getSizeWithReplica()); + } + + @Test + public void testDataSizeUnderBucketWithReplication() throws IOException { + setUpMultiBlockReplicatedKeys(); + Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, + false, true); + DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); + Assert.assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); + Assert.assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, + replicaDUResponse.getSizeWithReplica()); + Assert.assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, + replicaDUResponse.getDuData().get(0).getSizeWithReplica()); + } + + /** + * When calculating DU under dir1 + * there are 3 keys, file2, file3, file6. + * There is one direct key, file7. + * @throws IOException + */ + @Test + public void testDataSizeUnderDirWithReplication() throws IOException { + setUpMultiBlockReplicatedKeys(); + Response dir1Response = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, + false, true); + DUResponse replicaDUResponse = (DUResponse) dir1Response.getEntity(); + Assert.assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); + Assert.assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, + replicaDUResponse.getSizeWithReplica()); + Assert.assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR2, + replicaDUResponse.getDuData().get(0).getSizeWithReplica()); + } + + @Test + public void testDataSizeUnderKeyWithReplication() throws IOException { + setUpMultiBlockReplicatedKeys(); + Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, + false, true); + DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); + Assert.assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); + Assert.assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY, + replicaDUResponse.getSizeWithReplica()); + } + + @Test + public void testQuotaUsage() throws Exception { + // root level quota usage + Response rootResponse = nsSummaryEndpoint.getQuotaUsage(ROOT_PATH); + QuotaUsageResponse quRootRes = + (QuotaUsageResponse) rootResponse.getEntity(); + Assert.assertEquals(ROOT_QUOTA, quRootRes.getQuota()); + Assert.assertEquals(ROOT_DATA_SIZE, quRootRes.getQuotaUsed()); + + // volume level quota usage + Response volResponse = nsSummaryEndpoint.getQuotaUsage(VOL_PATH); + QuotaUsageResponse quVolRes = (QuotaUsageResponse) volResponse.getEntity(); + Assert.assertEquals(VOL_QUOTA, quVolRes.getQuota()); + Assert.assertEquals(VOL_DATA_SIZE, quVolRes.getQuotaUsed()); + + // bucket level quota usage + Response bucketRes = nsSummaryEndpoint.getQuotaUsage(BUCKET_ONE_PATH); + QuotaUsageResponse quBucketRes = (QuotaUsageResponse) bucketRes.getEntity(); + Assert.assertEquals(BUCKET_ONE_QUOTA, quBucketRes.getQuota()); + Assert.assertEquals(BUCKET_ONE_DATA_SIZE, quBucketRes.getQuotaUsed()); + + Response bucketRes2 = nsSummaryEndpoint.getQuotaUsage(BUCKET_TWO_PATH); + QuotaUsageResponse quBucketRes2 = + (QuotaUsageResponse) bucketRes2.getEntity(); + Assert.assertEquals(BUCKET_TWO_QUOTA, quBucketRes2.getQuota()); + Assert.assertEquals(BUCKET_TWO_DATA_SIZE, quBucketRes2.getQuotaUsed()); + + // other level not applicable + Response naResponse1 = nsSummaryEndpoint.getQuotaUsage(DIR_ONE_PATH); + QuotaUsageResponse quotaUsageResponse1 = + (QuotaUsageResponse) naResponse1.getEntity(); + Assert.assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, + quotaUsageResponse1.getResponseCode()); + + Response naResponse2 = nsSummaryEndpoint.getQuotaUsage(KEY_PATH); + QuotaUsageResponse quotaUsageResponse2 = + (QuotaUsageResponse) naResponse2.getEntity(); + Assert.assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, + quotaUsageResponse2.getResponseCode()); + + // invalid path request + Response invalidRes = nsSummaryEndpoint.getQuotaUsage(INVALID_PATH); + QuotaUsageResponse invalidResObj = + (QuotaUsageResponse) invalidRes.getEntity(); + Assert.assertEquals(ResponseStatus.PATH_NOT_FOUND, + invalidResObj.getResponseCode()); + } + + + @Test + public void testFileSizeDist() throws Exception { + checkFileSizeDist(ROOT_PATH, 2, 3, 4, 1); + checkFileSizeDist(VOL_PATH, 2, 1, 2, 1); + checkFileSizeDist(BUCKET_ONE_PATH, 1, 1, 1, 1); + checkFileSizeDist(DIR_ONE_PATH, 0, 1, 1, 1); + } + + public void checkFileSizeDist(String path, int bin0, + int bin1, int bin2, int bin3) throws Exception { + Response res = nsSummaryEndpoint.getFileSizeDistribution(path); + FileSizeDistributionResponse fileSizeDistResObj = + (FileSizeDistributionResponse) res.getEntity(); + int[] fileSizeDist = fileSizeDistResObj.getFileSizeDist(); + Assert.assertEquals(bin0, fileSizeDist[0]); + Assert.assertEquals(bin1, fileSizeDist[1]); + Assert.assertEquals(bin2, fileSizeDist[2]); + Assert.assertEquals(bin3, fileSizeDist[3]); + for (int i = 4; i < ReconConstants.NUM_OF_BINS; ++i) { + Assert.assertEquals(0, fileSizeDist[i]); + } + } + + /** + * Write directories and keys info into OM DB. + * @throws Exception + */ + @SuppressWarnings("checkstyle:MethodLength") + private void populateOMDB() throws Exception { + // write all directories + writeDirToOm(reconOMMetadataManager, + (DIR_ONE + OM_KEY_PREFIX), + BUCKET_ONE, + VOL, + DIR_ONE, + DIR_ONE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + getBucketLayout()); + writeDirToOm(reconOMMetadataManager, + (DIR_ONE + OM_KEY_PREFIX + DIR_TWO + OM_KEY_PREFIX), + BUCKET_ONE, + VOL, + DIR_TWO, + DIR_TWO_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + getBucketLayout()); + writeDirToOm(reconOMMetadataManager, + (DIR_ONE + OM_KEY_PREFIX + DIR_THREE + OM_KEY_PREFIX), + BUCKET_ONE, + VOL, + DIR_THREE, + DIR_THREE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + getBucketLayout()); + writeDirToOm(reconOMMetadataManager, + (DIR_ONE + OM_KEY_PREFIX + DIR_FOUR + OM_KEY_PREFIX), + BUCKET_ONE, + VOL, + DIR_FOUR, + DIR_FOUR_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + getBucketLayout()); + writeDirToOm(reconOMMetadataManager, + (DIR_FIVE + OM_KEY_PREFIX), + BUCKET_THREE, + VOL_TWO, + DIR_FIVE, + DIR_FIVE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, + getBucketLayout()); + + // write all keys + writeKeyToOm(reconOMMetadataManager, + KEY_ONE, + BUCKET_ONE, + VOL, + FILE_ONE, + KEY_ONE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + KEY_ONE_SIZE, + getBucketLayout()); + writeKeyToOm(reconOMMetadataManager, + KEY_TWO, + BUCKET_ONE, + VOL, + FILE_TWO, + KEY_TWO_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + KEY_TWO_SIZE, + getBucketLayout()); + writeKeyToOm(reconOMMetadataManager, + KEY_THREE, + BUCKET_ONE, + VOL, + FILE_THREE, + KEY_THREE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + KEY_THREE_SIZE, + getBucketLayout()); + writeKeyToOm(reconOMMetadataManager, + KEY_FOUR, + BUCKET_TWO, + VOL, + FILE_FOUR, + KEY_FOUR_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_TWO_OBJECT_ID, + VOL_OBJECT_ID, + KEY_FOUR_SIZE, + getBucketLayout()); + writeKeyToOm(reconOMMetadataManager, + KEY_FIVE, + BUCKET_TWO, + VOL, + FILE_FIVE, + KEY_FIVE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_TWO_OBJECT_ID, + VOL_OBJECT_ID, + KEY_FIVE_SIZE, + getBucketLayout()); + writeKeyToOm(reconOMMetadataManager, + KEY_SIX, + BUCKET_ONE, + VOL, + FILE_SIX, + KEY_SIX_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + KEY_SIX_SIZE, + getBucketLayout()); + writeKeyToOm(reconOMMetadataManager, + KEY_EIGHT, + BUCKET_THREE, + VOL_TWO, + FILE_EIGHT, + KEY_EIGHT_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, + KEY_EIGHT_SIZE, + getBucketLayout()); + writeKeyToOm(reconOMMetadataManager, + KEY_NINE, + BUCKET_THREE, + VOL_TWO, + FILE_NINE, + KEY_NINE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, + KEY_NINE_SIZE, + getBucketLayout()); + writeKeyToOm(reconOMMetadataManager, + KEY_TEN, + BUCKET_THREE, + VOL_TWO, + FILE_TEN, + KEY_TEN_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, + KEY_TEN_SIZE, + getBucketLayout()); + writeKeyToOm(reconOMMetadataManager, + KEY_ELEVEN, + BUCKET_FOUR, + VOL_TWO, + FILE_ELEVEN, + KEY_ELEVEN_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_FOUR_OBJECT_ID, + VOL_TWO_OBJECT_ID, + KEY_ELEVEN_SIZE, + getBucketLayout()); + } + + /** + * Create a new OM Metadata manager instance with one user, one vol, and two + * buckets. + * @throws IOException ioEx + */ + private static OMMetadataManager initializeNewOmMetadataManager( + File omDbDir, OzoneConfiguration omConfiguration) + throws IOException { + omConfiguration.set(OZONE_OM_DB_DIRS, + omDbDir.getAbsolutePath()); + omConfiguration.set(OMConfigKeys + .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); + OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( + omConfiguration); + + String volumeKey = omMetadataManager.getVolumeKey(VOL); + OmVolumeArgs args = + OmVolumeArgs.newBuilder() + .setObjectID(VOL_OBJECT_ID) + .setVolume(VOL) + .setAdminName(TEST_USER) + .setOwnerName(TEST_USER) + .setQuotaInBytes(VOL_QUOTA) + .build(); + + String volume2Key = omMetadataManager.getVolumeKey(VOL_TWO); + OmVolumeArgs args2 = + OmVolumeArgs.newBuilder() + .setObjectID(VOL_TWO_OBJECT_ID) + .setVolume(VOL_TWO) + .setAdminName(TEST_USER) + .setOwnerName(TEST_USER) + .setQuotaInBytes(VOL_TWO_QUOTA) + .build(); + + omMetadataManager.getVolumeTable().put(volumeKey, args); + omMetadataManager.getVolumeTable().put(volume2Key, args2); + + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() + .setVolumeName(VOL) + .setBucketName(BUCKET_ONE) + .setObjectID(BUCKET_ONE_OBJECT_ID) + .setQuotaInBytes(BUCKET_ONE_QUOTA) + .setBucketLayout(getBucketLayout()) + .build(); + + OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() + .setVolumeName(VOL) + .setBucketName(BUCKET_TWO) + .setObjectID(BUCKET_TWO_OBJECT_ID) + .setQuotaInBytes(BUCKET_TWO_QUOTA) + .setBucketLayout(getBucketLayout()) + .build(); + + OmBucketInfo bucketInfo3 = OmBucketInfo.newBuilder() + .setVolumeName(VOL_TWO) + .setBucketName(BUCKET_THREE) + .setObjectID(BUCKET_THREE_OBJECT_ID) + .setQuotaInBytes(BUCKET_THREE_QUOTA) + .setBucketLayout(getBucketLayout()) + .build(); + + OmBucketInfo bucketInfo4 = OmBucketInfo.newBuilder() + .setVolumeName(VOL_TWO) + .setBucketName(BUCKET_FOUR) + .setObjectID(BUCKET_FOUR_OBJECT_ID) + .setQuotaInBytes(BUCKET_FOUR_QUOTA) + .setBucketLayout(getBucketLayout()) + .build(); + + String bucketKey = omMetadataManager.getBucketKey( + bucketInfo.getVolumeName(), bucketInfo.getBucketName()); + String bucketKey2 = omMetadataManager.getBucketKey( + bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); + String bucketKey3 = omMetadataManager.getBucketKey( + bucketInfo3.getVolumeName(), bucketInfo3.getBucketName()); + String bucketKey4 = omMetadataManager.getBucketKey( + bucketInfo4.getVolumeName(), bucketInfo4.getBucketName()); + + omMetadataManager.getBucketTable().put(bucketKey, bucketInfo); + omMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); + omMetadataManager.getBucketTable().put(bucketKey3, bucketInfo3); + omMetadataManager.getBucketTable().put(bucketKey4, bucketInfo4); + + return omMetadataManager; + } + + private void setUpMultiBlockKey() throws IOException { + OmKeyLocationInfoGroup locationInfoGroup = + getLocationInfoGroup1(); + + // add the multi-block key to Recon's OM + writeKeyToOm(reconOMMetadataManager, + MULTI_BLOCK_KEY, + BUCKET_ONE, + VOL, + MULTI_BLOCK_FILE, + MULTI_BLOCK_KEY_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + Collections.singletonList(locationInfoGroup), + getBucketLayout()); + } + + private OmKeyLocationInfoGroup getLocationInfoGroup1() { + List locationInfoList = new ArrayList<>(); + BlockID block1 = new BlockID(CONTAINER_ONE_ID, 0L); + BlockID block2 = new BlockID(CONTAINER_TWO_ID, 0L); + BlockID block3 = new BlockID(CONTAINER_THREE_ID, 0L); + + OmKeyLocationInfo location1 = new OmKeyLocationInfo.Builder() + .setBlockID(block1) + .setLength(BLOCK_ONE_LENGTH) + .build(); + OmKeyLocationInfo location2 = new OmKeyLocationInfo.Builder() + .setBlockID(block2) + .setLength(BLOCK_TWO_LENGTH) + .build(); + OmKeyLocationInfo location3 = new OmKeyLocationInfo.Builder() + .setBlockID(block3) + .setLength(BLOCK_THREE_LENGTH) + .build(); + locationInfoList.add(location1); + locationInfoList.add(location2); + locationInfoList.add(location3); + + return new OmKeyLocationInfoGroup(0L, locationInfoList); + } + + /** + * Testing the following case. + * vol + * / \ + * bucket1 bucket2 + * / \ / \ + * file1 dir1 file4 file5 + * / \ \ \ + * dir2 dir3 dir4 file7 + * / \ \ + * file2 file3 file6 + * ---------------------------------------- + * vol2 + * / \ + * bucket3 bucket4 + * / \ / + * file8 dir5 file11 + * / \ + * file9 file10 + * Write these keys to OM and + * replicate them. + */ + private OmKeyLocationInfoGroup getLocationInfoGroup2() { + List locationInfoList = new ArrayList<>(); + BlockID block4 = new BlockID(CONTAINER_FOUR_ID, 0L); + BlockID block5 = new BlockID(CONTAINER_FIVE_ID, 0L); + BlockID block6 = new BlockID(CONTAINER_SIX_ID, 0L); + + OmKeyLocationInfo location4 = new OmKeyLocationInfo.Builder() + .setBlockID(block4) + .setLength(BLOCK_FOUR_LENGTH) + .build(); + OmKeyLocationInfo location5 = new OmKeyLocationInfo.Builder() + .setBlockID(block5) + .setLength(BLOCK_FIVE_LENGTH) + .build(); + OmKeyLocationInfo location6 = new OmKeyLocationInfo.Builder() + .setBlockID(block6) + .setLength(BLOCK_SIX_LENGTH) + .build(); + locationInfoList.add(location4); + locationInfoList.add(location5); + locationInfoList.add(location6); + return new OmKeyLocationInfoGroup(0L, locationInfoList); + + } + + @SuppressWarnings("checkstyle:MethodLength") + private void setUpMultiBlockReplicatedKeys() throws IOException { + OmKeyLocationInfoGroup locationInfoGroup1 = + getLocationInfoGroup1(); + OmKeyLocationInfoGroup locationInfoGroup2 = + getLocationInfoGroup2(); + + //vol/bucket1/file1 + writeKeyToOm(reconOMMetadataManager, + KEY_ONE, + BUCKET_ONE, + VOL, + FILE_ONE, + KEY_ONE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + Collections.singletonList(locationInfoGroup1), + getBucketLayout()); + + //vol/bucket1/dir1/dir2/file2 + writeKeyToOm(reconOMMetadataManager, + KEY_TWO, + BUCKET_ONE, + VOL, + FILE_TWO, + KEY_TWO_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + Collections.singletonList(locationInfoGroup2), + getBucketLayout()); + + //vol/bucket1/dir1/dir3/file3 + writeKeyToOm(reconOMMetadataManager, + KEY_THREE, + BUCKET_ONE, + VOL, + FILE_THREE, + KEY_THREE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + Collections.singletonList(locationInfoGroup1), + getBucketLayout()); + + //vol/bucket2/file4 + writeKeyToOm(reconOMMetadataManager, + KEY_FOUR, + BUCKET_TWO, + VOL, + FILE_FOUR, + KEY_FOUR_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_TWO_OBJECT_ID, + VOL_OBJECT_ID, + Collections.singletonList(locationInfoGroup2), + getBucketLayout()); + + //vol/bucket2/file5 + writeKeyToOm(reconOMMetadataManager, + KEY_FIVE, + BUCKET_TWO, + VOL, + FILE_FIVE, + KEY_FIVE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_TWO_OBJECT_ID, + VOL_OBJECT_ID, + Collections.singletonList(locationInfoGroup1), + getBucketLayout()); + + //vol/bucket1/dir1/dir4/file6 + writeKeyToOm(reconOMMetadataManager, + KEY_SIX, + BUCKET_ONE, + VOL, + FILE_SIX, + KEY_SIX_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + Collections.singletonList(locationInfoGroup2), + getBucketLayout()); + + //vol/bucket1/dir1/file7 + writeKeyToOm(reconOMMetadataManager, + KEY_SEVEN, + BUCKET_ONE, + VOL, + FILE_SEVEN, + KEY_SEVEN_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + Collections.singletonList(locationInfoGroup1), + getBucketLayout()); + + //vol2/bucket3/file8 + writeKeyToOm(reconOMMetadataManager, + KEY_EIGHT, + BUCKET_THREE, + VOL_TWO, + FILE_EIGHT, + KEY_EIGHT_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, + Collections.singletonList(locationInfoGroup2), + getBucketLayout()); + + //vol2/bucket3/dir5/file9 + writeKeyToOm(reconOMMetadataManager, + KEY_NINE, + BUCKET_THREE, + VOL_TWO, + FILE_NINE, + KEY_NINE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, + Collections.singletonList(locationInfoGroup1), + getBucketLayout()); + + //vol2/bucket3/dir5/file10 + writeKeyToOm(reconOMMetadataManager, + KEY_TEN, + BUCKET_THREE, + VOL_TWO, + FILE_TEN, + KEY_TEN_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, + Collections.singletonList(locationInfoGroup2), + getBucketLayout()); + + //vol2/bucket4/file11 + writeKeyToOm(reconOMMetadataManager, + KEY_ELEVEN, + BUCKET_FOUR, + VOL_TWO, + FILE_ELEVEN, + KEY_ELEVEN_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_FOUR_OBJECT_ID, + VOL_TWO_OBJECT_ID, + Collections.singletonList(locationInfoGroup1), + getBucketLayout()); + } + + /** + * Generate a set of mock container replica with a size of + * replication factor for container. + * @param replicationFactor number of replica + * @param containerID the container replicated based upon + * @return a set of container replica for testing + */ + private static Set generateMockContainerReplicas( + int replicationFactor, ContainerID containerID) { + Set result = new HashSet<>(); + for (int i = 0; i < replicationFactor; ++i) { + DatanodeDetails randomDatanode = randomDatanodeDetails(); + ContainerReplica replica = new ContainerReplica.ContainerReplicaBuilder() + .setContainerID(containerID) + .setContainerState(State.OPEN) + .setDatanodeDetails(randomDatanode) + .build(); + result.add(replica); + } + return result; + } + + private static ReconStorageContainerManagerFacade getMockReconSCM() + throws ContainerNotFoundException { + ReconStorageContainerManagerFacade reconSCM = + mock(ReconStorageContainerManagerFacade.class); + ContainerManager containerManager = mock(ContainerManager.class); + + // Container 1 is 3-way replicated + ContainerID containerID1 = new ContainerID(CONTAINER_ONE_ID); + Set containerReplicas1 = generateMockContainerReplicas( + CONTAINER_ONE_REPLICA_COUNT, containerID1); + when(containerManager.getContainerReplicas(containerID1)) + .thenReturn(containerReplicas1); + + // Container 2 is under replicated with 2 replica + ContainerID containerID2 = new ContainerID(CONTAINER_TWO_ID); + Set containerReplicas2 = generateMockContainerReplicas( + CONTAINER_TWO_REPLICA_COUNT, containerID2); + when(containerManager.getContainerReplicas(containerID2)) + .thenReturn(containerReplicas2); + + // Container 3 is over replicated with 4 replica + ContainerID containerID3 = new ContainerID(CONTAINER_THREE_ID); + Set containerReplicas3 = generateMockContainerReplicas( + CONTAINER_THREE_REPLICA_COUNT, containerID3); + when(containerManager.getContainerReplicas(containerID3)) + .thenReturn(containerReplicas3); + + // Container 4 is replicated with 5 replica + ContainerID containerID4 = new ContainerID(CONTAINER_FOUR_ID); + Set containerReplicas4 = generateMockContainerReplicas( + CONTAINER_FOUR_REPLICA_COUNT, containerID4); + when(containerManager.getContainerReplicas(containerID4)) + .thenReturn(containerReplicas4); + + // Container 5 is replicated with 2 replica + ContainerID containerID5 = new ContainerID(CONTAINER_FIVE_ID); + Set containerReplicas5 = generateMockContainerReplicas( + CONTAINER_FIVE_REPLICA_COUNT, containerID5); + when(containerManager.getContainerReplicas(containerID5)) + .thenReturn(containerReplicas5); + + // Container 6 is replicated with 3 replica + ContainerID containerID6 = new ContainerID(CONTAINER_SIX_ID); + Set containerReplicas6 = generateMockContainerReplicas( + CONTAINER_SIX_REPLICA_COUNT, containerID6); + when(containerManager.getContainerReplicas(containerID6)) + .thenReturn(containerReplicas6); + + when(reconSCM.getContainerManager()).thenReturn(containerManager); + return reconSCM; + } + + private static BucketLayout getBucketLayout() { + return BucketLayout.LEGACY; + } +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java new file mode 100644 index 000000000000..0c892bd3b3f5 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java @@ -0,0 +1,492 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.tasks; + +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.recon.ReconConstants; +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.ClassRule; +import org.junit.Assert; +import org.junit.experimental.runners.Enclosed; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Set; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; + +/** + * Test for NSSummaryTask. Create one bucket of each layout + * and test process and reprocess. Currently, there is no + * support for OBS buckets. Check that the NSSummary + * for the OBS bucket is null. + */ +@RunWith(Enclosed.class) +public final class TestNSSummaryTask { + + @ClassRule + public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder(); + + private static ReconNamespaceSummaryManager reconNamespaceSummaryManager; + private static OMMetadataManager omMetadataManager; + private static ReconOMMetadataManager reconOMMetadataManager; + private static NSSummaryTask nSSummaryTask; + private static OzoneConfiguration omConfiguration; + + // Object names + private static final String VOL = "vol"; + private static final String BUCKET_ONE = "bucket1"; + private static final String BUCKET_TWO = "bucket2"; + private static final String BUCKET_THREE = "bucket3"; + private static final String KEY_ONE = "file1"; + private static final String KEY_TWO = "file2"; + private static final String KEY_THREE = "file3"; + private static final String KEY_FIVE = "file5"; + private static final String FILE_ONE = "file1"; + private static final String FILE_TWO = "file2"; + private static final String FILE_THREE = "file3"; + private static final String FILE_FIVE = "file5"; + + private static final String TEST_USER = "TestUser"; + + private static final long PARENT_OBJECT_ID_ZERO = 0L; + private static final long VOL_OBJECT_ID = 0L; + private static final long BUCKET_ONE_OBJECT_ID = 1L; + private static final long BUCKET_TWO_OBJECT_ID = 2L; + private static final long BUCKET_THREE_OBJECT_ID = 4L; + private static final long KEY_ONE_OBJECT_ID = 3L; + private static final long KEY_TWO_OBJECT_ID = 5L; + private static final long KEY_THREE_OBJECT_ID = 8L; + private static final long KEY_FIVE_OBJECT_ID = 9L; + + private static final long KEY_ONE_SIZE = 500L; + private static final long KEY_TWO_SIZE = 1025L; + private static final long KEY_THREE_SIZE = + ReconConstants.MAX_FILE_SIZE_UPPER_BOUND - 100L; + private static final long KEY_FIVE_SIZE = 100L; + + private TestNSSummaryTask() { + } + + @BeforeClass + public static void setUp() throws Exception { + initializeNewOmMetadataManager(TEMPORARY_FOLDER.newFolder()); + OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = + getMockOzoneManagerServiceProvider(); + reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, + TEMPORARY_FOLDER.newFolder()); + + ReconTestInjector reconTestInjector = + new ReconTestInjector.Builder(TEMPORARY_FOLDER) + .withReconOm(reconOMMetadataManager) + .withOmServiceProvider(ozoneManagerServiceProvider) + .withReconSqlDb() + .withContainerDB() + .build(); + reconNamespaceSummaryManager = + reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); + + NSSummary nonExistentSummary = + reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); + Assert.assertNull(nonExistentSummary); + + populateOMDB(); + + nSSummaryTask = new NSSummaryTask(reconNamespaceSummaryManager, + reconOMMetadataManager, omConfiguration); + } + + /** + * Nested class for testing NSSummaryTaskWithLegacy reprocess. + */ + public static class TestReprocess { + + private static NSSummary nsSummaryForBucket1; + private static NSSummary nsSummaryForBucket2; + private static NSSummary nsSummaryForBucket3; + + @BeforeClass + public static void setUp() throws IOException { + // write a NSSummary prior to reprocess + // verify it got cleaned up after. + NSSummary staleNSSummary = new NSSummary(); + RDBBatchOperation rdbBatchOperation = new RDBBatchOperation(); + reconNamespaceSummaryManager.batchStoreNSSummaries(rdbBatchOperation, -1L, + staleNSSummary); + reconNamespaceSummaryManager.commitBatchOperation(rdbBatchOperation); + + // Verify commit + Assert.assertNotNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + + nSSummaryTask.reprocess(reconOMMetadataManager); + Assert.assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + + nsSummaryForBucket1 = + reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); + nsSummaryForBucket2 = + reconNamespaceSummaryManager.getNSSummary(BUCKET_TWO_OBJECT_ID); + nsSummaryForBucket3 = + reconNamespaceSummaryManager.getNSSummary(BUCKET_THREE_OBJECT_ID); + Assert.assertNotNull(nsSummaryForBucket1); + Assert.assertNotNull(nsSummaryForBucket2); + Assert.assertNull(nsSummaryForBucket3); + } + + @Test + public void testReprocessNSSummaryNull() throws IOException { + Assert.assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + } + + @Test + public void testReprocessGetFiles() { + Assert.assertEquals(1, nsSummaryForBucket1.getNumOfFiles()); + Assert.assertEquals(1, nsSummaryForBucket2.getNumOfFiles()); + + Assert.assertEquals(KEY_ONE_SIZE, nsSummaryForBucket1.getSizeOfFiles()); + Assert.assertEquals(KEY_TWO_SIZE, nsSummaryForBucket2.getSizeOfFiles()); + } + + @Test + public void testReprocessFileBucketSize() { + int[] fileDistBucket1 = nsSummaryForBucket1.getFileSizeBucket(); + int[] fileDistBucket2 = nsSummaryForBucket2.getFileSizeBucket(); + Assert.assertEquals(ReconConstants.NUM_OF_BINS, fileDistBucket1.length); + Assert.assertEquals(ReconConstants.NUM_OF_BINS, fileDistBucket2.length); + + Assert.assertEquals(1, fileDistBucket1[0]); + for (int i = 1; i < ReconConstants.NUM_OF_BINS; ++i) { + Assert.assertEquals(0, fileDistBucket1[i]); + } + Assert.assertEquals(1, fileDistBucket2[1]); + for (int i = 0; i < ReconConstants.NUM_OF_BINS; ++i) { + if (i == 1) { + continue; + } + Assert.assertEquals(0, fileDistBucket2[i]); + } + } + + } + + /** + * Nested class for testing NSSummaryTaskWithLegacy process. + */ + public static class TestProcess { + + private static NSSummary nsSummaryForBucket1; + private static NSSummary nsSummaryForBucket2; + private static NSSummary nsSummaryForBucket3; + + private static OMDBUpdateEvent keyEvent1; + private static OMDBUpdateEvent keyEvent2; + + @BeforeClass + public static void setUp() throws IOException { + nSSummaryTask.reprocess(reconOMMetadataManager); + nSSummaryTask.process(processEventBatch()); + + nsSummaryForBucket1 = + reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); + Assert.assertNotNull(nsSummaryForBucket1); + nsSummaryForBucket2 = + reconNamespaceSummaryManager.getNSSummary(BUCKET_TWO_OBJECT_ID); + Assert.assertNotNull(nsSummaryForBucket2); + nsSummaryForBucket3 = + reconNamespaceSummaryManager.getNSSummary(BUCKET_THREE_OBJECT_ID); + Assert.assertNull(nsSummaryForBucket3); + } + + private static OMUpdateEventBatch processEventBatch() throws IOException { + // put file5 under bucket 2 + String omPutKey = + OM_KEY_PREFIX + VOL + + OM_KEY_PREFIX + BUCKET_TWO + + OM_KEY_PREFIX + FILE_FIVE; + OmKeyInfo omPutKeyInfo = buildOmKeyInfo(VOL, BUCKET_TWO, KEY_FIVE, + FILE_FIVE, KEY_FIVE_OBJECT_ID, BUCKET_TWO_OBJECT_ID, KEY_FIVE_SIZE); + keyEvent1 = new OMDBUpdateEvent. + OMUpdateEventBuilder() + .setKey(omPutKey) + .setValue(omPutKeyInfo) + .setTable(omMetadataManager.getKeyTable(getLegacyBucketLayout()) + .getName()) + .setAction(OMDBUpdateEvent.OMDBUpdateAction.PUT) + .build(); + + // delete file 1 under bucket 1 + String omDeleteKey = BUCKET_ONE_OBJECT_ID + OM_KEY_PREFIX + FILE_ONE; + OmKeyInfo omDeleteInfo = buildOmKeyInfo( + VOL, BUCKET_ONE, KEY_ONE, FILE_ONE, + KEY_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID); + keyEvent2 = new OMDBUpdateEvent. + OMUpdateEventBuilder() + .setKey(omDeleteKey) + .setValue(omDeleteInfo) + .setTable(omMetadataManager.getKeyTable(getFSOBucketLayout()) + .getName()) + .setAction(OMDBUpdateEvent.OMDBUpdateAction.DELETE) + .build(); + + OMUpdateEventBatch omUpdateEventBatch = new OMUpdateEventBatch( + new ArrayList() {{ + add(keyEvent1); + add(keyEvent2); + }}); + + return omUpdateEventBatch; + } + + @Test + public void testProcessUpdateFileSize() throws IOException { + // file 1 is gone, so bucket 1 is empty now + Assert.assertNotNull(nsSummaryForBucket1); + Assert.assertEquals(0, nsSummaryForBucket1.getNumOfFiles()); + + Set childDirBucket1 = nsSummaryForBucket1.getChildDir(); + Assert.assertEquals(0, childDirBucket1.size()); + } + + @Test + public void testProcessBucket() throws IOException { + // file 5 is added under bucket 2, so bucket 2 has 2 keys now + Assert.assertNotNull(nsSummaryForBucket2); + Assert.assertEquals(2, nsSummaryForBucket2.getNumOfFiles()); + // key 2 + key 5 + Assert.assertEquals(KEY_TWO_SIZE + KEY_FIVE_SIZE, + nsSummaryForBucket2.getSizeOfFiles()); + + int[] fileSizeDist = nsSummaryForBucket2.getFileSizeBucket(); + Assert.assertEquals(ReconConstants.NUM_OF_BINS, fileSizeDist.length); + // 1025L + Assert.assertEquals(1, fileSizeDist[0]); + // 2050L + Assert.assertEquals(1, fileSizeDist[1]); + for (int i = 2; i < ReconConstants.NUM_OF_BINS; ++i) { + Assert.assertEquals(0, fileSizeDist[i]); + } + } + } + + /** + * Build a key info for put/update action. + * @param volume volume name + * @param bucket bucket name + * @param key key name + * @param fileName file name + * @param objectID object ID + * @param parentObjectId parent object ID + * @param dataSize file size + * @return the KeyInfo + */ + private static OmKeyInfo buildOmKeyInfo(String volume, + String bucket, + String key, + String fileName, + long objectID, + long parentObjectId, + long dataSize) { + return new OmKeyInfo.Builder() + .setBucketName(bucket) + .setVolumeName(volume) + .setKeyName(key) + .setFileName(fileName) + .setReplicationConfig( + StandaloneReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.ONE)) + .setObjectID(objectID) + .setParentObjectID(parentObjectId) + .setDataSize(dataSize) + .build(); + } + + /** + * Build a key info for delete action. + * @param volume volume name + * @param bucket bucket name + * @param key key name + * @param fileName file name + * @param objectID object ID + * @param parentObjectId parent object ID + * @return the KeyInfo + */ + private static OmKeyInfo buildOmKeyInfo(String volume, + String bucket, + String key, + String fileName, + long objectID, + long parentObjectId) { + return new OmKeyInfo.Builder() + .setBucketName(bucket) + .setVolumeName(volume) + .setKeyName(key) + .setFileName(fileName) + .setReplicationConfig( + StandaloneReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.ONE)) + .setObjectID(objectID) + .setParentObjectID(parentObjectId) + .build(); + } + + /** + * Populate OMDB with the following configs. + * vol + * / \ \ + * bucket1 bucket2 bucket3 + * / / / + * file1 file2 file3 + * + * @throws IOException + */ + private static void populateOMDB() throws IOException { + // Bucket1 FSO layout + writeKeyToOm(reconOMMetadataManager, + KEY_ONE, + BUCKET_ONE, + VOL, + FILE_ONE, + KEY_ONE_OBJECT_ID, + BUCKET_ONE_OBJECT_ID, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + KEY_ONE_SIZE, + getFSOBucketLayout()); + + // Bucket2 Legacy layout + writeKeyToOm(reconOMMetadataManager, + KEY_TWO, + BUCKET_TWO, + VOL, + FILE_TWO, + KEY_TWO_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_TWO_OBJECT_ID, + VOL_OBJECT_ID, + KEY_TWO_SIZE, + getLegacyBucketLayout()); + + // Bucket3 OBS layout + writeKeyToOm(reconOMMetadataManager, + KEY_THREE, + BUCKET_THREE, + VOL, + FILE_THREE, + KEY_THREE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_THREE_OBJECT_ID, + VOL_OBJECT_ID, + KEY_THREE_SIZE, + getOBSBucketLayout()); + } + + /** + * Create a new OM Metadata manager instance with one user, one vol, and two + * buckets. Bucket1 will have FSO layout, bucket2 will have Legacy layout + * and bucket3 will have OBS layout. + * @throws IOException ioEx + */ + private static void initializeNewOmMetadataManager( + File omDbDir) + throws IOException { + omConfiguration = new OzoneConfiguration(); + omConfiguration.set(OZONE_OM_DB_DIRS, + omDbDir.getAbsolutePath()); + omConfiguration.set(OMConfigKeys + .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); + omMetadataManager = new OmMetadataManagerImpl( + omConfiguration); + + String volumeKey = omMetadataManager.getVolumeKey(VOL); + OmVolumeArgs args = + OmVolumeArgs.newBuilder() + .setObjectID(VOL_OBJECT_ID) + .setVolume(VOL) + .setAdminName(TEST_USER) + .setOwnerName(TEST_USER) + .build(); + omMetadataManager.getVolumeTable().put(volumeKey, args); + + OmBucketInfo bucketInfo1 = OmBucketInfo.newBuilder() + .setVolumeName(VOL) + .setBucketName(BUCKET_ONE) + .setObjectID(BUCKET_ONE_OBJECT_ID) + .setBucketLayout(getFSOBucketLayout()) + .build(); + + OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() + .setVolumeName(VOL) + .setBucketName(BUCKET_TWO) + .setObjectID(BUCKET_TWO_OBJECT_ID) + .setBucketLayout(getLegacyBucketLayout()) + .build(); + + OmBucketInfo bucketInfo3 = OmBucketInfo.newBuilder() + .setVolumeName(VOL) + .setBucketName(BUCKET_THREE) + .setObjectID(BUCKET_THREE_OBJECT_ID) + .setBucketLayout(getOBSBucketLayout()) + .build(); + + String bucketKey = omMetadataManager.getBucketKey( + bucketInfo1.getVolumeName(), bucketInfo1.getBucketName()); + String bucketKey2 = omMetadataManager.getBucketKey( + bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); + String bucketKey3 = omMetadataManager.getBucketKey( + bucketInfo3.getVolumeName(), bucketInfo3.getBucketName()); + + omMetadataManager.getBucketTable().put(bucketKey, bucketInfo1); + omMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); + omMetadataManager.getBucketTable().put(bucketKey3, bucketInfo3); + } + + private static BucketLayout getFSOBucketLayout() { + return BucketLayout.FILE_SYSTEM_OPTIMIZED; + } + + private static BucketLayout getLegacyBucketLayout() { + return BucketLayout.LEGACY; + } + + private static BucketLayout getOBSBucketLayout() { + return BucketLayout.OBJECT_STORE; + } +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java index 76a05b5553d9..6b6b831c0664 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java @@ -134,7 +134,6 @@ public static void setUp() throws Exception { reconNamespaceSummaryManager = reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); - NSSummary nonExistentSummary = reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); Assert.assertNull(nonExistentSummary); @@ -142,7 +141,7 @@ public static void setUp() throws Exception { populateOMDB(); nSSummaryTaskWithFso = new NSSummaryTaskWithFSO( - reconNamespaceSummaryManager); + reconNamespaceSummaryManager, reconOMMetadataManager); } /** @@ -165,7 +164,11 @@ public static void setUp() throws IOException { // Verify commit Assert.assertNotNull(reconNamespaceSummaryManager.getNSSummary(-1L)); - nSSummaryTaskWithFso.reprocess(reconOMMetadataManager); + + // reinit Recon RocksDB's namespace CF. + reconNamespaceSummaryManager.clearNSSummaryTable(); + + nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); Assert.assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); nsSummaryForBucket1 = @@ -273,8 +276,8 @@ public static class TestProcess { private static OMDBUpdateEvent keyEvent7; @BeforeClass public static void setUp() throws IOException { - nSSummaryTaskWithFso.reprocess(reconOMMetadataManager); - nSSummaryTaskWithFso.process(processEventBatch()); + nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); + nSSummaryTaskWithFso.processWithFSO(processEventBatch()); } private static OMUpdateEventBatch processEventBatch() throws IOException { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java new file mode 100644 index 000000000000..332d88238a8c --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java @@ -0,0 +1,740 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.tasks; + +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.recon.ReconConstants; +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.ClassRule; +import org.junit.Assert; +import org.junit.experimental.runners.Enclosed; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Set; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; + +/** + * Test for NSSummaryTaskWithLegacy. + */ +@RunWith(Enclosed.class) +public final class TestNSSummaryTaskWithLegacy { + + @ClassRule + public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder(); + + private static ReconNamespaceSummaryManager reconNamespaceSummaryManager; + private static OMMetadataManager omMetadataManager; + private static ReconOMMetadataManager reconOMMetadataManager; + private static NSSummaryTaskWithLegacy nSSummaryTaskWithLegacy; + private static OzoneConfiguration omConfiguration; + + // Object names + private static final String VOL = "vol"; + private static final String BUCKET_ONE = "bucket1"; + private static final String BUCKET_TWO = "bucket2"; + private static final String KEY_ONE = "file1"; + private static final String KEY_TWO = "file2"; + private static final String KEY_THREE = "dir1/dir2/file3"; + private static final String KEY_FOUR = "file4"; + private static final String KEY_FIVE = "file5"; + private static final String FILE_ONE = "file1"; + private static final String FILE_TWO = "file2"; + private static final String FILE_THREE = "file3"; + private static final String FILE_FOUR = "file4"; + private static final String FILE_FIVE = "file5"; + private static final String DIR_ONE = "dir1"; + private static final String DIR_ONE_RENAME = "dir1_new"; + private static final String DIR_TWO = "dir2"; + private static final String DIR_THREE = "dir3"; + private static final String DIR_FOUR = "dir4"; + private static final String DIR_FIVE = "dir5"; + + private static final String TEST_USER = "TestUser"; + + private static final long PARENT_OBJECT_ID_ZERO = 0L; + private static final long VOL_OBJECT_ID = 0L; + private static final long BUCKET_ONE_OBJECT_ID = 1L; + private static final long BUCKET_TWO_OBJECT_ID = 2L; + private static final long KEY_ONE_OBJECT_ID = 3L; + private static final long DIR_ONE_OBJECT_ID = 4L; + private static final long KEY_TWO_OBJECT_ID = 5L; + private static final long KEY_FOUR_OBJECT_ID = 6L; + private static final long DIR_TWO_OBJECT_ID = 7L; + private static final long KEY_THREE_OBJECT_ID = 8L; + private static final long KEY_FIVE_OBJECT_ID = 9L; + private static final long DIR_THREE_OBJECT_ID = 10L; + private static final long DIR_FOUR_OBJECT_ID = 11L; + private static final long DIR_FIVE_OBJECT_ID = 12L; + + private static final long KEY_ONE_SIZE = 500L; + private static final long KEY_TWO_OLD_SIZE = 1025L; + private static final long KEY_TWO_UPDATE_SIZE = 1023L; + private static final long KEY_THREE_SIZE = + ReconConstants.MAX_FILE_SIZE_UPPER_BOUND - 100L; + private static final long KEY_FOUR_SIZE = 2050L; + private static final long KEY_FIVE_SIZE = 100L; + + private static Set bucketOneAns = new HashSet<>(); + private static Set bucketTwoAns = new HashSet<>(); + private static Set dirOneAns = new HashSet<>(); + + private TestNSSummaryTaskWithLegacy() { + } + + @BeforeClass + public static void setUp() throws Exception { + initializeNewOmMetadataManager(TEMPORARY_FOLDER.newFolder()); + OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = + getMockOzoneManagerServiceProvider(); + reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, + TEMPORARY_FOLDER.newFolder()); + + ReconTestInjector reconTestInjector = + new ReconTestInjector.Builder(TEMPORARY_FOLDER) + .withReconOm(reconOMMetadataManager) + .withOmServiceProvider(ozoneManagerServiceProvider) + .withReconSqlDb() + .withContainerDB() + .build(); + reconNamespaceSummaryManager = + reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); + + NSSummary nonExistentSummary = + reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); + Assert.assertNull(nonExistentSummary); + + populateOMDB(); + + nSSummaryTaskWithLegacy = new NSSummaryTaskWithLegacy( + reconNamespaceSummaryManager, + reconOMMetadataManager, omConfiguration); + } + + /** + * Nested class for testing NSSummaryTaskWithLegacy reprocess. + */ + public static class TestReprocess { + + private static NSSummary nsSummaryForBucket1; + private static NSSummary nsSummaryForBucket2; + + @BeforeClass + public static void setUp() throws IOException { + // write a NSSummary prior to reprocess + // verify it got cleaned up after. + NSSummary staleNSSummary = new NSSummary(); + RDBBatchOperation rdbBatchOperation = new RDBBatchOperation(); + reconNamespaceSummaryManager.batchStoreNSSummaries(rdbBatchOperation, -1L, + staleNSSummary); + reconNamespaceSummaryManager.commitBatchOperation(rdbBatchOperation); + + // Verify commit + Assert.assertNotNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + + // reinit Recon RocksDB's namespace CF. + reconNamespaceSummaryManager.clearNSSummaryTable(); + + nSSummaryTaskWithLegacy.reprocessWithLegacy(reconOMMetadataManager); + Assert.assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + + nsSummaryForBucket1 = + reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); + nsSummaryForBucket2 = + reconNamespaceSummaryManager.getNSSummary(BUCKET_TWO_OBJECT_ID); + Assert.assertNotNull(nsSummaryForBucket1); + Assert.assertNotNull(nsSummaryForBucket2); + } + + @Test + public void testReprocessNSSummaryNull() throws IOException { + Assert.assertNull(reconNamespaceSummaryManager.getNSSummary(-1L)); + } + + @Test + public void testReprocessGetFiles() { + Assert.assertEquals(1, nsSummaryForBucket1.getNumOfFiles()); + Assert.assertEquals(2, nsSummaryForBucket2.getNumOfFiles()); + + Assert.assertEquals(KEY_ONE_SIZE, nsSummaryForBucket1.getSizeOfFiles()); + Assert.assertEquals(KEY_TWO_OLD_SIZE + KEY_FOUR_SIZE, + nsSummaryForBucket2.getSizeOfFiles()); + } + + @Test + public void testReprocessFileBucketSize() { + int[] fileDistBucket1 = nsSummaryForBucket1.getFileSizeBucket(); + int[] fileDistBucket2 = nsSummaryForBucket2.getFileSizeBucket(); + Assert.assertEquals(ReconConstants.NUM_OF_BINS, fileDistBucket1.length); + Assert.assertEquals(ReconConstants.NUM_OF_BINS, fileDistBucket2.length); + + Assert.assertEquals(1, fileDistBucket1[0]); + for (int i = 1; i < ReconConstants.NUM_OF_BINS; ++i) { + Assert.assertEquals(0, fileDistBucket1[i]); + } + Assert.assertEquals(1, fileDistBucket2[1]); + Assert.assertEquals(1, fileDistBucket2[2]); + for (int i = 0; i < ReconConstants.NUM_OF_BINS; ++i) { + if (i == 1 || i == 2) { + continue; + } + Assert.assertEquals(0, fileDistBucket2[i]); + } + } + + @Test + public void testReprocessBucketDirs() { + // Bucket one has one dir, bucket two has none. + Set childDirBucketOne = nsSummaryForBucket1.getChildDir(); + Set childDirBucketTwo = nsSummaryForBucket2.getChildDir(); + Assert.assertEquals(1, childDirBucketOne.size()); + bucketOneAns.clear(); + bucketOneAns.add(DIR_ONE_OBJECT_ID); + Assert.assertEquals(bucketOneAns, childDirBucketOne); + Assert.assertEquals(0, childDirBucketTwo.size()); + } + + @Test + public void testReprocessDirsUnderDir() throws Exception { + + // Dir 1 has two dir: dir2 and dir3. + NSSummary nsSummaryInDir1 = reconNamespaceSummaryManager + .getNSSummary(DIR_ONE_OBJECT_ID); + Assert.assertNotNull(nsSummaryInDir1); + Set childDirForDirOne = nsSummaryInDir1.getChildDir(); + Assert.assertEquals(2, childDirForDirOne.size()); + dirOneAns.clear(); + dirOneAns.add(DIR_TWO_OBJECT_ID); + dirOneAns.add(DIR_THREE_OBJECT_ID); + Assert.assertEquals(dirOneAns, childDirForDirOne); + + NSSummary nsSummaryInDir2 = reconNamespaceSummaryManager + .getNSSummary(DIR_TWO_OBJECT_ID); + Assert.assertEquals(1, nsSummaryInDir2.getNumOfFiles()); + Assert.assertEquals(KEY_THREE_SIZE, nsSummaryInDir2.getSizeOfFiles()); + + int[] fileDistForDir2 = nsSummaryInDir2.getFileSizeBucket(); + Assert.assertEquals(ReconConstants.NUM_OF_BINS, fileDistForDir2.length); + Assert.assertEquals(1, fileDistForDir2[fileDistForDir2.length - 1]); + for (int i = 0; i < ReconConstants.NUM_OF_BINS - 1; ++i) { + Assert.assertEquals(0, fileDistForDir2[i]); + } + Assert.assertEquals(0, nsSummaryInDir2.getChildDir().size()); + + // bucket should have empty dirName + Assert.assertEquals(0, nsSummaryForBucket1.getDirName().length()); + Assert.assertEquals(0, nsSummaryForBucket2.getDirName().length()); + // check dirName is correctly written + Assert.assertEquals(DIR_ONE, nsSummaryInDir1.getDirName()); + Assert.assertEquals(DIR_ONE + OM_KEY_PREFIX + DIR_TWO, + nsSummaryInDir2.getDirName()); + } + } + + /** + * Nested class for testing NSSummaryTaskWithLegacy process. + */ + public static class TestProcess { + + private static NSSummary nsSummaryForBucket1; + private static NSSummary nsSummaryForBucket2; + + private static OMDBUpdateEvent keyEvent1; + private static OMDBUpdateEvent keyEvent2; + private static OMDBUpdateEvent keyEvent3; + private static OMDBUpdateEvent keyEvent4; + private static OMDBUpdateEvent keyEvent5; + private static OMDBUpdateEvent keyEvent6; + private static OMDBUpdateEvent keyEvent7; + + @BeforeClass + public static void setUp() throws IOException { + nSSummaryTaskWithLegacy.reprocessWithLegacy(reconOMMetadataManager); + nSSummaryTaskWithLegacy.processWithLegacy(processEventBatch()); + + nsSummaryForBucket1 = + reconNamespaceSummaryManager.getNSSummary(BUCKET_ONE_OBJECT_ID); + Assert.assertNotNull(nsSummaryForBucket1); + nsSummaryForBucket2 = + reconNamespaceSummaryManager.getNSSummary(BUCKET_TWO_OBJECT_ID); + Assert.assertNotNull(nsSummaryForBucket2); + } + + private static OMUpdateEventBatch processEventBatch() throws IOException { + // put file5 under bucket 2 + String omPutKey = + OM_KEY_PREFIX + VOL + + OM_KEY_PREFIX + BUCKET_TWO + + OM_KEY_PREFIX + FILE_FIVE; + OmKeyInfo omPutKeyInfo = buildOmKeyInfo(VOL, BUCKET_TWO, KEY_FIVE, + FILE_FIVE, KEY_FIVE_OBJECT_ID, BUCKET_TWO_OBJECT_ID, KEY_FIVE_SIZE); + keyEvent1 = new OMDBUpdateEvent. + OMUpdateEventBuilder() + .setKey(omPutKey) + .setValue(omPutKeyInfo) + .setTable(omMetadataManager.getKeyTable(getBucketLayout()) + .getName()) + .setAction(OMDBUpdateEvent.OMDBUpdateAction.PUT) + .build(); + + // delete file 1 under bucket 1 + String omDeleteKey = + OM_KEY_PREFIX + VOL + + OM_KEY_PREFIX + BUCKET_ONE + + OM_KEY_PREFIX + FILE_ONE; + OmKeyInfo omDeleteInfo = buildOmKeyInfo( + VOL, BUCKET_ONE, KEY_ONE, + FILE_ONE, KEY_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID); + keyEvent2 = new OMDBUpdateEvent. + OMUpdateEventBuilder() + .setKey(omDeleteKey) + .setValue(omDeleteInfo) + .setTable(omMetadataManager.getKeyTable(getBucketLayout()) + .getName()) + .setAction(OMDBUpdateEvent.OMDBUpdateAction.DELETE) + .build(); + + // update file 2's size under bucket 2 + String omUpdateKey = + OM_KEY_PREFIX + VOL + + OM_KEY_PREFIX + BUCKET_TWO + + OM_KEY_PREFIX + FILE_TWO; + OmKeyInfo omOldInfo = buildOmKeyInfo( + VOL, BUCKET_TWO, KEY_TWO, FILE_TWO, + KEY_TWO_OBJECT_ID, BUCKET_TWO_OBJECT_ID, KEY_TWO_OLD_SIZE); + OmKeyInfo omUpdateInfo = buildOmKeyInfo( + VOL, BUCKET_TWO, KEY_TWO, FILE_TWO, + KEY_TWO_OBJECT_ID, BUCKET_TWO_OBJECT_ID, KEY_TWO_UPDATE_SIZE); + keyEvent3 = new OMDBUpdateEvent. + OMUpdateEventBuilder() + .setKey(omUpdateKey) + .setValue(omUpdateInfo) + .setOldValue(omOldInfo) + .setTable(omMetadataManager.getKeyTable(getBucketLayout()) + .getName()) + .setAction(OMDBUpdateEvent.OMDBUpdateAction.UPDATE) + .build(); + + // add dir 4 under bucket 1 + String omDirPutKey1 = + OM_KEY_PREFIX + VOL + + OM_KEY_PREFIX + BUCKET_ONE + + OM_KEY_PREFIX + DIR_FOUR + OM_KEY_PREFIX; + OmKeyInfo omDirPutValue1 = buildOmDirKeyInfo(VOL, BUCKET_ONE, + (DIR_FOUR + OM_KEY_PREFIX), DIR_FOUR, + DIR_FOUR_OBJECT_ID); + keyEvent4 = new OMDBUpdateEvent. + OMUpdateEventBuilder() + .setKey(omDirPutKey1) + .setValue(omDirPutValue1) + .setAction(OMDBUpdateEvent.OMDBUpdateAction.PUT) + .setTable(omMetadataManager.getKeyTable(getBucketLayout()).getName()) + .build(); + + // add dir 5 under bucket 2 + String omDirPutKey2 = + OM_KEY_PREFIX + VOL + + OM_KEY_PREFIX + BUCKET_TWO + + OM_KEY_PREFIX + DIR_FIVE + OM_KEY_PREFIX; + OmKeyInfo omDirPutValue2 = buildOmDirKeyInfo(VOL, BUCKET_TWO, + (DIR_FIVE + OM_KEY_PREFIX), DIR_FIVE, + DIR_FIVE_OBJECT_ID); + keyEvent5 = new OMDBUpdateEvent. + OMUpdateEventBuilder() + .setKey(omDirPutKey2) + .setValue(omDirPutValue2) + .setAction(OMDBUpdateEvent.OMDBUpdateAction.PUT) + .setTable(omMetadataManager.getKeyTable(getBucketLayout()).getName()) + .build(); + + // delete dir 3 under dir 1 + String omDirDeleteKey = + OM_KEY_PREFIX + VOL + + OM_KEY_PREFIX + BUCKET_ONE + + OM_KEY_PREFIX + DIR_ONE + + OM_KEY_PREFIX + DIR_THREE + OM_KEY_PREFIX; + OmKeyInfo omDirDeleteValue = buildOmKeyInfo(VOL, BUCKET_ONE, + (DIR_ONE + OM_KEY_PREFIX + DIR_THREE + OM_KEY_PREFIX), + DIR_THREE, DIR_THREE_OBJECT_ID, DIR_ONE_OBJECT_ID); + keyEvent6 = new OMDBUpdateEvent. + OMUpdateEventBuilder() + .setKey(omDirDeleteKey) + .setValue(omDirDeleteValue) + .setAction(OMDBUpdateEvent.OMDBUpdateAction.DELETE) + .setTable(omMetadataManager.getKeyTable(getBucketLayout()).getName()) + .build(); + + // rename dir1 + String omDirUpdateKey = + OM_KEY_PREFIX + VOL + + OM_KEY_PREFIX + BUCKET_ONE + + OM_KEY_PREFIX + DIR_ONE + OM_KEY_PREFIX; + OmKeyInfo omDirOldValue = buildOmDirKeyInfo(VOL, BUCKET_ONE, + (DIR_ONE + OM_KEY_PREFIX), DIR_ONE, + DIR_ONE_OBJECT_ID); + OmKeyInfo omDirUpdateValue = buildOmDirKeyInfo(VOL, BUCKET_ONE, + (DIR_ONE_RENAME + OM_KEY_PREFIX), DIR_ONE_RENAME, + DIR_ONE_OBJECT_ID); + keyEvent7 = new OMDBUpdateEvent. + OMUpdateEventBuilder() + .setKey(omDirUpdateKey) + .setValue(omDirUpdateValue) + .setOldValue(omDirOldValue) + .setAction(OMDBUpdateEvent.OMDBUpdateAction.UPDATE) + .setTable(omMetadataManager.getKeyTable(getBucketLayout()).getName()) + .build(); + + OMUpdateEventBatch omUpdateEventBatch = new OMUpdateEventBatch( + new ArrayList() {{ + add(keyEvent1); + add(keyEvent2); + add(keyEvent3); + add(keyEvent4); + add(keyEvent5); + add(keyEvent6); + add(keyEvent7); + }}); + + return omUpdateEventBatch; + } + + @Test + public void testProcessUpdateFileSize() throws IOException { + // file 1 is gone, so bucket 1 is empty now + Assert.assertNotNull(nsSummaryForBucket1); + Assert.assertEquals(0, nsSummaryForBucket1.getNumOfFiles()); + + Set childDirBucket1 = nsSummaryForBucket1.getChildDir(); + // after put dir4, bucket1 now has two child dirs: dir1 and dir4 + Assert.assertEquals(2, childDirBucket1.size()); + bucketOneAns.clear(); + bucketOneAns.add(DIR_ONE_OBJECT_ID); + bucketOneAns.add(DIR_FOUR_OBJECT_ID); + Assert.assertEquals(bucketOneAns, childDirBucket1); + } + + @Test + public void testProcessBucket() throws IOException { + // file 5 is added under bucket 2, so bucket 2 has 3 keys now + // file 2 is updated with new datasize, + // so file size dist for bucket 2 should be updated + Assert.assertNotNull(nsSummaryForBucket2); + Assert.assertEquals(3, nsSummaryForBucket2.getNumOfFiles()); + // key 4 + key 5 + updated key 2 + Assert.assertEquals(KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_TWO_UPDATE_SIZE, + nsSummaryForBucket2.getSizeOfFiles()); + + int[] fileSizeDist = nsSummaryForBucket2.getFileSizeBucket(); + Assert.assertEquals(ReconConstants.NUM_OF_BINS, fileSizeDist.length); + // 1023L and 100L + Assert.assertEquals(2, fileSizeDist[0]); + // 2050L + Assert.assertEquals(1, fileSizeDist[2]); + for (int i = 0; i < ReconConstants.NUM_OF_BINS; ++i) { + if (i == 0 || i == 2) { + continue; + } + Assert.assertEquals(0, fileSizeDist[i]); + } + + // after put dir5, bucket 2 now has one dir + Set childDirBucket2 = nsSummaryForBucket2.getChildDir(); + Assert.assertEquals(1, childDirBucket2.size()); + bucketTwoAns.add(DIR_FIVE_OBJECT_ID); + Assert.assertEquals(bucketTwoAns, childDirBucket2); + } + + @Test + public void testProcessDirDeleteRename() throws IOException { + // after delete dir 3, dir 1 now has only one dir: dir2 + NSSummary nsSummaryForDir1 = reconNamespaceSummaryManager + .getNSSummary(DIR_ONE_OBJECT_ID); + Assert.assertNotNull(nsSummaryForDir1); + Set childDirForDir1 = nsSummaryForDir1.getChildDir(); + Assert.assertEquals(1, childDirForDir1.size()); + dirOneAns.clear(); + dirOneAns.add(DIR_TWO_OBJECT_ID); + Assert.assertEquals(dirOneAns, childDirForDir1); + + // after renaming dir1, check its new name + Assert.assertEquals(DIR_ONE_RENAME, nsSummaryForDir1.getDirName()); + } + } + + /** + * Build a key info for put/update action. + * @param volume volume name + * @param bucket bucket name + * @param key key name + * @param fileName file name + * @param objectID object ID + * @param parentObjectId parent object ID + * @param dataSize file size + * @return the KeyInfo + */ + private static OmKeyInfo buildOmKeyInfo(String volume, + String bucket, + String key, + String fileName, + long objectID, + long parentObjectId, + long dataSize) { + return new OmKeyInfo.Builder() + .setBucketName(bucket) + .setVolumeName(volume) + .setKeyName(key) + .setFileName(fileName) + .setReplicationConfig( + StandaloneReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.ONE)) + .setObjectID(objectID) + .setParentObjectID(parentObjectId) + .setDataSize(dataSize) + .build(); + } + + /** + * Build a key info for delete action. + * @param volume volume name + * @param bucket bucket name + * @param key key name + * @param fileName file name + * @param objectID object ID + * @param parentObjectId parent object ID + * @return the KeyInfo + */ + private static OmKeyInfo buildOmKeyInfo(String volume, + String bucket, + String key, + String fileName, + long objectID, + long parentObjectId) { + return new OmKeyInfo.Builder() + .setBucketName(bucket) + .setVolumeName(volume) + .setKeyName(key) + .setFileName(fileName) + .setReplicationConfig( + StandaloneReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.ONE)) + .setObjectID(objectID) + .setParentObjectID(parentObjectId) + .build(); + } + + /** + * Build a directory as key info for put/update action. + * We don't need to set size. + * @param volume volume name + * @param bucket bucket name + * @param key key name + * @param fileName file name + * @param objectID object ID + * @return the KeyInfo + */ + private static OmKeyInfo buildOmDirKeyInfo(String volume, + String bucket, + String key, + String fileName, + long objectID) { + return new OmKeyInfo.Builder() + .setBucketName(bucket) + .setVolumeName(volume) + .setKeyName(key) + .setFileName(fileName) + .setReplicationConfig( + StandaloneReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.ONE)) + .setObjectID(objectID) + .build(); + } + + /** + * Populate OMDB with the following configs. + * vol + * / \ + * bucket1 bucket2 + * / \ / \ + * file1 dir1 file2 file4 + * / \ + * dir2 dir3 + * / + * file3 + * + * @throws IOException + */ + private static void populateOMDB() throws IOException { + writeKeyToOm(reconOMMetadataManager, + KEY_ONE, + BUCKET_ONE, + VOL, + FILE_ONE, + KEY_ONE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + KEY_ONE_SIZE, + getBucketLayout()); + writeKeyToOm(reconOMMetadataManager, + KEY_TWO, + BUCKET_TWO, + VOL, + FILE_TWO, + KEY_TWO_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_TWO_OBJECT_ID, + VOL_OBJECT_ID, + KEY_TWO_OLD_SIZE, + getBucketLayout()); + writeKeyToOm(reconOMMetadataManager, + KEY_THREE, + BUCKET_ONE, + VOL, + FILE_THREE, + KEY_THREE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + KEY_THREE_SIZE, + getBucketLayout()); + writeKeyToOm(reconOMMetadataManager, + KEY_FOUR, + BUCKET_TWO, + VOL, + FILE_FOUR, + KEY_FOUR_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_TWO_OBJECT_ID, + VOL_OBJECT_ID, + KEY_FOUR_SIZE, + getBucketLayout()); + + writeDirToOm(reconOMMetadataManager, + (DIR_ONE + OM_KEY_PREFIX), + BUCKET_ONE, + VOL, + DIR_ONE, + DIR_ONE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + getBucketLayout()); + writeDirToOm(reconOMMetadataManager, + (DIR_ONE + OM_KEY_PREFIX + + DIR_TWO + OM_KEY_PREFIX), + BUCKET_ONE, + VOL, + DIR_TWO, + DIR_TWO_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + getBucketLayout()); + writeDirToOm(reconOMMetadataManager, + (DIR_ONE + OM_KEY_PREFIX + + DIR_THREE + OM_KEY_PREFIX), + BUCKET_ONE, + VOL, + DIR_THREE, + DIR_THREE_OBJECT_ID, + PARENT_OBJECT_ID_ZERO, + BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, + getBucketLayout()); + } + + /** + * Create a new OM Metadata manager instance with one user, one vol, and two + * buckets. + * @throws IOException ioEx + */ + private static void initializeNewOmMetadataManager( + File omDbDir) + throws IOException { + omConfiguration = new OzoneConfiguration(); + omConfiguration.set(OZONE_OM_DB_DIRS, + omDbDir.getAbsolutePath()); + omConfiguration.set(OMConfigKeys + .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); + omMetadataManager = new OmMetadataManagerImpl( + omConfiguration); + + String volumeKey = omMetadataManager.getVolumeKey(VOL); + OmVolumeArgs args = + OmVolumeArgs.newBuilder() + .setObjectID(VOL_OBJECT_ID) + .setVolume(VOL) + .setAdminName(TEST_USER) + .setOwnerName(TEST_USER) + .build(); + omMetadataManager.getVolumeTable().put(volumeKey, args); + + OmBucketInfo bucketInfo1 = OmBucketInfo.newBuilder() + .setVolumeName(VOL) + .setBucketName(BUCKET_ONE) + .setObjectID(BUCKET_ONE_OBJECT_ID) + .setBucketLayout(getBucketLayout()) + .build(); + + OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() + .setVolumeName(VOL) + .setBucketName(BUCKET_TWO) + .setObjectID(BUCKET_TWO_OBJECT_ID) + .setBucketLayout(getBucketLayout()) + .build(); + + String bucketKey = omMetadataManager.getBucketKey( + bucketInfo1.getVolumeName(), bucketInfo1.getBucketName()); + String bucketKey2 = omMetadataManager.getBucketKey( + bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); + + omMetadataManager.getBucketTable().put(bucketKey, bucketInfo1); + omMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); + } + + private static BucketLayout getBucketLayout() { + return BucketLayout.LEGACY; + } +} \ No newline at end of file diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java index 3a4a16d54d33..59348894326a 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java @@ -32,7 +32,7 @@ import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.makeHttpCall; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.parseInputPath; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printEmptyPathRequest; -import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printFSOReminder; +import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printBucketReminder; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printKVSeparator; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printNewLines; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printPathNotFound; @@ -106,8 +106,9 @@ public Void call() throws Exception { if (duResponse.get("status").equals("PATH_NOT_FOUND")) { printPathNotFound(); } else { - if (!parent.isFileSystemOptimizedBucket(path)) { - printFSOReminder(); + if (parent.isObjectStoreBucket(path) || + !parent.bucketIsPresentInThePath(path)) { + printBucketReminder(); } long totalSize = (long)(double)duResponse.get("size"); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java index 9f02121c8fc8..5a2a2d11c025 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java @@ -28,7 +28,7 @@ import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.getResponseMap; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.makeHttpCall; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printEmptyPathRequest; -import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printFSOReminder; +import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printBucketReminder; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printNewLines; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printPathNotFound; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printSpaces; @@ -80,8 +80,9 @@ public Void call() throws Exception { } else if (distResponse.get("status").equals("TYPE_NOT_APPLICABLE")) { printTypeNA("File Size Distribution"); } else { - if (!parent.isFileSystemOptimizedBucket(path)) { - printFSOReminder(); + if (parent.isObjectStoreBucket(path) || + !parent.bucketIsPresentInThePath(path)) { + printBucketReminder(); } printWithUnderline("File Size Distribution", true); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java index 220365883d77..727be27670a9 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java @@ -30,12 +30,14 @@ import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.kohsuke.MetaInfServices; import picocli.CommandLine; import java.io.IOException; import java.util.HashSet; +import java.util.Objects; import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_DEFAULT; import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY; @@ -108,6 +110,61 @@ public boolean isFileSystemOptimizedBucket(String path) throws IOException { } } + public boolean isObjectStoreBucket(String path) throws IOException { + OFSPath ofsPath = new OFSPath(path); + + boolean enableFileSystemPaths = getOzoneConfig() + .getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, + OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT); + + OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(getOzoneConfig()); + ObjectStore objectStore = ozoneClient.getObjectStore(); + + try { + OzoneBucket bucket = objectStore.getVolume(ofsPath.getVolumeName()) + .getBucket(ofsPath.getBucketName()); + + // Resolve the bucket layout in case this is a Link Bucket. + BucketLayout resolvedBucketLayout = + OzoneClientUtils.resolveLinkBucketLayout(bucket, objectStore, + new HashSet<>()); + + return resolvedBucketLayout.isObjectStore(enableFileSystemPaths); + } catch (IOException e) { + System.out.println( + "Bucket layout couldn't be verified for path: " + ofsPath + + ". Exception: " + e); + return false; + } + } + + /** + * Checking if the bucket is part of the path. + * Return false if path is root, just a volume or invalid. + * @param path + * @return true if the bucket + * is not part of the given path. + * @throws IOException + */ + public boolean bucketIsPresentInThePath(String path) throws IOException { + OFSPath ofsPath = new OFSPath(path); + + OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(getOzoneConfig()); + ObjectStore objectStore = ozoneClient.getObjectStore(); + + try { + OzoneBucket bucket = objectStore.getVolume(ofsPath.getVolumeName()) + .getBucket(ofsPath.getBucketName()); + + return Objects.nonNull(bucket); + } catch (IOException e) { + System.out.println( + "Bucket layout couldn't be verified for path: " + ofsPath + + ". Exception: " + e); + return false; + } + } + /** * e.g. Input: "0.0.0.0:9891" -> Output: "0.0.0.0" */ diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java index 9c56924af2f6..729aa20c5ce3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java @@ -152,12 +152,13 @@ public static void printWithUnderline(String str, boolean newLine) { } } - public static void printFSOReminder() { + public static void printBucketReminder() { printNewLines(1); System.out.println( - "[Warning] Namespace CLI is only designed for FSO mode.\n" + - "Bucket being accessed must be of type FILE_SYSTEM_OPTIMIZED" + - " bucket layout."); + "[Warning] Namespace CLI is not designed for OBS bucket layout.\n" + + "Bucket being accessed must be of type FILE_SYSTEM_OPTIMIZED " + + "bucket layout or \nLEGACY bucket layout with " + + "'ozone.om.enable.filesystem.paths' set to true."); printNewLines(1); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java index 88a7b2a55408..c3494cf4ffba 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java @@ -27,7 +27,7 @@ import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.getResponseMap; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.makeHttpCall; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printEmptyPathRequest; -import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printFSOReminder; +import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printBucketReminder; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printKVSeparator; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printNewLines; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printPathNotFound; @@ -80,8 +80,9 @@ public Void call() throws Exception { } else if (quotaResponse.get("status").equals("TYPE_NOT_APPLICABLE")) { printTypeNA("Quota"); } else { - if (!parent.isFileSystemOptimizedBucket(path)) { - printFSOReminder(); + if (parent.isObjectStoreBucket(path) || + !parent.bucketIsPresentInThePath(path)) { + printBucketReminder(); } printWithUnderline("Quota", true); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java index c0d2ed7f0cb8..4a4946bb8092 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java @@ -27,7 +27,7 @@ import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.makeHttpCall; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.parseInputPath; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printEmptyPathRequest; -import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printFSOReminder; +import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printBucketReminder; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printKVSeparator; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printNewLines; import static org.apache.hadoop.ozone.admin.nssummary.NSSummaryCLIUtils.printPathNotFound; @@ -76,8 +76,9 @@ public Void call() throws Exception { if (summaryResponse.get("status").equals("PATH_NOT_FOUND")) { printPathNotFound(); } else { - if (!parent.isFileSystemOptimizedBucket(path)) { - printFSOReminder(); + if (parent.isObjectStoreBucket(path) || + !parent.bucketIsPresentInThePath(path)) { + printBucketReminder(); } printWithUnderline("Entity Type", false); From 462f32dd6963ef44da95fa6c4dd006b2f5559725 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Wed, 26 Oct 2022 09:55:50 +0800 Subject: [PATCH 26/48] HDDS-7396. Force close non-RATIS containers in ReplicationManager (#3877) --- .../health/ClosingContainerHandler.java | 5 +- .../health/TestClosingContainerHandler.java | 103 ++++++++++-------- 2 files changed, 63 insertions(+), 45 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java index 9259d084c05f..103f7d66463d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java @@ -51,10 +51,13 @@ public boolean handle(ContainerCheckRequest request) { return false; } + boolean forceClose = request.getContainerInfo().getReplicationConfig() + .getReplicationType() != HddsProtos.ReplicationType.RATIS; + for (ContainerReplica replica : request.getContainerReplicas()) { if (replica.getState() != ContainerReplicaProto.State.UNHEALTHY) { replicationManager.sendCloseContainerReplicaCommand( - containerInfo, replica.getDatanodeDetails(), false); + containerInfo, replica.getDatanodeDetails(), forceClose); } } return true; diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java index 2999884e1d26..06c2c7fa63e5 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; @@ -32,13 +33,20 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import java.util.Collections; +import java.util.HashSet; import java.util.Set; +import java.util.stream.Stream; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSING; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.EC; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS; /** * Tests for {@link ClosingContainerHandler}. @@ -46,18 +54,21 @@ public class TestClosingContainerHandler { private ReplicationManager replicationManager; private ClosingContainerHandler closingContainerHandler; - private ECReplicationConfig ecReplicationConfig; - private RatisReplicationConfig ratisReplicationConfig; + private static final ECReplicationConfig EC_REPLICATION_CONFIG = + new ECReplicationConfig(3, 2); + private static final RatisReplicationConfig RATIS_REPLICATION_CONFIG = + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE); @BeforeEach public void setup() { - ecReplicationConfig = new ECReplicationConfig(3, 2); - ratisReplicationConfig = RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE); replicationManager = Mockito.mock(ReplicationManager.class); closingContainerHandler = new ClosingContainerHandler(replicationManager); } + private static Stream replicationConfigs() { + return Stream.of(RATIS_REPLICATION_CONFIG, EC_REPLICATION_CONFIG); + } + /** * If a container is not closing, it should not be handled by * ClosingContainerHandler. It should return false so the request can be @@ -66,7 +77,7 @@ public void setup() { @Test public void testNonClosingContainerReturnsFalse() { ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( - ecReplicationConfig, 1, CLOSED); + EC_REPLICATION_CONFIG, 1, CLOSED); Set containerReplicas = ReplicationTestUtil .createReplicas(containerInfo.containerID(), ContainerReplicaProto.State.CLOSING, 1, 2, 3, 4, 5); @@ -84,7 +95,7 @@ public void testNonClosingContainerReturnsFalse() { @Test public void testNonClosingRatisContainerReturnsFalse() { ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( - ratisReplicationConfig, 1, CLOSED); + RATIS_REPLICATION_CONFIG, 1, CLOSED); Set containerReplicas = ReplicationTestUtil .createReplicas(containerInfo.containerID(), ContainerReplicaProto.State.CLOSING, 0, 0, 0); @@ -107,7 +118,7 @@ public void testNonClosingRatisContainerReturnsFalse() { @Test public void testUnhealthyReplicaIsNotClosed() { ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( - ecReplicationConfig, 1, CLOSING); + EC_REPLICATION_CONFIG, 1, CLOSING); Set containerReplicas = ReplicationTestUtil .createReplicas(containerInfo.containerID(), ContainerReplicaProto.State.UNHEALTHY, 1, 2, 3, 4); @@ -130,7 +141,7 @@ public void testUnhealthyReplicaIsNotClosed() { @Test public void testUnhealthyRatisReplicaIsNotClosed() { ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( - ratisReplicationConfig, 1, CLOSING); + RATIS_REPLICATION_CONFIG, 1, CLOSING); Set containerReplicas = ReplicationTestUtil .createReplicas(containerInfo.containerID(), ContainerReplicaProto.State.UNHEALTHY, 0, 0); @@ -153,51 +164,55 @@ public void testUnhealthyRatisReplicaIsNotClosed() { /** * Close commands should be sent for Open or Closing replicas. */ - @Test - public void testOpenOrClosingReplicasAreClosed() { + @ParameterizedTest + @MethodSource("replicationConfigs") + public void testOpenOrClosingReplicasAreClosed(ReplicationConfig repConfig) { ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( - ecReplicationConfig, 1, CLOSING); - Set containerReplicas = ReplicationTestUtil - .createReplicas(containerInfo.containerID(), - ContainerReplicaProto.State.CLOSING, 1, 2); - containerReplicas.add(ReplicationTestUtil.createContainerReplica( - containerInfo.containerID(), 3, - HddsProtos.NodeOperationalState.IN_SERVICE, - ContainerReplicaProto.State.OPEN)); + repConfig, 1, CLOSING); + + final int replicas = repConfig.getRequiredNodes(); + final int closing = replicas / 2; + final boolean force = repConfig.getReplicationType() != RATIS; + + Set containerReplicas = new HashSet<>(); + + // Add CLOSING container replicas. + // For EC, replica index will be in [1, closing]. + for (int i = 1; i <= closing; i++) { + containerReplicas.add(ReplicationTestUtil.createContainerReplica( + containerInfo.containerID(), + repConfig.getReplicationType() == EC ? i : 0, + HddsProtos.NodeOperationalState.IN_SERVICE, + ContainerReplicaProto.State.CLOSING)); + } + + // Add OPEN container replicas. + // For EC, replica index will be in [closing + 1, replicas]. + for (int i = closing + 1; i <= replicas; i++) { + containerReplicas.add(ReplicationTestUtil.createContainerReplica( + containerInfo.containerID(), + repConfig.getReplicationType() == EC ? i : 0, + HddsProtos.NodeOperationalState.IN_SERVICE, + ContainerReplicaProto.State.OPEN)); + } ContainerCheckRequest request = new ContainerCheckRequest.Builder() - .setPendingOps(Collections.EMPTY_LIST) + .setPendingOps(Collections.emptyList()) .setReport(new ReplicationManagerReport()) .setContainerInfo(containerInfo) .setContainerReplicas(containerReplicas) .build(); - assertAndVerify(request, true, 3); - } - - @Test - public void testOpenOrClosingRatisReplicasAreClosed() { - ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( - ratisReplicationConfig, 1, CLOSING); - Set containerReplicas = ReplicationTestUtil - .createReplicas(containerInfo.containerID(), - ContainerReplicaProto.State.CLOSING, 0, 0); - containerReplicas.add(ReplicationTestUtil.createContainerReplica( - containerInfo.containerID(), 0, - HddsProtos.NodeOperationalState.IN_SERVICE, - ContainerReplicaProto.State.OPEN)); - - ContainerCheckRequest request = new ContainerCheckRequest.Builder() - .setPendingOps(Collections.EMPTY_LIST) - .setReport(new ReplicationManagerReport()) - .setContainerInfo(containerInfo) - .setContainerReplicas(containerReplicas) - .build(); - - assertAndVerify(request, true, 3); + ArgumentCaptor forceCaptor = + ArgumentCaptor.forClass(Boolean.class); + Assertions.assertTrue(closingContainerHandler.handle(request)); + Mockito.verify(replicationManager, Mockito.times(replicas)) + .sendCloseContainerReplicaCommand(Mockito.any(ContainerInfo.class), + Mockito.any(DatanodeDetails.class), forceCaptor.capture()); + forceCaptor.getAllValues() + .forEach(f -> Assertions.assertEquals(force, f)); } - private void assertAndVerify(ContainerCheckRequest request, boolean assertion, int times) { Assertions.assertEquals(assertion, closingContainerHandler.handle(request)); From 561788ebe7da445aba93356c61e018f74c9cde52 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Wed, 26 Oct 2022 11:31:28 +0800 Subject: [PATCH 27/48] Revert "HDDS-7253. Fix exception when '/' in key name (#3774)" This reverts commit b9a47f6bbf72605d5391cafc8d382190f822b375. --- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 81 +------------------ .../hadoop/ozone/om/TestKeyManagerImpl.java | 70 +--------------- .../hadoop/ozone/om/KeyManagerImpl.java | 68 +++------------- 3 files changed, 20 insertions(+), 199 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index 1f688029b6d1..f3e8cf10bee3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -153,7 +153,6 @@ public TestOzoneFileSystem(boolean setDefaultFs, boolean enableOMRatis) { private static OzoneManagerProtocol writeClient; private static FileSystem fs; private static OzoneFileSystem o3fs; - private static OzoneBucket ozoneBucket; private static String volumeName; private static String bucketName; private static Trash trash; @@ -180,9 +179,10 @@ private void init() throws Exception { writeClient = cluster.getRpcClient().getObjectStore() .getClientProxy().getOzoneManagerClient(); // create a volume and a bucket to be used by OzoneFileSystem - ozoneBucket = TestDataUtil.createVolumeAndBucket(cluster, bucketLayout); - volumeName = ozoneBucket.getVolumeName(); - bucketName = ozoneBucket.getName(); + OzoneBucket bucket = + TestDataUtil.createVolumeAndBucket(cluster, bucketLayout); + volumeName = bucket.getVolumeName(); + bucketName = bucket.getName(); String rootPath = String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName); @@ -334,30 +334,6 @@ public void testMakeDirsWithAnExistingDirectoryPath() throws Exception { assertTrue("Shouldn't send error if dir exists", status); } - @Test - public void testMakeDirsWithAnFakeDirectory() throws Exception { - /* - * Op 1. commit a key -> "dir1/dir2/key1" - * Op 2. create dir -> "dir1/testDir", the dir1 is a fake dir, - * "dir1/testDir" can be created normal - */ - - String fakeGrandpaKey = "dir1"; - String fakeParentKey = fakeGrandpaKey + "/dir2"; - String fullKeyName = fakeParentKey + "/key1"; - TestDataUtil.createKey(ozoneBucket, fullKeyName, ""); - - // /dir1/dir2 should not exist - assertFalse(fs.exists(new Path(fakeParentKey))); - - // /dir1/dir2/key2 should be created because has a fake parent directory - Path subdir = new Path(fakeParentKey, "key2"); - assertTrue(fs.mkdirs(subdir)); - // the intermediate directories /dir1 and /dir1/dir2 will be created too - assertTrue(fs.exists(new Path(fakeGrandpaKey))); - assertTrue(fs.exists(new Path(fakeParentKey))); - } - @Test public void testCreateWithInvalidPaths() throws Exception { // Test for path with .. @@ -751,37 +727,6 @@ public void testListStatusOnLargeDirectory() throws Exception { } } - @Test - public void testListStatusOnKeyNameContainDelimiter() throws Exception { - /* - * op1: create a key -> "dir1/dir2/key1" - * op2: `ls /` child dir "/dir1/" will be return - * op2: `ls /dir1` child dir "/dir1/dir2/" will be return - * op3: `ls /dir1/dir2` file "/dir1/dir2/key" will be return - * - * the "/dir1", "/dir1/dir2/" are fake directory - * */ - String keyName = "dir1/dir2/key1"; - TestDataUtil.createKey(ozoneBucket, keyName, ""); - FileStatus[] fileStatuses; - - fileStatuses = fs.listStatus(new Path("/")); - assertEquals(1, fileStatuses.length); - assertEquals("/dir1", fileStatuses[0].getPath().toUri().getPath()); - assertTrue(fileStatuses[0].isDirectory()); - - fileStatuses = fs.listStatus(new Path("/dir1")); - assertEquals(1, fileStatuses.length); - assertEquals("/dir1/dir2", fileStatuses[0].getPath().toUri().getPath()); - assertTrue(fileStatuses[0].isDirectory()); - - fileStatuses = fs.listStatus(new Path("/dir1/dir2")); - assertEquals(1, fileStatuses.length); - assertEquals("/dir1/dir2/key1", - fileStatuses[0].getPath().toUri().getPath()); - assertTrue(fileStatuses[0].isFile()); - } - /** * Cleanup files and directories. * @@ -1328,24 +1273,6 @@ public void testRenameFileToDir() throws Exception { "file1"))); } - @Test - public void testRenameContainDelimiterFile() throws Exception { - String fakeGrandpaKey = "dir1"; - String fakeParentKey = fakeGrandpaKey + "/dir2"; - String sourceKeyName = fakeParentKey + "/key1"; - String targetKeyName = fakeParentKey + "/key2"; - TestDataUtil.createKey(ozoneBucket, sourceKeyName, ""); - - Path sourcePath = new Path(fs.getUri().toString() + "/" + sourceKeyName); - Path targetPath = new Path(fs.getUri().toString() + "/" + targetKeyName); - assertTrue(fs.rename(sourcePath, targetPath)); - assertFalse(fs.exists(sourcePath)); - assertTrue(fs.exists(targetPath)); - // intermediate directories will not be created - assertFalse(fs.exists(new Path(fakeGrandpaKey))); - assertFalse(fs.exists(new Path(fakeParentKey))); - } - /** * Fails if the (a) parent of dst does not exist or (b) parent is a file. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index e30d27fc7460..492173b71cf9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -800,20 +800,19 @@ public void testLookupKeyWithLocation() throws IOException { List locationList = keySession.getKeyInfo().getLatestVersionLocations().getLocationList(); Assert.assertEquals(1, locationList.size()); - long containerID = locationList.get(0).getContainerID(); locationInfoList.add( new OmKeyLocationInfo.Builder().setPipeline(pipeline) - .setBlockID(new BlockID(containerID, + .setBlockID(new BlockID(locationList.get(0).getContainerID(), locationList.get(0).getLocalID())).build()); keyArgs.setLocationInfoList(locationInfoList); writeClient.commitKey(keyArgs, keySession.getId()); - ContainerInfo containerInfo = new ContainerInfo.Builder() - .setContainerID(containerID).setPipelineID(pipeline.getId()).build(); + ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(1L) + .setPipelineID(pipeline.getId()).build(); List containerWithPipelines = Arrays.asList( new ContainerWithPipeline(containerInfo, pipeline)); when(mockScmContainerClient.getContainerWithPipelineBatch( - Arrays.asList(containerID))).thenReturn(containerWithPipelines); + Arrays.asList(1L))).thenReturn(containerWithPipelines); OmKeyInfo key = keyManager.lookupKey(keyArgs, null); Assert.assertEquals(key.getKeyName(), keyName); @@ -1274,67 +1273,6 @@ public void testListStatus() throws IOException { } } - @Test - public void testGetFileStatus() throws IOException { - // create a key - String keyName = RandomStringUtils.randomAlphabetic(5); - OmKeyArgs keyArgs = createBuilder() - .setKeyName(keyName) - .setLatestVersionLocation(true) - .build(); - writeClient.createFile(keyArgs, false, false); - OpenKeySession keySession = writeClient.createFile(keyArgs, true, true); - keyArgs.setLocationInfoList( - keySession.getKeyInfo().getLatestVersionLocations().getLocationList()); - writeClient.commitKey(keyArgs, keySession.getId()); - OzoneFileStatus ozoneFileStatus = keyManager.getFileStatus(keyArgs); - Assert.assertEquals(keyName, ozoneFileStatus.getKeyInfo().getFileName()); - } - - @Test - public void testGetFileStatusWithFakeDir() throws IOException { - String parentDir = "dir1"; - String key = "key1"; - String fullKeyName = parentDir + OZONE_URI_DELIMITER + key; - OzoneFileStatus ozoneFileStatus; - - // create a key "dir1/key1" - OmKeyArgs keyArgs = createBuilder().setKeyName(fullKeyName).build(); - OpenKeySession keySession = writeClient.openKey(keyArgs); - keyArgs.setLocationInfoList( - keySession.getKeyInfo().getLatestVersionLocations().getLocationList()); - writeClient.commitKey(keyArgs, keySession.getId()); - - // verify - String keyArg; - keyArg = metadataManager.getOzoneKey(VOLUME_NAME, BUCKET_NAME, parentDir); - Assert.assertNull( - metadataManager.getKeyTable(getDefaultBucketLayout()).get(keyArg)); - keyArg = metadataManager.getOzoneKey(VOLUME_NAME, BUCKET_NAME, fullKeyName); - Assert.assertNotNull(metadataManager.getKeyTable(getDefaultBucketLayout()) - .get(keyArg)); - - // get a non-existing "dir1", since the key is prefixed "dir1/key1", - // a fake "/dir1" will be returned - keyArgs = createBuilder().setKeyName(parentDir).build(); - ozoneFileStatus = keyManager.getFileStatus(keyArgs); - Assert.assertEquals(parentDir, ozoneFileStatus.getKeyInfo().getFileName()); - Assert.assertTrue(ozoneFileStatus.isDirectory()); - - // get a non-existing "dir", since the key is not prefixed "dir1/key1", - // a `OMException` will be thrown - keyArgs = createBuilder().setKeyName("dir").build(); - OmKeyArgs finalKeyArgs = keyArgs; - Assert.assertThrows(OMException.class, () -> keyManager.getFileStatus( - finalKeyArgs)); - - // get a file "dir1/key1" - keyArgs = createBuilder().setKeyName(fullKeyName).build(); - ozoneFileStatus = keyManager.getFileStatus(keyArgs); - Assert.assertEquals(key, ozoneFileStatus.getKeyInfo().getFileName()); - Assert.assertTrue(ozoneFileStatus.isFile()); - } - @Test public void testRefreshPipeline() throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 483a2e88b045..dd66c33a1e3b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1209,8 +1209,6 @@ private OzoneFileStatus getOzoneFileStatus(OmKeyArgs args, final String keyName = args.getKeyName(); OmKeyInfo fileKeyInfo = null; - OmKeyInfo dirKeyInfo = null; - OmKeyInfo fakeDirKeyInfo = null; metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, bucketName); try { @@ -1223,27 +1221,28 @@ private OzoneFileStatus getOzoneFileStatus(OmKeyArgs args, // Check if the key is a file. String fileKeyBytes = metadataManager.getOzoneKey( volumeName, bucketName, keyName); - BucketLayout layout = - getBucketLayout(metadataManager, volumeName, bucketName); - fileKeyInfo = metadataManager.getKeyTable(layout).get(fileKeyBytes); - String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); + fileKeyInfo = metadataManager + .getKeyTable(getBucketLayout(metadataManager, volumeName, bucketName)) + .get(fileKeyBytes); // Check if the key is a directory. if (fileKeyInfo == null) { + String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); String dirKeyBytes = metadataManager.getOzoneKey( volumeName, bucketName, dirKey); - dirKeyInfo = metadataManager.getKeyTable(layout).get(dirKeyBytes); - if (dirKeyInfo == null) { - fakeDirKeyInfo = - createFakeDirIfShould(volumeName, bucketName, keyName, layout); + OmKeyInfo dirKeyInfo = metadataManager.getKeyTable( + getBucketLayout(metadataManager, volumeName, bucketName)) + .get(dirKeyBytes); + if (dirKeyInfo != null) { + return new OzoneFileStatus(dirKeyInfo, scmBlockSize, true); } } } finally { metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, bucketName); + + // if the key is a file then do refresh pipeline info in OM by asking SCM if (fileKeyInfo != null) { - // if the key is a file - // then do refresh pipeline info in OM by asking SCM if (args.getLatestVersionLocation()) { slimLocationVersion(fileKeyInfo); } @@ -1258,21 +1257,10 @@ private OzoneFileStatus getOzoneFileStatus(OmKeyArgs args, sortDatanodes(clientAddress, fileKeyInfo); } } + return new OzoneFileStatus(fileKeyInfo, scmBlockSize, false); } } - if (fileKeyInfo != null) { - return new OzoneFileStatus(fileKeyInfo, scmBlockSize, false); - } - - if (dirKeyInfo != null) { - return new OzoneFileStatus(dirKeyInfo, scmBlockSize, true); - } - - if (fakeDirKeyInfo != null) { - return new OzoneFileStatus(fakeDirKeyInfo, scmBlockSize, true); - } - // Key is not found, throws exception if (LOG.isDebugEnabled()) { LOG.debug("Unable to get file status for the key: volume: {}, bucket:" + @@ -1284,37 +1272,6 @@ private OzoneFileStatus getOzoneFileStatus(OmKeyArgs args, FILE_NOT_FOUND); } - /** - * Create a fake directory if the key is a path prefix, - * otherwise returns null. - * Some keys may contain '/' Ozone will treat '/' as directory separator - * such as : key name is 'a/b/c', 'a' and 'b' may not really exist, - * but Ozone treats 'a' and 'b' as a directory. - * we need create a fake directory 'a' or 'a/b' - * - * @return OmKeyInfo if the key is a path prefix, otherwise returns null. - */ - private OmKeyInfo createFakeDirIfShould(String volume, String bucket, - String keyName, BucketLayout layout) throws IOException { - OmKeyInfo fakeDirKeyInfo = null; - String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); - String fileKeyBytes = metadataManager.getOzoneKey(volume, bucket, keyName); - Table.KeyValue keyValue = - metadataManager.getKeyTable(layout).iterator().seek(fileKeyBytes); - - if (keyValue != null) { - Path fullPath = Paths.get(keyValue.getValue().getKeyName()); - Path subPath = Paths.get(dirKey); - OmKeyInfo omKeyInfo = keyValue.getValue(); - if (fullPath.startsWith(subPath)) { - // create fake directory - fakeDirKeyInfo = createDirectoryKey(omKeyInfo, dirKey); - } - } - - return fakeDirKeyInfo; - } - private OzoneFileStatus getOzoneFileStatusFSO(OmKeyArgs args, String clientAddress, boolean skipFileNotFoundError) throws IOException { @@ -1392,7 +1349,6 @@ private OmKeyInfo createDirectoryKey(OmKeyInfo keyInfo, String keyName) .setVolumeName(keyInfo.getVolumeName()) .setBucketName(keyInfo.getBucketName()) .setKeyName(dir) - .setFileName(OzoneFSUtils.getFileName(keyName)) .setOmKeyLocationInfos(Collections.singletonList( new OmKeyLocationInfoGroup(0, new ArrayList<>()))) .setCreationTime(Time.now()) From 9449747ff7c655af24f19985901669aa8648e10b Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran <47532440+swamirishi@users.noreply.github.com> Date: Wed, 26 Oct 2022 00:26:45 -0700 Subject: [PATCH 28/48] HDDS-7413. Fix logging while marking container state unhealthy (#3887) --- .../hadoop/ozone/container/keyvalue/KeyValueContainer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java index 9fc8c446629d..7412e766d0c0 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java @@ -328,6 +328,7 @@ public void markContainerForClose() throws StorageContainerException { @Override public void markContainerUnhealthy() throws StorageContainerException { writeLock(); + ContainerDataProto.State prevState = containerData.getState(); try { updateContainerData(() -> containerData.setState(ContainerDataProto.State.UNHEALTHY)); @@ -335,9 +336,9 @@ public void markContainerUnhealthy() throws StorageContainerException { } finally { writeUnlock(); } - LOG.warn("Moving container {} to state UNHEALTHY from state:{} Trace:{}", + LOG.warn("Moving container {} to state {} from state:{} Trace:{}", containerData.getContainerPath(), containerData.getState(), - StringUtils.getStackTrace(Thread.currentThread())); + prevState, StringUtils.getStackTrace(Thread.currentThread())); } @Override From a664ccac6a52bd01619ebb29a76f8c171a71f1cf Mon Sep 17 00:00:00 2001 From: Cyrill Date: Wed, 26 Oct 2022 13:16:35 +0300 Subject: [PATCH 29/48] HDDS-7342. Move encryption-related code from MultipartCryptoKeyInputStream to OzoneCryptoInputStream (#3852) --- .../io/MultipartCryptoKeyInputStream.java | 135 +--------------- .../client/io/OzoneCryptoInputStream.java | 152 +++++++++++++++++- .../hadoop/ozone/client/rpc/RpcClient.java | 6 +- 3 files changed, 156 insertions(+), 137 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/MultipartCryptoKeyInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/MultipartCryptoKeyInputStream.java index 530084e0f38c..c7fc21cbb242 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/MultipartCryptoKeyInputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/MultipartCryptoKeyInputStream.java @@ -65,17 +65,6 @@ public class MultipartCryptoKeyInputStream extends OzoneInputStream // can be reset if a new position is seeked. private int prevPartIndex = 0; - // If a read's start/ length position doesn't coincide with a Crypto buffer - // boundary, it will be adjusted as reads should happen only at the buffer - // boundaries for decryption to happen correctly. In this case, after the - // data has been read and decrypted, only the requested data should be - // returned to the client. readPositionAdjustedBy and readLengthAdjustedBy - // store these adjustment information. Before returning to client, the first - // readPositionAdjustedBy number of bytes and the last readLengthAdjustedBy - // number of bytes must be discarded. - private int readPositionAdjustedBy = 0; - private int readLengthAdjustedBy = 0; - public MultipartCryptoKeyInputStream(String keyName, List inputStreams) { @@ -130,65 +119,7 @@ public int read(byte[] b, int off, int len) throws IOException { // Get the current partStream and read data from it OzoneCryptoInputStream current = partStreams.get(partIndex); - // CryptoInputStream reads hadoop.security.crypto.buffer.size number of - // bytes (default 8KB) at a time. This needs to be taken into account - // in calculating the numBytesToRead. - int numBytesToRead = getNumBytesToRead(len, (int)current.getRemaining(), - current.getBufferSize()); - int numBytesRead; - - if (readPositionAdjustedBy != 0 || readLengthAdjustedBy != 0) { - // There was some adjustment made in position and/ or length of data - // to be read to account for Crypto buffer boundary. Hence, read the - // data into a temp buffer and then copy only the requested data into - // clients buffer. - byte[] tempBuffer = new byte[numBytesToRead]; - int actualNumBytesRead = current.read(tempBuffer, 0, - numBytesToRead); - numBytesRead = actualNumBytesRead - readPositionAdjustedBy - - readLengthAdjustedBy; - - if (actualNumBytesRead != numBytesToRead) { - throw new IOException(String.format("Inconsistent read for key=%s " + - "part=%d length=%d numBytesToRead(accounting for Crypto " + - "boundaries)=%d numBytesRead(actual)=%d " + - "numBytesToBeRead(into client buffer discarding crypto " + - "boundary adjustments)=%d", - key, partIndex, current.getLength(), numBytesToRead, - actualNumBytesRead, numBytesRead)); - } - - // TODO: Byte array copies are not optimal. If there is a better and - // more optimal solution to copy only a part of read data into - // client buffer, this should be updated. - System.arraycopy(tempBuffer, readPositionAdjustedBy, b, off, - numBytesRead); - - LOG.debug("OzoneCryptoInputStream for key: {} part: {} read {} bytes " + - "instead of {} bytes to account for Crypto buffer boundary. " + - "Client buffer will be copied with read data from position {}" + - "upto position {}, discarding the extra bytes read to " + - "maintain Crypto buffer boundary limits", key, partIndex, - actualNumBytesRead, numBytesRead, readPositionAdjustedBy, - actualNumBytesRead - readPositionAdjustedBy); - - if (readLengthAdjustedBy > 0) { - current.seek(current.getPos() - readLengthAdjustedBy); - } - - // Reset readPositionAdjustedBy and readLengthAdjustedBy - readPositionAdjustedBy = 0; - readLengthAdjustedBy = 0; - } else { - numBytesRead = current.read(b, off, numBytesToRead); - if (numBytesRead != numBytesToRead) { - throw new IOException(String.format("Inconsistent read for key=%s " + - "part=%d length=%d numBytesToRead=%d numBytesRead=%d", - key, partIndex, current.getLength(), numBytesToRead, - numBytesRead)); - } - } - + int numBytesRead = current.read(b, off, len); totalReadLen += numBytesRead; off += numBytesRead; len -= numBytesRead; @@ -202,70 +133,6 @@ public int read(byte[] b, int off, int len) throws IOException { return totalReadLen; } - /** - * Get number of bytes to read from the current stream based on the length - * to be read, number of bytes remaining in the stream and the Crypto buffer - * size. - * Reads should be performed at the CryptoInputStream Buffer boundaries only. - * Otherwise, the decryption will be incorrect. - */ - private int getNumBytesToRead(int lenToRead, int remaining, - int cryptoBufferSize) throws IOException { - - Preconditions.checkArgument(readPositionAdjustedBy == 0); - Preconditions.checkArgument(readLengthAdjustedBy == 0); - - // Check and adjust position if required - adjustReadPosition(cryptoBufferSize); - remaining += readPositionAdjustedBy; - lenToRead += readPositionAdjustedBy; - - return adjustNumBytesToRead(lenToRead, remaining, cryptoBufferSize); - } - - /** - * Reads should be performed at the CryptoInputStream Buffer boundary size. - * Otherwise, the decryption will be incorrect. Hence, if the position is - * not at the boundary limit, we have to adjust the position and might need - * to read more data than requested. The extra data will be filtered out - * before returning to the client. - */ - private void adjustReadPosition(long cryptoBufferSize) throws IOException { - // Position of the buffer in current stream - long currentPosOfStream = partStreams.get(partIndex).getPos(); - int modulus = (int) (currentPosOfStream % cryptoBufferSize); - if (modulus != 0) { - // Adjustment required. - // Update readPositionAdjustedBy and seek to the adjusted position - readPositionAdjustedBy = modulus; - // Seek current partStream to adjusted position. We do not need to - // reset the seeked positions of other streams. - partStreams.get(partIndex) - .seek(currentPosOfStream - readPositionAdjustedBy); - LOG.debug("OzoneCryptoInputStream for key: {} part: {} adjusted " + - "position {} by -{} to account for Crypto buffer boundary", - key, partIndex, currentPosOfStream, readPositionAdjustedBy); - } - } - - /** - * If the length of data requested does not end at a Crypto Buffer - * boundary, the number of bytes to be read must be adjusted accordingly. - * The extra data will be filtered out before returning to the client. - */ - private int adjustNumBytesToRead(int lenToRead, int remaining, - int cryptoBufferSize) { - int numBytesToRead = Math.min(cryptoBufferSize, remaining); - if (lenToRead < numBytesToRead) { - // Adjustment required; Update readLengthAdjustedBy. - readLengthAdjustedBy = numBytesToRead - lenToRead; - LOG.debug("OzoneCryptoInputStream for key: {} part: {} adjusted length " + - "by +{} to account for Crypto buffer boundary", - key, partIndex, readLengthAdjustedBy); - } - return numBytesToRead; - } - /** * Seeks the InputStream to the specified position. This involves 2 steps: * 1. Updating the partIndex to the partStream corresponding to the diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneCryptoInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneCryptoInputStream.java index 9d5d888688f4..1d0cb2bb82a2 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneCryptoInputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneCryptoInputStream.java @@ -19,10 +19,14 @@ package org.apache.hadoop.ozone.client.io; import java.io.IOException; + +import com.google.common.base.Preconditions; import org.apache.hadoop.crypto.CryptoCodec; import org.apache.hadoop.crypto.CryptoInputStream; import org.apache.hadoop.crypto.CryptoStreamUtils; import org.apache.hadoop.fs.Seekable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A CryptoInputStream for Ozone with length. This stream is used to read @@ -31,16 +35,35 @@ public class OzoneCryptoInputStream extends CryptoInputStream implements Seekable { + private static final Logger LOG = + LoggerFactory.getLogger(OzoneCryptoInputStream.class); + private final long length; private final int bufferSize; + private final String keyName; + private final int partIndex; + + // If a read's start/ length position doesn't coincide with a Crypto buffer + // boundary, it will be adjusted as reads should happen only at the buffer + // boundaries for decryption to happen correctly. In this case, after the + // data has been read and decrypted, only the requested data should be + // returned to the client. readPositionAdjustedBy and readLengthAdjustedBy + // store these adjustment information. Before returning to client, the first + // readPositionAdjustedBy number of bytes and the last readLengthAdjustedBy + // number of bytes must be discarded. + private int readPositionAdjustedBy = 0; + private int readLengthAdjustedBy = 0; public OzoneCryptoInputStream(LengthInputStream in, - CryptoCodec codec, byte[] key, byte[] iv) throws IOException { + CryptoCodec codec, byte[] key, byte[] iv, + String keyName, int partIndex) throws IOException { super(in.getWrappedStream(), codec, key, iv); this.length = in.getLength(); // This is the buffer size used while creating the CryptoInputStream // internally this.bufferSize = CryptoStreamUtils.getBufferSize(codec.getConf()); + this.keyName = keyName; + this.partIndex = partIndex; } public long getLength() { @@ -54,4 +77,131 @@ public int getBufferSize() { public long getRemaining() throws IOException { return length - getPos(); } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + // CryptoInputStream reads hadoop.security.crypto.buffer.size number of + // bytes (default 8KB) at a time. This needs to be taken into account + // in calculating the numBytesToRead. + int numBytesToRead = getNumBytesToRead(len, (int)getRemaining(), + getBufferSize()); + int numBytesRead; + + if (readPositionAdjustedBy != 0 || readLengthAdjustedBy != 0) { + // There was some adjustment made in position and/ or length of data + // to be read to account for Crypto buffer boundary. Hence, read the + // data into a temp buffer and then copy only the requested data into + // clients buffer. + byte[] tempBuffer = new byte[numBytesToRead]; + int actualNumBytesRead = super.read(tempBuffer, 0, + numBytesToRead); + numBytesRead = actualNumBytesRead - readPositionAdjustedBy - + readLengthAdjustedBy; + + if (actualNumBytesRead != numBytesToRead) { + throw new IOException(String.format("Inconsistent read for key=%s " + + "part=%d length=%d numBytesToRead(accounting for Crypto " + + "boundaries)=%d numBytesRead(actual)=%d " + + "numBytesToBeRead(into client buffer discarding crypto " + + "boundary adjustments)=%d", + keyName, partIndex, getLength(), numBytesToRead, + actualNumBytesRead, numBytesRead)); + } + + // TODO: Byte array copies are not optimal. If there is a better and + // more optimal solution to copy only a part of read data into + // client buffer, this should be updated. + System.arraycopy(tempBuffer, readPositionAdjustedBy, b, off, + numBytesRead); + + LOG.debug("OzoneCryptoInputStream for key: {} part: {} read {} bytes " + + "instead of {} bytes to account for Crypto buffer boundary. " + + "Client buffer will be copied with read data from position {}" + + "upto position {}, discarding the extra bytes read to " + + "maintain Crypto buffer boundary limits", keyName, partIndex, + actualNumBytesRead, numBytesRead, readPositionAdjustedBy, + actualNumBytesRead - readPositionAdjustedBy); + + if (readLengthAdjustedBy > 0) { + seek(getPos() - readLengthAdjustedBy); + } + + // Reset readPositionAdjustedBy and readLengthAdjustedBy + readPositionAdjustedBy = 0; + readLengthAdjustedBy = 0; + } else { + numBytesRead = super.read(b, off, numBytesToRead); + if (numBytesRead != numBytesToRead) { + throw new IOException(String.format("Inconsistent read for key=%s " + + "part=%d length=%d numBytesToRead=%d numBytesRead=%d", + keyName, partIndex, getLength(), numBytesToRead, + numBytesRead)); + } + } + return numBytesRead; + } + + /** + * Get number of bytes to read from the current stream based on the length + * to be read, number of bytes remaining in the stream and the Crypto buffer + * size. + * Reads should be performed at the CryptoInputStream Buffer boundaries only. + * Otherwise, the decryption will be incorrect. + */ + private int getNumBytesToRead(int lenToRead, int remaining, + int cryptoBufferSize) throws IOException { + + Preconditions.checkArgument(readPositionAdjustedBy == 0); + Preconditions.checkArgument(readLengthAdjustedBy == 0); + + // Check and adjust position if required + adjustReadPosition(cryptoBufferSize); + remaining += readPositionAdjustedBy; + lenToRead += readPositionAdjustedBy; + + return adjustNumBytesToRead(lenToRead, remaining, cryptoBufferSize); + } + + /** + * Reads should be performed at the CryptoInputStream Buffer boundary size. + * Otherwise, the decryption will be incorrect. Hence, if the position is + * not at the boundary limit, we have to adjust the position and might need + * to read more data than requested. The extra data will be filtered out + * before returning to the client. + */ + private void adjustReadPosition(long cryptoBufferSize) throws IOException { + // Position of the buffer in current stream + long currentPosOfStream = getPos(); + int modulus = (int) (currentPosOfStream % cryptoBufferSize); + if (modulus != 0) { + // Adjustment required. + // Update readPositionAdjustedBy and seek to the adjusted position + readPositionAdjustedBy = modulus; + // Seek current partStream to adjusted position. We do not need to + // reset the seeked positions of other streams. + seek(currentPosOfStream - readPositionAdjustedBy); + LOG.debug("OzoneCryptoInputStream for key: {} part: {} adjusted " + + "position {} by -{} to account for Crypto buffer boundary", + keyName, partIndex, currentPosOfStream, readPositionAdjustedBy); + } + } + + /** + * If the length of data requested does not end at a Crypto Buffer + * boundary, the number of bytes to be read must be adjusted accordingly. + * The extra data will be filtered out before returning to the client. + */ + private int adjustNumBytesToRead(int lenToRead, int remaining, + int cryptoBufferSize) { + int numBytesToRead = Math.min(cryptoBufferSize, remaining); + if (lenToRead < numBytesToRead) { + // Adjustment required; Update readLengthAdjustedBy. + readLengthAdjustedBy = numBytesToRead - lenToRead; + LOG.debug("OzoneCryptoInputStream for key: {} part: {} adjusted length " + + "by +{} to account for Crypto buffer boundary", + keyName, partIndex, readLengthAdjustedBy); + } + return numBytesToRead; + } + } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 973f98ff3b9a..f3f87c52a8cb 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1865,11 +1865,13 @@ private OzoneInputStream createInputStream( final KeyProvider.KeyVersion decrypted = getDEK(feInfo); List cryptoInputStreams = new ArrayList<>(); - for (LengthInputStream lengthInputStream : lengthInputStreams) { + for (int i = 0; i < lengthInputStreams.size(); i++) { + LengthInputStream lengthInputStream = lengthInputStreams.get(i); final OzoneCryptoInputStream ozoneCryptoInputStream = new OzoneCryptoInputStream(lengthInputStream, OzoneKMSUtil.getCryptoCodec(conf, feInfo), - decrypted.getMaterial(), feInfo.getIV()); + decrypted.getMaterial(), feInfo.getIV(), + keyInfo.getKeyName(), i); cryptoInputStreams.add(ozoneCryptoInputStream); } return new MultipartCryptoKeyInputStream(keyInfo.getKeyName(), From b5ecea61defb727d2ed5eaaaf54c732dc52f6a23 Mon Sep 17 00:00:00 2001 From: Sumit Agrawal Date: Wed, 26 Oct 2022 23:41:31 +0530 Subject: [PATCH 30/48] HDDS-7349. Flaky integration test have memory leak for RatisDropwizardExports (#3858) --- .../hadoop/ozone/HddsDatanodeService.java | 2 +- .../server/http/RatisDropwizardExports.java | 19 ++++++++++++++----- .../scm/server/StorageContainerManager.java | 11 +++++++++-- .../apache/hadoop/ozone/om/OzoneManager.java | 2 +- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index d36a17855031..92dfbf237a76 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -213,7 +213,7 @@ public void start() { serviceRuntimeInfo.setStartTime(); RatisDropwizardExports. - registerRatisMetricReporters(ratisMetricsMap); + registerRatisMetricReporters(ratisMetricsMap, () -> isStopped.get()); OzoneConfiguration.activate(); HddsServerUtil.initializeMetrics(conf, "HddsDatanode"); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/RatisDropwizardExports.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/RatisDropwizardExports.java index 76e65a1d726c..50d840a7ad9a 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/RatisDropwizardExports.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/RatisDropwizardExports.java @@ -22,6 +22,7 @@ import io.prometheus.client.CollectorRegistry; import io.prometheus.client.dropwizard.DropwizardExports; import io.prometheus.client.dropwizard.samplebuilder.DefaultSampleBuilder; +import java.util.function.BooleanSupplier; import org.apache.ratis.metrics.MetricRegistries; import org.apache.ratis.metrics.MetricsReporting; import org.apache.ratis.metrics.RatisMetricRegistry; @@ -43,14 +44,15 @@ public RatisDropwizardExports(MetricRegistry registry) { } public static void registerRatisMetricReporters( - Map ratisMetricsMap) { + Map ratisMetricsMap, + BooleanSupplier checkStopped) { //All the Ratis metrics (registered from now) will be published via JMX and //via the prometheus exporter (used by the /prom servlet MetricRegistries.global() .addReporterRegistration(MetricsReporting.jmxReporter(), MetricsReporting.stopJmxReporter()); MetricRegistries.global().addReporterRegistration( - r1 -> registerDropwizard(r1, ratisMetricsMap), + r1 -> registerDropwizard(r1, ratisMetricsMap, checkStopped), r2 -> deregisterDropwizard(r2, ratisMetricsMap)); } @@ -69,12 +71,19 @@ public static void clear(Map } private static void registerDropwizard(RatisMetricRegistry registry, - Map ratisMetricsMap) { + Map ratisMetricsMap, + BooleanSupplier checkStopped) { + if (checkStopped.getAsBoolean()) { + return; + } + RatisDropwizardExports rde = new RatisDropwizardExports( registry.getDropWizardMetricRegistry()); - CollectorRegistry.defaultRegistry.register(rde); String name = registry.getMetricRegistryInfo().getName(); - ratisMetricsMap.putIfAbsent(name, rde); + if (null == ratisMetricsMap.putIfAbsent(name, rde)) { + // new rde is added for the name, so need register + CollectorRegistry.defaultRegistry.register(rde); + } } private static void deregisterDropwizard(RatisMetricRegistry registry, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 09844681ab76..0b49a243a12d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -28,6 +28,7 @@ import com.google.common.cache.RemovalListener; import com.google.protobuf.BlockingService; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.HddsConfigKeys; @@ -290,7 +291,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl // Used to keep track of pending replication and pending deletes for // container replicas. private ContainerReplicaPendingOps containerReplicaPendingOps; - + private final AtomicBoolean isStopped = new AtomicBoolean(false); + /** * Creates a new StorageContainerManager. Configuration will be * updated with information on the actual listening addresses used @@ -584,7 +586,8 @@ private void initializeSystemManagers(OzoneConfiguration conf, clusterMap = new NetworkTopologyImpl(conf); } // This needs to be done before initializing Ratis. - RatisDropwizardExports.registerRatisMetricReporters(ratisMetricsMap); + RatisDropwizardExports.registerRatisMetricReporters(ratisMetricsMap, + () -> isStopped.get()); if (configurator.getSCMHAManager() != null) { scmHAManager = configurator.getSCMHAManager(); } else { @@ -1522,6 +1525,10 @@ private void persistSCMCertificates() throws IOException { */ @Override public void stop() { + if (isStopped.getAndSet(true)) { + LOG.info("Storage Container Manager is not running."); + return; + } try { if (containerBalancer.isBalancerRunning()) { LOG.info("Stopping Container Balancer service."); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 24a81d677966..7dcb71fd66c2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -1967,7 +1967,7 @@ private void initializeRatisServer(boolean shouldBootstrap) if (omRatisServer == null) { // This needs to be done before initializing Ratis. RatisDropwizardExports. - registerRatisMetricReporters(ratisMetricsMap); + registerRatisMetricReporters(ratisMetricsMap, () -> isStopped()); omRatisServer = OzoneManagerRatisServer.newOMRatisServer( configuration, this, omNodeDetails, peerNodesMap, secConfig, certClient, shouldBootstrap); From 0652ba433b5759fa89463f75ca64311d83724d1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Oct 2022 21:48:35 +0200 Subject: [PATCH 31/48] HDDS-7422. Bump woodstox-core from 5.0.3 to 5.4.0 (#3886) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3d6b0316bc65..24873c7f3080 100644 --- a/pom.xml +++ b/pom.xml @@ -1224,7 +1224,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs com.fasterxml.woodstox woodstox-core - 5.0.3 + 5.4.0 com.fasterxml.jackson From 557d7f84ff107aa3b075fe9ed274efdcdf62a9e3 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Thu, 27 Oct 2022 02:21:02 +0100 Subject: [PATCH 32/48] HDDS-7402. Adapt CommandQueue to track the count of each queued command type (#3891) --- .../hadoop/hdds/scm/node/CommandQueue.java | 51 ++++++--- .../hadoop/hdds/scm/node/NodeManager.java | 9 ++ .../hadoop/hdds/scm/node/SCMNodeManager.java | 13 +++ .../hdds/scm/container/MockNodeManager.java | 12 +++ .../scm/container/SimpleMockNodeManager.java | 12 +++ .../hdds/scm/node/TestCommandQueue.java | 101 ++++++++++++++++++ .../hdds/scm/node/TestSCMNodeManager.java | 36 +++++++ .../testutils/ReplicationNodeManagerMock.java | 12 +++ 8 files changed, 229 insertions(+), 17 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestCommandQueue.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java index aa930251c4f1..d17411c517fb 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java @@ -19,10 +19,12 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; import org.apache.hadoop.ozone.protocol.commands.SCMCommand; import org.apache.hadoop.util.Time; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,9 +40,8 @@ * there where queued. */ public class CommandQueue { - // This list is used as default return value. - private static final List DEFAULT_LIST = new ArrayList<>(); private final Map commandMap; + private final Map> summaryMap; private final Lock lock; private long commandsInQueue; @@ -59,6 +60,7 @@ public long getCommandsInQueue() { */ public CommandQueue() { commandMap = new HashMap<>(); + summaryMap = new HashMap<>(); lock = new ReentrantLock(); commandsInQueue = 0; } @@ -71,6 +73,7 @@ public void clear() { lock.lock(); try { commandMap.clear(); + summaryMap.clear(); commandsInQueue = 0; } finally { lock.unlock(); @@ -90,6 +93,7 @@ List getCommand(final UUID datanodeUuid) { lock.lock(); try { Commands cmds = commandMap.remove(datanodeUuid); + summaryMap.remove(datanodeUuid); List cmdList = null; if (cmds != null) { cmdList = cmds.getCommands(); @@ -97,7 +101,28 @@ List getCommand(final UUID datanodeUuid) { // A post condition really. Preconditions.checkState(commandsInQueue >= 0); } - return cmds == null ? DEFAULT_LIST : cmdList; + return cmds == null ? Collections.emptyList() : cmdList; + } finally { + lock.unlock(); + } + } + + /** + * Returns the count of commands of the give type currently queued for the + * given datanode. + * @param datanodeUuid Datanode UUID. + * @param commandType The type of command for which to get the count. + * @return The currently queued command count, or zero if none are queued. + */ + public int getDatanodeCommandCount( + final UUID datanodeUuid, SCMCommandProto.Type commandType) { + lock.lock(); + try { + Map summary = summaryMap.get(datanodeUuid); + if (summary == null) { + return 0; + } + return summary.getOrDefault(commandType, 0); } finally { lock.unlock(); } @@ -113,11 +138,12 @@ public void addCommand(final UUID datanodeUuid, final SCMCommand command) { lock.lock(); try { - if (commandMap.containsKey(datanodeUuid)) { - commandMap.get(datanodeUuid).add(command); - } else { - commandMap.put(datanodeUuid, new Commands(command)); - } + commandMap.computeIfAbsent(datanodeUuid, s -> new Commands()) + .add(command); + Map summary = + summaryMap.computeIfAbsent(datanodeUuid, s -> new HashMap<>()); + summary.put(command.getType(), + summary.getOrDefault(command.getType(), 0) + 1); commandsInQueue++; } finally { lock.unlock(); @@ -141,15 +167,6 @@ private static class Commands { readTime = 0; } - /** - * Creates the object and populates with the command. - * @param command command to add to queue. - */ - Commands(SCMCommand command) { - this(); - this.add(command); - } - /** * Gets the last time the commands for this node was updated. * @return Time stamp diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java index e2f69f77db79..5514db8ecff4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java @@ -317,6 +317,15 @@ void processNodeCommandQueueReport(DatanodeDetails datanodeDetails, int getNodeQueuedCommandCount(DatanodeDetails datanodeDetails, SCMCommandProto.Type cmdType) throws NodeNotFoundException; + /** + * Get the number of commands of the given type queued in the SCM CommandQueue + * for the given datanode. + * @param dnID The UUID of the datanode. + * @param cmdType The Type of command to query the current count for. + * @return The count of commands queued, or zero if none. + */ + int getCommandQueueCount(UUID dnID, SCMCommandProto.Type cmdType); + /** * Get list of SCMCommands in the Command Queue for a particular Datanode. * @param dnID - Datanode uuid. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 62ef30d27c77..dea4f0605b18 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -729,12 +729,25 @@ public void processNodeCommandQueueReport(DatanodeDetails datanodeDetails, * @param cmdType * @return The queued count or -1 if no data has been received from the DN. */ + @Override public int getNodeQueuedCommandCount(DatanodeDetails datanodeDetails, SCMCommandProto.Type cmdType) throws NodeNotFoundException { DatanodeInfo datanodeInfo = nodeStateManager.getNode(datanodeDetails); return datanodeInfo.getCommandCount(cmdType); } + /** + * Get the number of commands of the given type queued in the SCM CommandQueue + * for the given datanode. + * @param dnID The UUID of the datanode. + * @param cmdType The Type of command to query the current count for. + * @return The count of commands queued, or zero if none. + */ + @Override + public int getCommandQueueCount(UUID dnID, SCMCommandProto.Type cmdType) { + return commandQueue.getDatanodeCommandCount(dnID, cmdType); + } + /** * Returns the aggregated node stats. * diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index 9083125f47cc..3fa14f7dd0ef 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -615,6 +615,18 @@ public int getNodeQueuedCommandCount(DatanodeDetails datanodeDetails, return -1; } + /** + * Get the number of commands of the given type queued in the SCM CommandQueue + * for the given datanode. + * @param dnID The UUID of the datanode. + * @param cmdType The Type of command to query the current count for. + * @return The count of commands queued, or zero if none. + */ + @Override + public int getCommandQueueCount(UUID dnID, SCMCommandProto.Type cmdType) { + return 0; + } + /** * Update set of containers available on a datanode. * @param uuid - DatanodeID diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java index 35ce3cb8a9d5..e14383f2daa3 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java @@ -309,6 +309,18 @@ public int getNodeQueuedCommandCount(DatanodeDetails datanodeDetails, return -1; } + /** + * Get the number of commands of the given type queued in the SCM CommandQueue + * for the given datanode. + * @param dnID The UUID of the datanode. + * @param cmdType The Type of command to query the current count for. + * @return The count of commands queued, or zero if none. + */ + @Override + public int getCommandQueueCount(UUID dnID, SCMCommandProto.Type cmdType) { + return 0; + } + @Override public List getCommandQueue(UUID dnID) { return null; diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestCommandQueue.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestCommandQueue.java new file mode 100644 index 000000000000..5c7679923665 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestCommandQueue.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand; +import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand; +import org.apache.hadoop.ozone.protocol.commands.SCMCommand; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +/** + * Test for the CommandQueue class. + */ + +public class TestCommandQueue { + + @Test + public void testSummaryUpdated() { + CommandQueue commandQueue = new CommandQueue(); + long containerID = 1; + SCMCommand closeContainerCommand = + new CloseContainerCommand(containerID, PipelineID.randomId()); + SCMCommand createPipelineCommand = + new CreatePipelineCommand(PipelineID.randomId(), + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.THREE, Collections.emptyList()); + + UUID datanode1UUID = UUID.randomUUID(); + UUID datanode2UUID = UUID.randomUUID(); + + commandQueue.addCommand(datanode1UUID, closeContainerCommand); + commandQueue.addCommand(datanode1UUID, closeContainerCommand); + commandQueue.addCommand(datanode1UUID, createPipelineCommand); + + commandQueue.addCommand(datanode2UUID, closeContainerCommand); + commandQueue.addCommand(datanode2UUID, createPipelineCommand); + commandQueue.addCommand(datanode2UUID, createPipelineCommand); + + // Check zero returned for unknown DN + Assert.assertEquals(0, commandQueue.getDatanodeCommandCount( + UUID.randomUUID(), SCMCommandProto.Type.closeContainerCommand)); + + Assert.assertEquals(2, commandQueue.getDatanodeCommandCount( + datanode1UUID, SCMCommandProto.Type.closeContainerCommand)); + Assert.assertEquals(1, commandQueue.getDatanodeCommandCount( + datanode1UUID, SCMCommandProto.Type.createPipelineCommand)); + Assert.assertEquals(0, commandQueue.getDatanodeCommandCount( + datanode1UUID, SCMCommandProto.Type.closePipelineCommand)); + + Assert.assertEquals(1, commandQueue.getDatanodeCommandCount( + datanode2UUID, SCMCommandProto.Type.closeContainerCommand)); + Assert.assertEquals(2, commandQueue.getDatanodeCommandCount( + datanode2UUID, SCMCommandProto.Type.createPipelineCommand)); + + // Ensure the counts are cleared when the commands are retrieved + List cmds = commandQueue.getCommand(datanode1UUID); + Assert.assertEquals(3, cmds.size()); + + Assert.assertEquals(0, commandQueue.getDatanodeCommandCount( + datanode1UUID, SCMCommandProto.Type.closeContainerCommand)); + Assert.assertEquals(0, commandQueue.getDatanodeCommandCount( + datanode1UUID, SCMCommandProto.Type.createPipelineCommand)); + Assert.assertEquals(0, commandQueue.getDatanodeCommandCount( + datanode1UUID, SCMCommandProto.Type.closePipelineCommand)); + + Assert.assertEquals(1, commandQueue.getDatanodeCommandCount( + datanode2UUID, SCMCommandProto.Type.closeContainerCommand)); + Assert.assertEquals(2, commandQueue.getDatanodeCommandCount( + datanode2UUID, SCMCommandProto.Type.createPipelineCommand)); + + // Ensure the commands are zeroed when the queue is cleared + commandQueue.clear(); + Assert.assertEquals(0, commandQueue.getDatanodeCommandCount( + datanode2UUID, SCMCommandProto.Type.closeContainerCommand)); + Assert.assertEquals(0, commandQueue.getDatanodeCommandCount( + datanode2UUID, SCMCommandProto.Type.createPipelineCommand)); + } + +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index a936e5652f76..66120f92fc41 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -69,6 +69,7 @@ import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils; import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; +import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand; import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; import org.apache.hadoop.ozone.protocol.commands.SCMCommand; import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; @@ -76,6 +77,7 @@ import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.ozone.test.GenericTestUtils; import org.apache.hadoop.test.PathUtils; +import org.junit.Assert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -977,6 +979,40 @@ scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), node1, SCMCommandProto.Type.closeContainerCommand)); } + @Test + public void testCommandCount() + throws AuthenticationException, IOException { + SCMNodeManager nodeManager = createNodeManager(getConf()); + + UUID datanode1 = UUID.randomUUID(); + UUID datanode2 = UUID.randomUUID(); + long containerID = 1; + + SCMCommand closeContainerCommand = + new CloseContainerCommand(containerID, PipelineID.randomId()); + SCMCommand createPipelineCommand = + new CreatePipelineCommand(PipelineID.randomId(), + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.THREE, Collections.emptyList()); + + nodeManager.onMessage( + new CommandForDatanode<>(datanode1, closeContainerCommand), null); + nodeManager.onMessage( + new CommandForDatanode<>(datanode1, closeContainerCommand), null); + nodeManager.onMessage( + new CommandForDatanode<>(datanode1, createPipelineCommand), null); + + Assert.assertEquals(2, nodeManager.getCommandQueueCount( + datanode1, SCMCommandProto.Type.closeContainerCommand)); + Assert.assertEquals(1, nodeManager.getCommandQueueCount( + datanode1, SCMCommandProto.Type.createPipelineCommand)); + Assert.assertEquals(0, nodeManager.getCommandQueueCount( + datanode1, SCMCommandProto.Type.closePipelineCommand)); + + Assert.assertEquals(0, nodeManager.getCommandQueueCount( + datanode2, SCMCommandProto.Type.closeContainerCommand)); + } + /** * Check for NPE when datanodeDetails is passed null for sendHeartbeat. * diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java index 19b5d1a4b9a0..a93792b2c92f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java @@ -451,6 +451,18 @@ public int getNodeQueuedCommandCount(DatanodeDetails datanodeDetails, return -1; } + /** + * Get the number of commands of the given type queued in the SCM CommandQueue + * for the given datanode. + * @param dnID The UUID of the datanode. + * @param cmdType The Type of command to query the current count for. + * @return The count of commands queued, or zero if none. + */ + @Override + public int getCommandQueueCount(UUID dnID, SCMCommandProto.Type cmdType) { + return commandQueue.getDatanodeCommandCount(dnID, cmdType); + } + @Override public void onMessage(CommandForDatanode commandForDatanode, EventPublisher publisher) { From 3fb0dfb464e74f13fb45f7b816ac67b71dcec9f5 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 27 Oct 2022 10:14:14 +0800 Subject: [PATCH 33/48] HDDS-7407. EC: Block allocation should not be stripped across the EC group (#3882) --- .../scm/pipeline/WritableECContainerProvider.java | 11 ++++------- .../scm/pipeline/TestWritableECContainerProvider.java | 5 ++--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java index 76e31d152414..7833f150f473 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java @@ -93,16 +93,13 @@ public WritableECContainerProvider(ConfigurationSource conf, public ContainerInfo getContainer(final long size, ECReplicationConfig repConfig, String owner, ExcludeList excludeList) throws IOException, TimeoutException { - // Bound this at a minimum of 1 byte in case a request is made for a very - // small size, which when divided by EC DataNum is zero. - long requiredSpace = Math.max(1, size / repConfig.getData()); synchronized (this) { int openPipelineCount = pipelineManager.getPipelineCount(repConfig, Pipeline.PipelineState.OPEN); if (openPipelineCount < providerConfig.getMinimumPipelines()) { try { return allocateContainer( - repConfig, requiredSpace, owner, excludeList); + repConfig, size, owner, excludeList); } catch (IOException e) { LOG.warn("Unable to allocate a container for {} with {} existing " + "containers", repConfig, openPipelineCount, e); @@ -115,7 +112,7 @@ public ContainerInfo getContainer(final long size, PipelineRequestInformation pri = PipelineRequestInformation.Builder.getBuilder() - .setSize(requiredSpace) + .setSize(size) .build(); while (existingPipelines.size() > 0) { Pipeline pipeline = @@ -129,7 +126,7 @@ public ContainerInfo getContainer(final long size, try { ContainerInfo containerInfo = getContainerFromPipeline(pipeline); if (containerInfo == null - || !containerHasSpace(containerInfo, requiredSpace)) { + || !containerHasSpace(containerInfo, size)) { // This is O(n), which isn't great if there are a lot of pipelines // and we keep finding pipelines without enough space. existingPipelines.remove(pipeline); @@ -153,7 +150,7 @@ public ContainerInfo getContainer(final long size, // allocate a new one and usePipelineManagerV2Impl.java it. try { synchronized (this) { - return allocateContainer(repConfig, requiredSpace, owner, excludeList); + return allocateContainer(repConfig, size, owner, excludeList); } } catch (IOException e) { LOG.error("Unable to allocate a container for {} after trying all " diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java index 09ae77581f17..329618870781 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java @@ -298,12 +298,11 @@ public void testNewContainerAllocatedAndPipelinesClosedIfNoSpaceInExisting() // Update all the containers to make them nearly full, but with enough space // for an EC block to be striped across them. for (ContainerInfo c : allocatedContainers) { - c.setUsedBytes(getMaxContainerSize() - 30 * 1024 * 1024); + c.setUsedBytes(getMaxContainerSize() - 90 * 1024 * 1024); } // Get a new container of size 50 and ensure it is one of the original set. - // We ask for a space of 50, but as it is stripped across the EC group it - // will actually need 50 / dataNum space + // We ask for a space of 50 MB, and will actually need 50 MB space. ContainerInfo newContainer = provider.getContainer(50 * 1024 * 1024, repConfig, OWNER, new ExcludeList()); From 901bcf2dcc5eee378e40255392057bd92383dc2a Mon Sep 17 00:00:00 2001 From: Jie Yao Date: Thu, 27 Oct 2022 15:40:56 +0800 Subject: [PATCH 34/48] HDDS-7424. Bump jetty to 9.4.49.v20220914 (#3894) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 24873c7f3080..a13d46e057b1 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs false true - 9.4.43.v20210629 + 9.4.49.v20220914 _ _ From a84baced7eb962bb4e61c65bde1974fd75774305 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Thu, 27 Oct 2022 16:58:56 +0800 Subject: [PATCH 35/48] HDDS-7220. SCM should use sub-ca certificate for token signature without HA enabled (#3752) --- .../scm/server/SCMClientProtocolServer.java | 4 +- .../scm/server/StorageContainerManager.java | 44 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 4acb109e7d53..21d179b59e45 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -763,9 +763,9 @@ public ScmInfo getScmInfo() throws IOException { // In case, there is no ratis, there is no ratis role. // This will just print the hostname with ratis port as the default // behaviour. - String adddress = scm.getSCMHANodeDetails().getLocalNodeDetails() + String address = scm.getSCMHANodeDetails().getLocalNodeDetails() .getRatisHostPortStr(); - builder.setRatisPeerRoles(Arrays.asList(adddress)); + builder.setRatisPeerRoles(Arrays.asList(address)); } return builder.build(); } catch (Exception ex) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 0b49a243a12d..5dff347fb8be 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -791,45 +791,45 @@ private void initializeCAnSecurityProtocol(OzoneConfiguration conf, final CertificateServer scmCertificateServer; final CertificateServer rootCertificateServer; + + // Start specific instance SCM CA server. + String subject = SCM_SUB_CA_PREFIX + + InetAddress.getLocalHost().getHostName(); + if (configurator.getCertificateServer() != null) { + scmCertificateServer = configurator.getCertificateServer(); + } else { + scmCertificateServer = new DefaultCAServer(subject, + scmStorageConfig.getClusterID(), scmStorageConfig.getScmId(), + certificateStore, new DefaultProfile(), + scmCertificateClient.getComponentName()); + // INTERMEDIARY_CA which issues certs to DN and OM. + scmCertificateServer.init(new SecurityConfig(configuration), + CertificateServer.CAType.INTERMEDIARY_CA); + } + // If primary SCM node Id is set it means this is a cluster which has // performed init with SCM HA version code. if (scmStorageConfig.checkPrimarySCMIdInitialized()) { - // Start specific instance SCM CA server. - String subject = SCM_SUB_CA_PREFIX + - InetAddress.getLocalHost().getHostName(); - if (configurator.getCertificateServer() != null) { - scmCertificateServer = configurator.getCertificateServer(); - } else { - scmCertificateServer = new DefaultCAServer(subject, - scmStorageConfig.getClusterID(), scmStorageConfig.getScmId(), - certificateStore, new DefaultProfile(), - scmCertificateClient.getComponentName()); - // INTERMEDIARY_CA which issues certs to DN and OM. - scmCertificateServer.init(new SecurityConfig(configuration), - CertificateServer.CAType.INTERMEDIARY_CA); - } - if (primaryScmNodeId.equals(scmStorageConfig.getScmId())) { if (configurator.getCertificateServer() != null) { rootCertificateServer = configurator.getCertificateServer(); } else { rootCertificateServer = - HASecurityUtils.initializeRootCertificateServer( - conf, certificateStore, scmStorageConfig, new DefaultCAProfile()); + HASecurityUtils.initializeRootCertificateServer(conf, + certificateStore, scmStorageConfig, new DefaultCAProfile()); } persistPrimarySCMCerts(); } else { rootCertificateServer = null; } } else { - // On a upgraded cluster primary scm nodeId will not be set as init will - // not be run again after upgrade. So for a upgraded cluster where init - // has not happened again we will have setup like before where it has - // one CA server which is issuing certificates to DN and OM. + // On an upgraded cluster primary scm nodeId will not be set as init will + // not be run again after upgrade. For an upgraded cluster, besides one + // intermediate CA server which is issuing certificates to DN and OM, + // we will have one root CA server too. rootCertificateServer = HASecurityUtils.initializeRootCertificateServer(conf, certificateStore, scmStorageConfig, new DefaultProfile()); - scmCertificateServer = rootCertificateServer; } // We need to pass getCACertificate as rootCA certificate, From 880f87cd4ea4235affbb2e0296e64c53a74cd33a Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Thu, 27 Oct 2022 11:40:24 +0100 Subject: [PATCH 36/48] HDDS-7370. Add pending commands in SCM to Datanode command count (#3867) --- .../scm/node/CommandQueueReportHandler.java | 3 +- .../hadoop/hdds/scm/node/DatanodeInfo.java | 29 +++++++++++++----- .../hadoop/hdds/scm/node/NodeManager.java | 10 +++++-- .../hadoop/hdds/scm/node/SCMNodeManager.java | 7 +++-- .../SCMDatanodeHeartbeatDispatcher.java | 20 +++++++++++-- .../hdds/scm/container/MockNodeManager.java | 4 ++- .../scm/container/SimpleMockNodeManager.java | 3 +- .../node/TestCommandQueueReportHandler.java | 30 +++++++++++++++---- .../hdds/scm/node/TestSCMNodeManager.java | 25 ++++++++++++++-- .../testutils/ReplicationNodeManagerMock.java | 3 +- 10 files changed, 108 insertions(+), 26 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueueReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueueReportHandler.java index 4a857d22028c..a6d9ea56f29e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueueReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueueReportHandler.java @@ -44,6 +44,7 @@ public void onMessage(CommandQueueReportFromDatanode queueReportFromDatanode, Preconditions.checkNotNull(dn, "QueueReport is " + "missing DatanodeDetails."); nodeManager.processNodeCommandQueueReport(dn, - queueReportFromDatanode.getReport()); + queueReportFromDatanode.getReport(), + queueReportFromDatanode.getCommandsToBeSent()); } } \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java index 2d2415c8586c..a2d8bb2dffe8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java @@ -286,11 +286,20 @@ public void setNodeStatus(NodeStatus newNodeStatus) { * Set the current command counts for this datanode, as reported in the last * heartbeat. * @param cmds Proto message containing a list of command count pairs. + * @param commandsToBeSent Summary of commands which will be sent to the DN + * as the heartbeat is processed and should be added + * to the command count. */ - public void setCommandCounts(CommandQueueReportProto cmds) { + public void setCommandCounts(CommandQueueReportProto cmds, + Map commandsToBeSent) { try { int count = cmds.getCommandCount(); + Map mutableCmds + = new HashMap<>(commandsToBeSent); lock.writeLock().lock(); + // Purge the existing counts, as each report should completely replace + // the existing counts. + commandCounts.clear(); for (int i = 0; i < count; i++) { SCMCommandProto.Type command = cmds.getCommand(i); if (command == SCMCommandProto.Type.unknownScmCommand) { @@ -305,8 +314,19 @@ public void setCommandCounts(CommandQueueReportProto cmds) { "Setting it to zero", cmdCount, this); cmdCount = 0; } + cmdCount += mutableCmds.getOrDefault(command, 0); + // Each CommandType will be in the report once only. So we remove any + // we have seen, so we can add anything the DN has not reported but + // there is a command queued for. The DNs should return a count for all + // command types even if they have a zero count, so this is really to + // handle something being wrong on the DN where it sends a spare report. + // It really should never happen. + mutableCmds.remove(command); commandCounts.put(command, cmdCount); } + // Add any counts which the DN did not report. See comment above. This + // should not happen. + commandCounts.putAll(mutableCmds); } finally { lock.writeLock().unlock(); } @@ -322,12 +342,7 @@ public void setCommandCounts(CommandQueueReportProto cmds) { public int getCommandCount(SCMCommandProto.Type cmd) { try { lock.readLock().lock(); - Integer count = commandCounts.get(cmd); - if (count == null) { - return -1; - } else { - return count.intValue(); - } + return commandCounts.getOrDefault(cmd, -1); } finally { lock.readLock().unlock(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java index 5514db8ecff4..8f72375bcd4f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java @@ -301,11 +301,15 @@ void processLayoutVersionReport(DatanodeDetails datanodeDetails, /** * Process the Command Queue Report sent from datanodes as part of the * heartbeat message. - * @param datanodeDetails - * @param commandReport + * @param datanodeDetails DatanodeDetails the report is from + * @param commandReport Command summary report from the DN when the heartbeat + * was created. + * @param commandsToBeSent Summary of command counts that will be sent to + * the Datanode as part of the current heartbeat */ void processNodeCommandQueueReport(DatanodeDetails datanodeDetails, - CommandQueueReportProto commandReport); + CommandQueueReportProto commandReport, + Map commandsToBeSent); /** * Get the number of commands of the given type queued on the datanode at the diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index dea4f0605b18..c5ec348f2a96 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -698,10 +698,12 @@ public void processLayoutVersionReport(DatanodeDetails datanodeDetails, * * @param datanodeDetails * @param commandQueueReportProto + * @param commandsToBeSent */ @Override public void processNodeCommandQueueReport(DatanodeDetails datanodeDetails, - CommandQueueReportProto commandQueueReportProto) { + CommandQueueReportProto commandQueueReportProto, + Map commandsToBeSent) { LOG.debug("Processing Command Queue Report from [datanode={}]", datanodeDetails.getHostName()); if (LOG.isTraceEnabled()) { @@ -712,7 +714,8 @@ public void processNodeCommandQueueReport(DatanodeDetails datanodeDetails, try { DatanodeInfo datanodeInfo = nodeStateManager.getNode(datanodeDetails); if (commandQueueReportProto != null) { - datanodeInfo.setCommandCounts(commandQueueReportProto); + datanodeInfo.setCommandCounts(commandQueueReportProto, + commandsToBeSent); metrics.incNumNodeCommandQueueReportProcessed(); } } catch (NodeNotFoundException e) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java index 16b5ef8fba29..1e68b92e76b9 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.protocol.proto @@ -48,7 +49,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_ACTIONS; @@ -136,9 +139,14 @@ public List dispatch(SCMHeartbeatRequestProto heartbeat) { if (heartbeat.hasCommandQueueReport()) { LOG.debug("Dispatching Queued Command Report"); + Map cmdSummary = new HashMap<>(); + for (SCMCommand c : commands) { + cmdSummary.put(c.getType(), + cmdSummary.getOrDefault(c.getType(), 0) + 1); + } eventPublisher.fireEvent(COMMAND_QUEUE_REPORT, new CommandQueueReportFromDatanode(datanodeDetails, - heartbeat.getCommandQueueReport())); + heartbeat.getCommandQueueReport(), cmdSummary)); } if (heartbeat.hasContainerReport()) { @@ -247,9 +255,17 @@ public NodeReportFromDatanode(DatanodeDetails datanodeDetails, */ public static class CommandQueueReportFromDatanode extends ReportFromDatanode { + + private final Map commandsToBeSent; public CommandQueueReportFromDatanode(DatanodeDetails datanodeDetails, - CommandQueueReportProto report) { + CommandQueueReportProto report, + Map commandsToBeSent) { super(datanodeDetails, report); + this.commandsToBeSent = commandsToBeSent; + } + + public Map getCommandsToBeSent() { + return commandsToBeSent; } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index 3fa14f7dd0ef..e1eaf251f51c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -595,10 +595,12 @@ public void processLayoutVersionReport(DatanodeDetails dnUuid, * heartbeat message. * @param datanodeDetails * @param commandReport + * @param commandsToBeSent */ @Override public void processNodeCommandQueueReport(DatanodeDetails datanodeDetails, - CommandQueueReportProto commandReport) { + CommandQueueReportProto commandReport, + Map commandsToBeSent) { // do nothing. } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java index e14383f2daa3..22e01b977093 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java @@ -293,7 +293,8 @@ public void processLayoutVersionReport(DatanodeDetails datanodeDetails, @Override public void processNodeCommandQueueReport(DatanodeDetails datanodeDetails, - CommandQueueReportProto commandReport) { + CommandQueueReportProto commandReport, + Map commandsToBeSent) { } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestCommandQueueReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestCommandQueueReportHandler.java index 63f91ed7d6a2..4c5f1cf89fbb 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestCommandQueueReportHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestCommandQueueReportHandler.java @@ -39,6 +39,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; @@ -77,35 +79,51 @@ public void resetEventCollector() throws IOException { public void testQueueReportProcessed() throws NodeNotFoundException { DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); nodeManager.register(dn, null, null); + + // Add some queued commands to be sent to the DN on this heartbeat. This + // means the real queued count will be the value reported plus the commands + // sent to the DN. + Map commandsToBeSent = new HashMap<>(); + commandsToBeSent.put(SCMCommandProto.Type.valueOf(1), 2); + commandsToBeSent.put(SCMCommandProto.Type.valueOf(2), 1); + SCMDatanodeHeartbeatDispatcher.CommandQueueReportFromDatanode - commandReport = getQueueReport(dn); + commandReport = getQueueReport(dn, commandsToBeSent); commandQueueReportHandler.onMessage(commandReport, this); int commandCount = commandReport.getReport().getCommandCount(); for (int i = 0; i < commandCount; i++) { int storedCount = nodeManager.getNodeQueuedCommandCount(dn, commandReport.getReport().getCommand(i)); - Assertions.assertEquals(commandReport.getReport().getCount(i), - storedCount); + int expectedCount = commandReport.getReport().getCount(i); + // For the first two commands, we added extra commands + if (i == 0) { + expectedCount += 2; + } + if (i == 1) { + expectedCount += 1; + } + Assertions.assertEquals(expectedCount, storedCount); Assertions.assertTrue(storedCount > 0); } } private SCMDatanodeHeartbeatDispatcher.CommandQueueReportFromDatanode - getQueueReport(DatanodeDetails dn) { + getQueueReport(DatanodeDetails dn, + Map commandsToBeSent) { CommandQueueReportProto.Builder builder = CommandQueueReportProto.newBuilder(); int i = 10; for (SCMCommandProto.Type cmd : SCMCommandProto.Type.values()) { if (cmd == SCMCommandProto.Type.unknownScmCommand) { - // Do not include the unknow command type in the message. + // Do not include the unknown command type in the message. continue; } builder.addCommand(cmd); builder.addCount(i++); } return new SCMDatanodeHeartbeatDispatcher.CommandQueueReportFromDatanode( - dn, builder.build()); + dn, builder.build(), commandsToBeSent); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 66120f92fc41..73dc7b61759c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -23,6 +23,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Set; import java.util.UUID; @@ -964,16 +965,36 @@ scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), HddsTestUtils.createRandomDatanodeAndRegister(nodeManager); verify(eventPublisher, times(1)).fireEvent(NEW_NODE, node1); + Map commandsToBeSent = new HashMap<>(); + commandsToBeSent.put(SCMCommandProto.Type.replicateContainerCommand, 3); + commandsToBeSent.put(SCMCommandProto.Type.deleteBlocksCommand, 5); + nodeManager.processNodeCommandQueueReport(node1, CommandQueueReportProto.newBuilder() .addCommand(SCMCommandProto.Type.replicateContainerCommand) .addCount(123) .addCommand(SCMCommandProto.Type.closeContainerCommand) .addCount(11) - .build()); + .build(), + commandsToBeSent); assertEquals(-1, nodeManager.getNodeQueuedCommandCount( node1, SCMCommandProto.Type.closePipelineCommand)); - assertEquals(123, nodeManager.getNodeQueuedCommandCount( + assertEquals(126, nodeManager.getNodeQueuedCommandCount( + node1, SCMCommandProto.Type.replicateContainerCommand)); + assertEquals(11, nodeManager.getNodeQueuedCommandCount( + node1, SCMCommandProto.Type.closeContainerCommand)); + assertEquals(5, nodeManager.getNodeQueuedCommandCount( + node1, SCMCommandProto.Type.deleteBlocksCommand)); + + // Send another report missing an earlier entry, and ensure it is not + // still reported as a stale value. + nodeManager.processNodeCommandQueueReport(node1, + CommandQueueReportProto.newBuilder() + .addCommand(SCMCommandProto.Type.closeContainerCommand) + .addCount(11) + .build(), + Collections.emptyMap()); + assertEquals(-1, nodeManager.getNodeQueuedCommandCount( node1, SCMCommandProto.Type.replicateContainerCommand)); assertEquals(11, nodeManager.getNodeQueuedCommandCount( node1, SCMCommandProto.Type.closeContainerCommand)); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java index a93792b2c92f..436bccb09db8 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java @@ -434,7 +434,8 @@ public void processLayoutVersionReport(DatanodeDetails dnUuid, @Override public void processNodeCommandQueueReport(DatanodeDetails datanodeDetails, - CommandQueueReportProto commandReport) { + CommandQueueReportProto commandReport, + Map commandsToBeSent) { // Do nothing. } From f6cad7c55f4edaa0fffde5b0131a65be3aee3624 Mon Sep 17 00:00:00 2001 From: Nibiru Date: Thu, 27 Oct 2022 19:23:22 +0800 Subject: [PATCH 37/48] HDDS-7328. Improve Deletion of FSO Paths (#3844) --- .../TestDirectoryDeletingServiceWithFSO.java | 107 +++++----- .../om/service/DirectoryDeletingService.java | 195 +++++++++++------- 2 files changed, 181 insertions(+), 121 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java index 1c060fe5e209..b790284cfcfe 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java @@ -41,7 +41,6 @@ import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterAll; -import org.junit.Assert; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -54,10 +53,12 @@ import java.util.concurrent.TimeoutException; import java.util.function.LongSupplier; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; -import static org.junit.Assert.fail; /** * Directory deletion service test cases. @@ -78,7 +79,7 @@ public class TestDirectoryDeletingServiceWithFSO { @BeforeAll public static void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); - conf.setInt(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, 1); + conf.setInt(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, 2000); conf.setInt(OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK, 5); conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); @@ -147,6 +148,7 @@ public void testDeleteEmptyDirectory() throws Exception { assertTableRowCount(dirTable, 2); assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0); + assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0); assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0); // Delete the appRoot, empty dir @@ -159,13 +161,14 @@ public void testDeleteEmptyDirectory() throws Exception { assertTableRowCount(dirTable, 1); assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 1); + assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0); assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0); - Assert.assertTrue(dirTable.iterator().hasNext()); - Assert.assertEquals(root.getName(), + assertTrue(dirTable.iterator().hasNext()); + assertEquals(root.getName(), dirTable.iterator().next().getValue().getName()); - Assert.assertTrue(dirDeletingService.getRunCount() > 1); + assertTrue(dirDeletingService.getRunCount() > 1); } /** @@ -211,8 +214,11 @@ public void testDeleteWithLargeSubPathsThanBatchSize() throws Exception { assertTableRowCount(dirTable, 20); assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0); + assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0); assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0); + long preRunCount = dirDeletingService.getRunCount(); + // Delete the appRoot fs.delete(appRoot, true); @@ -224,9 +230,14 @@ public void testDeleteWithLargeSubPathsThanBatchSize() throws Exception { assertTableRowCount(dirTable, 1); assertSubPathsCount(dirDeletingService::getMovedFilesCount, 15); + // 15 subDir + 3 parentDir + assertSubPathsCount(dirDeletingService::getMovedDirsCount, 18); assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 19); - Assert.assertTrue(dirDeletingService.getRunCount() > 1); + long elapsedRunCount = dirDeletingService.getRunCount() - preRunCount; + assertTrue(dirDeletingService.getRunCount() > 1); + // Ensure dir deleting speed, here provide a backup value for safe CI + assertTrue(elapsedRunCount == 8 || elapsedRunCount == 9); } @Test @@ -258,6 +269,7 @@ public void testDeleteWithMultiLevels() throws Exception { assertTableRowCount(keyTable, 3); assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0); + assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0); assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0); // Delete the rootDir, which should delete all keys. @@ -271,45 +283,10 @@ public void testDeleteWithMultiLevels() throws Exception { assertTableRowCount(dirTable, 0); assertSubPathsCount(dirDeletingService::getMovedFilesCount, 3); + assertSubPathsCount(dirDeletingService::getMovedDirsCount, 4); assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 5); - Assert.assertTrue(dirDeletingService.getRunCount() > 1); - } - - static void assertSubPathsCount(LongSupplier pathCount, long expectedCount) - throws TimeoutException, InterruptedException { - GenericTestUtils.waitFor(() -> pathCount.getAsLong() >= expectedCount, - 1000, 120000); - } - - private void assertTableRowCount(Table table, int count) - throws TimeoutException, InterruptedException { - GenericTestUtils.waitFor(() -> assertTableRowCount(count, table), 1000, - 120000); // 2 minutes - } - - private boolean assertTableRowCount(int expectedCount, - Table table) { - long count = 0L; - try { - count = cluster.getOzoneManager().getMetadataManager() - .countRowsInTable(table); - LOG.info("{} actual row count={}, expectedCount={}", table.getName(), - count, expectedCount); - } catch (IOException ex) { - fail("testDoubleBuffer failed with: " + ex); - } - return count == expectedCount; - } - - private void checkPath(Path path) { - try { - fs.getFileStatus(path); - fail("testRecursiveDelete failed"); - } catch (IOException ex) { - Assert.assertTrue(ex instanceof FileNotFoundException); - Assert.assertTrue(ex.getMessage().contains("No such file or directory")); - } + assertTrue(dirDeletingService.getRunCount() > 1); } @Test @@ -367,10 +344,11 @@ public void testDeleteFilesAndSubFiles() throws Exception { assertTableRowCount(deletedKeyTable, 0); assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0); + assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0); assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0); // verify whether KeyDeletingService has purged the keys long currentDeletedKeyCount = keyDeletingService.getDeletedKeyCount().get(); - Assert.assertEquals(prevDeletedKeyCount + 3, currentDeletedKeyCount); + assertEquals(prevDeletedKeyCount + 3, currentDeletedKeyCount); // Case-2) Delete dir, this will cleanup sub-files under the deleted dir. @@ -385,10 +363,47 @@ public void testDeleteFilesAndSubFiles() throws Exception { assertTableRowCount(deletedKeyTable, 0); assertSubPathsCount(dirDeletingService::getMovedFilesCount, 2); + assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0); assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 1); // verify whether KeyDeletingService has purged the keys currentDeletedKeyCount = keyDeletingService.getDeletedKeyCount().get(); - Assert.assertEquals(prevDeletedKeyCount + 5, currentDeletedKeyCount); + assertEquals(prevDeletedKeyCount + 5, currentDeletedKeyCount); + } + + static void assertSubPathsCount(LongSupplier pathCount, long expectedCount) + throws TimeoutException, InterruptedException { + GenericTestUtils.waitFor(() -> pathCount.getAsLong() >= expectedCount, + 1000, 120000); + } + + private void assertTableRowCount(Table table, int count) + throws TimeoutException, InterruptedException { + GenericTestUtils.waitFor(() -> assertTableRowCount(count, table), 1000, + 120000); // 2 minutes + } + + private boolean assertTableRowCount(int expectedCount, + Table table) { + long count = 0L; + try { + count = cluster.getOzoneManager().getMetadataManager() + .countRowsInTable(table); + LOG.info("{} actual row count={}, expectedCount={}", table.getName(), + count, expectedCount); + } catch (IOException ex) { + fail("testDoubleBuffer failed with: " + ex); + } + return count == expectedCount; + } + + private void checkPath(Path path) { + try { + fs.getFileStatus(path); + fail("testRecursiveDelete failed"); + } catch (IOException ex) { + assertTrue(ex instanceof FileNotFoundException); + assertTrue(ex.getMessage().contains("No such file or directory")); + } } private static BucketLayout getFSOBucketLayout() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index b22341779eef..8115b176085b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -24,20 +24,24 @@ import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.Table.KeyValue; +import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgePathRequest; import org.apache.hadoop.util.Time; import org.apache.ratis.protocol.ClientId; import org.apache.ratis.protocol.Message; import org.apache.ratis.protocol.RaftClientRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -63,10 +67,13 @@ * components of an orphan directory is visited. */ public class DirectoryDeletingService extends BackgroundService { + public static final Logger LOG = + LoggerFactory.getLogger(DirectoryDeletingService.class); private final OzoneManager ozoneManager; - private AtomicLong deletedDirsCount; - private AtomicLong deletedFilesCount; + private final AtomicLong deletedDirsCount; + private final AtomicLong movedDirsCount; + private final AtomicLong movedFilesCount; private final AtomicLong runCount; private static ClientId clientId = ClientId.randomId(); @@ -86,7 +93,8 @@ public DirectoryDeletingService(long interval, TimeUnit unit, DIR_DELETING_CORE_POOL_SIZE, serviceTimeout); this.ozoneManager = ozoneManager; this.deletedDirsCount = new AtomicLong(0); - this.deletedFilesCount = new AtomicLong(0); + this.movedDirsCount = new AtomicLong(0); + this.movedFilesCount = new AtomicLong(0); this.runCount = new AtomicLong(0); this.pathLimitPerTask = configuration .getInt(OZONE_PATH_DELETING_LIMIT_PER_TASK, @@ -125,73 +133,96 @@ public int getPriority() { @Override public BackgroundTaskResult call() throws Exception { if (shouldRun()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Running DirectoryDeletingService"); + } runCount.incrementAndGet(); - long count = pathLimitPerTask; + int dirNum = 0; + int subDirNum = 0; + int subFileNum = 0; + long remainNum = pathLimitPerTask; + List purgePathRequestList = new ArrayList<>(); + + Table.KeyValue pendingDeletedDirInfo; try { + TableIterator> + deleteTableIterator = ozoneManager.getMetadataManager(). + getDeletedDirTable().iterator(); + long startTime = Time.monotonicNow(); - // step-1) Get one pending deleted directory - Table.KeyValue pendingDeletedDirInfo = - ozoneManager.getKeyManager().getPendingDeletionDir(); - if (pendingDeletedDirInfo != null) { + while (remainNum > 0 && deleteTableIterator.hasNext()) { + pendingDeletedDirInfo = deleteTableIterator.next(); + // step-0: Get one pending deleted directory if (LOG.isDebugEnabled()) { LOG.debug("Pending deleted dir name: {}", pendingDeletedDirInfo.getValue().getKeyName()); } final String[] keys = pendingDeletedDirInfo.getKey() - .split(OM_KEY_PREFIX); + .split(OM_KEY_PREFIX); final long volumeId = Long.parseLong(keys[1]); final long bucketId = Long.parseLong(keys[2]); // step-1: get all sub directories under the deletedDir - List dirs = ozoneManager.getKeyManager() + List subDirs = ozoneManager.getKeyManager() .getPendingDeletionSubDirs(volumeId, bucketId, - pendingDeletedDirInfo.getValue(), count); - count = count - dirs.size(); - List deletedSubDirList = new ArrayList<>(); - for (OmKeyInfo dirInfo : dirs) { - deletedSubDirList.add(dirInfo); - if (LOG.isDebugEnabled()) { - LOG.debug("deleted sub dir name: {}", - dirInfo.getKeyName()); + pendingDeletedDirInfo.getValue(), remainNum); + remainNum = remainNum - subDirs.size(); + + if (LOG.isDebugEnabled()) { + for (OmKeyInfo dirInfo : subDirs) { + LOG.debug("Moved sub dir name: {}", dirInfo.getKeyName()); } } // step-2: get all sub files under the deletedDir - List purgeDeletedFiles = ozoneManager.getKeyManager() + List subFiles = ozoneManager.getKeyManager() .getPendingDeletionSubFiles(volumeId, bucketId, - pendingDeletedDirInfo.getValue(), count); - count = count - purgeDeletedFiles.size(); + pendingDeletedDirInfo.getValue(), remainNum); + remainNum = remainNum - subFiles.size(); if (LOG.isDebugEnabled()) { - for (OmKeyInfo fileInfo : purgeDeletedFiles) { - LOG.debug("deleted sub file name: {}", fileInfo.getKeyName()); + for (OmKeyInfo fileInfo : subFiles) { + LOG.debug("Moved sub file name: {}", fileInfo.getKeyName()); } } - // step-3: Since there is a boundary condition of 'numEntries' in - // each batch, check whether the sub paths count reached batch size - // limit. If count reached limit then there can be some more child - // paths to be visited and will keep the parent deleted directory - // for one more pass. - final Optional purgeDeletedDir = count > 0 ? - Optional.of(pendingDeletedDirInfo.getKey()) : - Optional.empty(); - - if (isRatisEnabled()) { - submitPurgePaths(volumeId, bucketId, purgeDeletedDir, - purgeDeletedFiles, deletedSubDirList); + // step-3: Since there is a boundary condition of 'numEntries' in + // each batch, check whether the sub paths count reached batch size + // limit. If count reached limit then there can be some more child + // paths to be visited and will keep the parent deleted directory + // for one more pass. + String purgeDeletedDir = remainNum > 0 ? + pendingDeletedDirInfo.getKey() : null; + + PurgePathRequest request = wrapPurgeRequest(volumeId, bucketId, + purgeDeletedDir, subFiles, subDirs); + purgePathRequestList.add(request); + + // Count up the purgeDeletedDir, subDirs and subFiles + if (purgeDeletedDir != null) { + dirNum++; } - // TODO: need to handle delete with non-ratis + subDirNum += subDirs.size(); + subFileNum += subFiles.size(); + } - deletedDirsCount.incrementAndGet(); - deletedFilesCount.addAndGet(purgeDeletedFiles.size()); - if (LOG.isDebugEnabled()) { - LOG.debug("Number of dirs deleted: {}, Number of files moved:" + - " {} to DeletedTable, elapsed time: {}ms", - deletedDirsCount, deletedFilesCount, - Time.monotonicNow() - startTime); - } + // TODO: need to handle delete with non-ratis + if (isRatisEnabled()) { + submitPurgePaths(purgePathRequestList); + } + + if (dirNum != 0 || subDirNum != 0 || subFileNum != 0) { + deletedDirsCount.addAndGet(dirNum); + movedDirsCount.addAndGet(subDirNum); + movedFilesCount.addAndGet(subFileNum); + LOG.info("Number of dirs deleted: {}, Number of sub-files moved:" + + " {} to DeletedTable, Number of sub-dirs moved {} to " + + "DeletedDirectoryTable, iteration elapsed: {}ms," + + " totalRunCount: {}", + dirNum, subFileNum, subDirNum, + Time.monotonicNow() - startTime, getRunCount()); } + } catch (IOException e) { LOG.error("Error while running delete directories and files " + "background task. Will retry at next run.", e); @@ -213,6 +244,16 @@ public long getDeletedDirsCount() { return deletedDirsCount.get(); } + /** + * Returns the number of sub-dirs deleted by the background service. + * + * @return Long count. + */ + @VisibleForTesting + public long getMovedDirsCount() { + return movedDirsCount.get(); + } + /** * Returns the number of files moved to DeletedTable by the background * service. @@ -221,7 +262,7 @@ public long getDeletedDirsCount() { */ @VisibleForTesting public long getMovedFilesCount() { - return deletedFilesCount.get(); + return movedFilesCount.get(); } /** @@ -234,33 +275,10 @@ public long getRunCount() { return runCount.get(); } - private int submitPurgePaths(final long volumeId, final long bucketId, - final Optional purgeDeletedDir, - final List purgeDeletedFiles, - final List markDirsAsDeleted) { - // Put all keys to be purged in a list - int deletedCount = 0; - OzoneManagerProtocolProtos.PurgePathRequest.Builder purgePathsRequest = - OzoneManagerProtocolProtos.PurgePathRequest.newBuilder(); - purgePathsRequest.setVolumeId(volumeId); - purgePathsRequest.setBucketId(bucketId); - purgeDeletedDir.ifPresent(purgePathsRequest::setDeletedDir); - for (OmKeyInfo purgeFile : purgeDeletedFiles) { - purgePathsRequest.addDeletedSubFiles( - purgeFile.getProtobuf(true, ClientVersion.CURRENT_VERSION)); - } - - // Add these directories to deletedDirTable, so that its sub-paths will be - // traversed in next iteration to ensure cleanup all sub-children. - for (OmKeyInfo dir : markDirsAsDeleted) { - purgePathsRequest.addMarkDeletedSubDirs( - dir.getProtobuf(ClientVersion.CURRENT_VERSION)); - } - + private void submitPurgePaths(List requests) { OzoneManagerProtocolProtos.PurgeDirectoriesRequest.Builder purgeDirRequest = - OzoneManagerProtocolProtos.PurgeDirectoriesRequest.newBuilder(); - purgeDirRequest.addDeletedPath(purgePathsRequest.build()); - + OzoneManagerProtocolProtos.PurgeDirectoriesRequest.newBuilder(); + purgeDirRequest.addAllDeletedPath(requests); OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() @@ -277,9 +295,36 @@ private int submitPurgePaths(final long volumeId, final long bucketId, raftClientRequest); } catch (ServiceException e) { LOG.error("PurgePaths request failed. Will retry at next run."); - return 0; } - return deletedCount; + } + + private PurgePathRequest wrapPurgeRequest(final long volumeId, + final long bucketId, + final String purgeDeletedDir, + final List purgeDeletedFiles, + final List markDirsAsDeleted) { + // Put all keys to be purged in a list + PurgePathRequest.Builder purgePathsRequest = PurgePathRequest.newBuilder(); + purgePathsRequest.setVolumeId(volumeId); + purgePathsRequest.setBucketId(bucketId); + + if (purgeDeletedDir != null) { + purgePathsRequest.setDeletedDir(purgeDeletedDir); + } + + for (OmKeyInfo purgeFile : purgeDeletedFiles) { + purgePathsRequest.addDeletedSubFiles( + purgeFile.getProtobuf(true, ClientVersion.CURRENT_VERSION)); + } + + // Add these directories to deletedDirTable, so that its sub-paths will be + // traversed in next iteration to ensure cleanup all sub-children. + for (OmKeyInfo dir : markDirsAsDeleted) { + purgePathsRequest.addMarkDeletedSubDirs( + dir.getProtobuf(ClientVersion.CURRENT_VERSION)); + } + + return purgePathsRequest.build(); } From dd67faa0b06169c38ae772a00dc795fb73ebf951 Mon Sep 17 00:00:00 2001 From: Nibiru Date: Thu, 27 Oct 2022 20:43:51 +0800 Subject: [PATCH 38/48] HDDS-7406. Remove unused the exception & improve debug log in KeyDeletingService (#3880) --- .../ozone/om/service/KeyDeletingService.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 98d3f7070d06..b0c6b7a13ade 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeletedKeys; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgeKeysRequest; @@ -57,7 +58,6 @@ import org.apache.ratis.protocol.Message; import org.apache.ratis.protocol.RaftClientRequest; import org.apache.ratis.util.Preconditions; -import org.rocksdb.RocksDBException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -178,8 +178,10 @@ public BackgroundTaskResult call() throws Exception { // OMRequest model. delCount = deleteAllKeys(results); } - LOG.debug("Number of keys deleted: {}, elapsed time: {}ms", - delCount, Time.monotonicNow() - startTime); + if (LOG.isDebugEnabled()) { + LOG.debug("Number of keys deleted: {}, elapsed time: {}ms", + delCount, Time.monotonicNow() - startTime); + } deletedKeyCount.addAndGet(delCount); } } @@ -196,13 +198,12 @@ public BackgroundTaskResult call() throws Exception { * Deletes all the keys that SCM has acknowledged and queued for delete. * * @param results DeleteBlockGroups returned by SCM. - * @throws RocksDBException on Error. * @throws IOException on Error */ private int deleteAllKeys(List results) - throws RocksDBException, IOException { - Table deletedTable = manager.getMetadataManager().getDeletedTable(); - + throws IOException { + Table deletedTable = + manager.getMetadataManager().getDeletedTable(); DBStore store = manager.getMetadataManager().getStore(); // Put all keys to delete in a single transaction and call for delete. @@ -213,7 +214,9 @@ private int deleteAllKeys(List results) // Purge key from OM DB. deletedTable.deleteWithBatch(writeBatch, result.getObjectKey()); - LOG.debug("Key {} deleted from OM DB", result.getObjectKey()); + if (LOG.isDebugEnabled()) { + LOG.debug("Key {} deleted from OM DB", result.getObjectKey()); + } deletedCount++; } } @@ -226,9 +229,7 @@ private int deleteAllKeys(List results) /** * Submits PurgeKeys request for the keys whose blocks have been deleted * by SCM. - * * @param results DeleteBlockGroups returned by SCM. - * @throws IOException on Error */ public int submitPurgeKeysRequest(List results) { Map, List> purgeKeysMapPerBucket = @@ -242,7 +243,9 @@ public int submitPurgeKeysRequest(List results) { String deletedKey = result.getObjectKey(); // Parse Volume and BucketName addToMap(purgeKeysMapPerBucket, deletedKey); - LOG.debug("Key {} set to be purged from OM DB", deletedKey); + if (LOG.isDebugEnabled()) { + LOG.debug("Key {} set to be purged from OM DB", deletedKey); + } deletedCount++; } } From 3a52215bb7e5889050ddfdcb18f969fdb583e263 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Thu, 27 Oct 2022 21:22:23 +0800 Subject: [PATCH 39/48] HDDS-1157. TestOzoneContainerWithTLS is failing with SSLHandshakeException (#3888) --- .../ozone/client/CertificateClientTestImpl.java | 11 +++++++++-- .../ozoneimpl/TestOzoneContainerWithTLS.java | 4 +--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java index 58021f30ebf8..15b6a12336dc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java @@ -53,6 +53,11 @@ public class CertificateClientTestImpl implements CertificateClient { private final X509Certificate x509Certificate; public CertificateClientTestImpl(OzoneConfiguration conf) throws Exception { + this(conf, true); + } + + public CertificateClientTestImpl(OzoneConfiguration conf, boolean rootCA) + throws Exception { securityConfig = new SecurityConfig(conf); HDDSKeyGenerator keyGen = new HDDSKeyGenerator(securityConfig.getConfiguration()); @@ -66,8 +71,10 @@ public CertificateClientTestImpl(OzoneConfiguration conf) throws Exception { .setKey(keyPair) .setSubject("localhost") .setConfiguration(config) - .setScmID("TestScmId1") - .makeCA(); + .setScmID("TestScmId1"); + if (rootCA) { + builder.makeCA(); + } X509CertificateHolder certificateHolder = null; certificateHolder = builder.build(); x509Certificate = new JcaX509CertificateConverter().getCertificate( diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java index c5a4a9339bc4..99c38152dba0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java @@ -40,7 +40,6 @@ import org.apache.ozone.test.GenericTestUtils; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -68,7 +67,6 @@ * Tests ozone containers via secure grpc/netty. */ @RunWith(Parameterized.class) -@Ignore("TODO:HDDS-1157") public class TestOzoneContainerWithTLS { private static final Logger LOG = LoggerFactory.getLogger( TestOzoneContainerWithTLS.class); @@ -122,7 +120,7 @@ public void setup() throws Exception { HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT, TimeUnit.MILLISECONDS); - caClient = new CertificateClientTestImpl(conf); + caClient = new CertificateClientTestImpl(conf, false); secretManager = new OzoneBlockTokenSecretManager(new SecurityConfig(conf), expiryTime, caClient.getCertificate(). getSerialNumber().toString()); From f0d41c5fcbd7e365936ffadf91d4ce0a47da6262 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Thu, 27 Oct 2022 20:49:39 +0530 Subject: [PATCH 40/48] HDDS-7316. Print stacktrace to identify the location of RocksObject leaks. (#3827) --- hadoop-hdds/framework/pom.xml | 5 +++ .../hdds/utils/db/managed/ManagedObject.java | 22 ++++++++++++- .../db/managed/ManagedRocksObjectUtils.java | 23 ++++++++------ .../hdds/utils/db/TestRDBStoreIterator.java | 31 +++++++++++++++++++ .../ReconContainerMetadataManagerImpl.java | 30 +++++++++--------- 5 files changed, 87 insertions(+), 24 deletions(-) diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml index 5707688626ed..270d68fddde1 100644 --- a/hadoop-hdds/framework/pom.xml +++ b/hadoop-hdds/framework/pom.xml @@ -155,6 +155,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.junit.jupiter junit-jupiter-params + + com.github.spotbugs + spotbugs-annotations + compile + diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java index 16422681e524..a5da164c8478 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java @@ -27,8 +27,15 @@ class ManagedObject implements AutoCloseable { private final T original; + private final StackTraceElement[] elements; + ManagedObject(T original) { this.original = original; + if (ManagedRocksObjectUtils.LOG.isDebugEnabled()) { + this.elements = Thread.currentThread().getStackTrace(); + } else { + this.elements = null; + } } public T get() { @@ -42,7 +49,20 @@ public void close() { @Override protected void finalize() throws Throwable { - ManagedRocksObjectUtils.assertClosed(original); + ManagedRocksObjectUtils.assertClosed(this); super.finalize(); } + + public String getStackTrace() { + if (elements != null && elements.length > 0) { + StringBuilder sb = new StringBuilder(); + for (int line = 1; line < elements.length; line++) { + sb.append(elements[line]); + sb.append("\n"); + } + return sb.toString(); + } + return ""; + } + } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java index 026044e27957..fc6d2bdd6139 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java @@ -33,20 +33,25 @@ private ManagedRocksObjectUtils() { LoggerFactory.getLogger(ManagedRocksObjectUtils.class); static void assertClosed(RocksObject rocksObject) { - ManagedRocksObjectMetrics.INSTANCE.increaseManagedObject(); - if (rocksObject.isOwningHandle()) { - ManagedRocksObjectMetrics.INSTANCE.increaseLeakObject(); - LOG.warn("{} is not closed properly", - rocksObject.getClass().getSimpleName()); - } + assertClosed(rocksObject, null); + } + + public static void assertClosed(ManagedObject object) { + assertClosed(object.get(), object.getStackTrace()); } - static void assertClosed(RocksObject rocksObject, Throwable stack) { + static void assertClosed(RocksObject rocksObject, String stackTrace) { ManagedRocksObjectMetrics.INSTANCE.increaseManagedObject(); if (rocksObject.isOwningHandle()) { ManagedRocksObjectMetrics.INSTANCE.increaseLeakObject(); - LOG.warn("{} is not closed properly", - rocksObject.getClass().getSimpleName(), stack); + String warning = String.format("%s is not closed properly", + rocksObject.getClass().getSimpleName()); + if (stackTrace != null && LOG.isDebugEnabled()) { + String debugMessage = String + .format("%n StackTrace for unclosed instance: %s", stackTrace); + warning = warning.concat(debugMessage); + } + LOG.warn(warning); } } } diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStoreIterator.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStoreIterator.java index 8ee47ec6b6f0..bd99c0e55f26 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStoreIterator.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStoreIterator.java @@ -18,7 +18,12 @@ */ package org.apache.hadoop.hdds.utils.db; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -61,6 +66,7 @@ public void setup() { rocksDBIteratorMock = mock(RocksIterator.class); managedRocksIterator = new ManagedRocksIterator(rocksDBIteratorMock); rocksTableMock = mock(RDBTable.class); + Logger.getLogger(ManagedRocksObjectUtils.class).setLevel(Level.DEBUG); } @Test @@ -293,4 +299,29 @@ public void testNormalPrefixedIterator() throws IOException { iter.close(); } + + @Test + @SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT") + public void testGetStackTrace() { + ManagedRocksIterator iterator = mock(ManagedRocksIterator.class); + RocksIterator mock = mock(RocksIterator.class); + when(iterator.get()).thenReturn(mock); + when(mock.isOwningHandle()).thenReturn(true); + ManagedRocksObjectUtils.assertClosed(iterator); + verify(iterator, times(1)).getStackTrace(); + + iterator = new ManagedRocksIterator(rocksDBIteratorMock); + + // construct the expected trace. + StackTraceElement[] traceElements = Thread.currentThread().getStackTrace(); + StringBuilder sb = new StringBuilder(); + // first 2 lines will differ. + for (int i = 2; i < traceElements.length; i++) { + sb.append(traceElements[i]); + sb.append("\n"); + } + String expectedTrace = sb.toString(); + String fromObjectInit = iterator.getStackTrace(); + Assert.assertTrue(fromObjectInit.contains(expectedTrace)); + } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java index 53aca7ade575..6c40afc0fdd9 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java @@ -613,21 +613,23 @@ public Map getContainerForKeyPrefixes( private void initializeKeyContainerTable() throws IOException { Instant start = Instant.now(); - TableIterator> iterator = containerKeyTable.iterator(); - KeyValue keyValue; - long count = 0; - while (iterator.hasNext()) { - keyValue = iterator.next(); - ContainerKeyPrefix containerKeyPrefix = keyValue.getKey(); - if (!StringUtils.isEmpty(containerKeyPrefix.getKeyPrefix()) && - containerKeyPrefix.getContainerId() != -1) { - keyContainerTable.put(containerKeyPrefix.toKeyPrefixContainer(), 1); + try (TableIterator> iterator = containerKeyTable.iterator()) { + KeyValue keyValue; + long count = 0; + while (iterator.hasNext()) { + keyValue = iterator.next(); + ContainerKeyPrefix containerKeyPrefix = keyValue.getKey(); + if (!StringUtils.isEmpty(containerKeyPrefix.getKeyPrefix()) + && containerKeyPrefix.getContainerId() != -1) { + keyContainerTable.put(containerKeyPrefix.toKeyPrefixContainer(), 1); + } + count++; } - count++; + long duration = Duration.between(start, Instant.now()).toMillis(); + LOG.info("It took {} seconds to initialized {} records" + + " to KEY_CONTAINER table", (double) duration / 1000, count); } - long duration = Duration.between(start, Instant.now()).toMillis(); - LOG.info("It took {} seconds to initialized {} records" + - " to KEY_CONTAINER table", (double) duration / 1000, count); } } From bd8a1610b30c89de5410648900a125a1eaff9d75 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Thu, 27 Oct 2022 20:54:17 +0200 Subject: [PATCH 41/48] HDDS-7421. Respect OZONE_LOGLEVEL and OZONE_ROOT_LOGGER for CLI commands (#3892) * [wip] get rid of ozone-shell.log, suppress CLI log by default, except Freon * Output by org/apache/hadoop/fs/shell/Delete * Skip assertion for now, URL contains duplicate / --- .../dev-support/bin/dist-layout-stitching | 1 - .../smoketest/basic/ozone-shell-lib.robot | 7 ++-- .../src/main/smoketest/ozonefs/ozonefs.robot | 3 +- .../shell/conf/ozone-shell-log4j.properties | 34 ------------------- hadoop-ozone/dist/src/shell/ozone/ozone | 21 ++++++++---- .../dist/src/shell/ozone/ozone-functions.sh | 3 ++ 6 files changed, 23 insertions(+), 46 deletions(-) delete mode 100644 hadoop-ozone/dist/src/shell/conf/ozone-shell-log4j.properties diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index 078824c701b1..857147d248fb 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -83,7 +83,6 @@ run cp "${ROOT}/hadoop-ozone/dist/src/shell/conf/om-audit-log4j2.properties" "et run cp "${ROOT}/hadoop-ozone/dist/src/shell/conf/dn-audit-log4j2.properties" "etc/hadoop" run cp "${ROOT}/hadoop-ozone/dist/src/shell/conf/scm-audit-log4j2.properties" "etc/hadoop" run cp "${ROOT}/hadoop-ozone/dist/src/shell/conf/s3g-audit-log4j2.properties" "etc/hadoop" -run cp "${ROOT}/hadoop-ozone/dist/src/shell/conf/ozone-shell-log4j.properties" "etc/hadoop" run cp "${ROOT}/hadoop-ozone/dist/src/shell/conf/ozone-site.xml" "etc/hadoop" run cp -f "${ROOT}/hadoop-ozone/dist/src/shell/conf/log4j.properties" "etc/hadoop" run cp "${ROOT}/hadoop-hdds/common/src/main/resources/network-topology-default.xml" "etc/hadoop" diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot index 79a0699d0527..52994f5dec34 100644 --- a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot @@ -33,7 +33,7 @@ Test ozone shell ${result} = Execute and checkrc ozone sh volume info ${protocol}${server}/${volume} 255 Should contain ${result} VOLUME_NOT_FOUND ${result} = Execute ozone sh volume create ${protocol}${server}/${volume} --space-quota 100TB --namespace-quota 100 - Should not contain ${result} Failed + Should Be Empty ${result} ${result} = Execute ozone sh volume list ${protocol}${server}/ | jq -r '.[] | select(.name=="${volume}")' Should contain ${result} creationTime ${result} = Execute ozone sh volume list | jq -r '.[] | select(.name=="${volume}")' @@ -44,7 +44,8 @@ Test ozone shell # Should Be Equal ${result} bill ${result} = Execute ozone sh volume info ${protocol}${server}/${volume} | jq -r '. | select(.name=="${volume}") | .quotaInBytes' Should Be Equal ${result} 10995116277760 - Execute ozone sh bucket create ${protocol}${server}/${volume}/bb1 --space-quota 10TB --namespace-quota 100 + ${result} = Execute ozone sh bucket create ${protocol}${server}/${volume}/bb1 --space-quota 10TB --namespace-quota 100 + Should Be Empty ${result} ${result} = Execute ozone sh bucket info ${protocol}${server}/${volume}/bb1 | jq -r '. | select(.name=="bb1") | .storageType' Should Be Equal ${result} DISK ${result} = Execute ozone sh bucket info ${protocol}${server}/${volume}/bb1 | jq -r '. | select(.name=="bb1") | .quotaInBytes' @@ -202,4 +203,4 @@ Test prefix Acls Execute ozone sh key put ${protocol}${server}/${volume}/bb1/prefix1/key1 /opt/hadoop/NOTICE.txt ${result} = Execute ozone sh key getacl ${protocol}${server}/${volume}/bb1/prefix1/key1 Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \"superuser1\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"READ\", \"WRITE\", \"READ_ACL\", \"WRITE_ACL\" - Should Match Regexp ${result} \"type\" : \"GROUP\",\n.*\"name\" : \"superuser1\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"ALL\" . \ No newline at end of file + Should Match Regexp ${result} \"type\" : \"GROUP\",\n.*\"name\" : \"superuser1\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"ALL\" . diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot index 5e03d0c17513..ccd44a9e1b5c 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot @@ -49,7 +49,8 @@ Copy from local Should Be Equal ${result} THREE Put - Execute ozone fs -put NOTICE.txt ${DEEP_URL}/PUTFILE.txt + ${result} = Execute ozone fs -put NOTICE.txt ${DEEP_URL}/PUTFILE.txt + Should Be Empty ${result} ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET} | jq -r '.[].name' Should contain ${result} PUTFILE.txt diff --git a/hadoop-ozone/dist/src/shell/conf/ozone-shell-log4j.properties b/hadoop-ozone/dist/src/shell/conf/ozone-shell-log4j.properties deleted file mode 100644 index f6935d798ea3..000000000000 --- a/hadoop-ozone/dist/src/shell/conf/ozone-shell-log4j.properties +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -hadoop.log.dir=. -hadoop.log.file=ozone-shell.log - -log4j.rootLogger=INFO,FILE - -log4j.threshold=ALL - -log4j.appender.FILE=org.apache.log4j.DailyRollingFileAppender -log4j.appender.FILE.file=${hadoop.log.dir}/${hadoop.log.file} -log4j.appender.FILE.layout=org.apache.log4j.PatternLayout -log4j.appender.FILE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{1}:%L - %m%n - -log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR -log4j.logger.org.apache.ratis.conf.ConfUtils=WARN -log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR -log4j.logger.org.apache.ratis.grpc.client.GrpcClientProtocolClient=WARN -log4j.logger.org.apache.hadoop.io.retry.RetryInvocationHandler=WARN diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index 8bd2e75b1bd8..06d46e3dead9 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone @@ -72,11 +72,6 @@ function ozonecmd_case subcmd=$1 shift - ozone_default_log4j="${OZONE_CONF_DIR}/log4j.properties" - ozone_shell_log4j="${OZONE_CONF_DIR}/ozone-shell-log4j.properties" - if [ ! -f "${ozone_shell_log4j}" ]; then - ozone_shell_log4j=${ozone_default_log4j} - fi # Add JVM parameter (org.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads=false) # for disabling netty PooledByteBufAllocator thread caches for non-netty threads. # This parameter significantly reduces GC pressure for Datanode. @@ -149,7 +144,6 @@ function ozonecmd_case sh | shell) OZONE_CLASSNAME=org.apache.hadoop.ozone.shell.OzoneShell ozone_deprecate_envvar HDFS_OM_SH_OPTS OZONE_SH_OPTS - OZONE_SH_OPTS="-Dhadoop.log.file=ozone-shell.log -Dlog4j.configuration=file:${ozone_shell_log4j} ${OZONE_SH_OPTS}" OZONE_RUN_ARTIFACT_NAME="ozone-tools" ;; s3) @@ -187,7 +181,6 @@ function ozonecmd_case ;; fs) OZONE_CLASSNAME=org.apache.hadoop.fs.ozone.OzoneFsShell - OZONE_CLIENT_OPTS="-Dhadoop.log.file=ozone-shell.log -Dlog4j.configuration=file:${ozone_shell_log4j} ${OZONE_CLIENT_OPTS}" OZONE_RUN_ARTIFACT_NAME="ozone-tools" ;; daemonlog) @@ -227,6 +220,19 @@ function ozonecmd_case esac } +## @description turn off logging for CLI by default +## @audience private +function ozone_suppress_shell_log +{ + if [[ "${OZONE_RUN_ARTIFACT_NAME}" == "ozone-tools" ]] \ + && [[ "${OZONE_CLASSNAME}" != "org.apache.hadoop.ozone.freon.Freon" ]] \ + && [[ -z "${OZONE_ORIGINAL_LOGLEVEL}" ]] \ + && [[ -z "${OZONE_ORIGINAL_ROOT_LOGGER}" ]]; then + OZONE_LOGLEVEL=OFF + OZONE_ROOT_LOGGER="${OZONE_LOGLEVEL},console" + fi +} + # load functions for dir in "${OZONE_LIBEXEC_DIR}" "${OZONE_HOME}/libexec" "${HADOOP_LIBEXEC_DIR}" "${HADOOP_HOME}/libexec" "${bin}/../libexec"; do if [[ -e "${dir}/ozone-functions.sh" ]]; then @@ -275,6 +281,7 @@ else ozonecmd_case "${OZONE_SUBCMD}" "${OZONE_SUBCMD_ARGS[@]}" fi +ozone_suppress_shell_log ozone_assemble_classpath ozone_add_client_opts diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh index 91ede8df4a68..11ae1ef6cc95 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh +++ b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh @@ -839,6 +839,9 @@ function ozone_basic_init # default policy file for service-level authorization OZONE_POLICYFILE=${OZONE_POLICYFILE:-"hadoop-policy.xml"} + OZONE_ORIGINAL_LOGLEVEL="${OZONE_LOGLEVEL:-}" + OZONE_ORIGINAL_ROOT_LOGGER="${OZONE_ROOT_LOGGER:-}" + # if for some reason the shell doesn't have $USER defined # (e.g., ssh'd in to execute a command) # let's get the effective username and use that From 30d1a4e359de6cf5d1393a2a922a1a04f5ff6a52 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Fri, 28 Oct 2022 05:16:18 +0200 Subject: [PATCH 42/48] HDDS-7420. Bump Spring framework from 5.2.20 to 5.3.23 (#3902) --- hadoop-ozone/dist/src/main/license/jar-report.txt | 10 +++++----- hadoop-ozone/pom.xml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt index 6147466757ed..f71dec825ac5 100644 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -252,11 +252,11 @@ share/ozone/lib/slf4j-api.jar share/ozone/lib/slf4j-reload4j.jar share/ozone/lib/snakeyaml.jar share/ozone/lib/snappy-java.jar -share/ozone/lib/spring-beans.RELEASE.jar -share/ozone/lib/spring-core.RELEASE.jar -share/ozone/lib/spring-jcl.RELEASE.jar -share/ozone/lib/spring-jdbc.RELEASE.jar -share/ozone/lib/spring-tx.RELEASE.jar +share/ozone/lib/spring-beans.jar +share/ozone/lib/spring-core.jar +share/ozone/lib/spring-jcl.jar +share/ozone/lib/spring-jdbc.jar +share/ozone/lib/spring-tx.jar share/ozone/lib/sqlite-jdbc.jar share/ozone/lib/stax2-api.jar share/ozone/lib/stax-ex.jar diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index c33b804ae0f1..f6f35f947295 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -26,7 +26,7 @@ apache/ozone:${project.version} - 5.2.20.RELEASE + 5.3.23 3.11.10 From cc574c6c5f3439417d44b157191f66fafb391b7e Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Fri, 28 Oct 2022 09:17:57 +0100 Subject: [PATCH 43/48] HDDS-7432. Move command summary into Commands object in CommandQueue (#3901) --- .../hadoop/hdds/scm/node/CommandQueue.java | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java index d17411c517fb..579da23008e4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java @@ -41,7 +41,6 @@ */ public class CommandQueue { private final Map commandMap; - private final Map> summaryMap; private final Lock lock; private long commandsInQueue; @@ -60,7 +59,6 @@ public long getCommandsInQueue() { */ public CommandQueue() { commandMap = new HashMap<>(); - summaryMap = new HashMap<>(); lock = new ReentrantLock(); commandsInQueue = 0; } @@ -73,7 +71,6 @@ public void clear() { lock.lock(); try { commandMap.clear(); - summaryMap.clear(); commandsInQueue = 0; } finally { lock.unlock(); @@ -93,7 +90,6 @@ List getCommand(final UUID datanodeUuid) { lock.lock(); try { Commands cmds = commandMap.remove(datanodeUuid); - summaryMap.remove(datanodeUuid); List cmdList = null; if (cmds != null) { cmdList = cmds.getCommands(); @@ -118,11 +114,11 @@ public int getDatanodeCommandCount( final UUID datanodeUuid, SCMCommandProto.Type commandType) { lock.lock(); try { - Map summary = summaryMap.get(datanodeUuid); - if (summary == null) { + Commands commands = commandMap.get(datanodeUuid); + if (commands == null) { return 0; } - return summary.getOrDefault(commandType, 0); + return commands.getCommandSummary(commandType); } finally { lock.unlock(); } @@ -140,10 +136,6 @@ public void addCommand(final UUID datanodeUuid, final SCMCommand try { commandMap.computeIfAbsent(datanodeUuid, s -> new Commands()) .add(command); - Map summary = - summaryMap.computeIfAbsent(datanodeUuid, s -> new HashMap<>()); - summary.put(command.getType(), - summary.getOrDefault(command.getType(), 0) + 1); commandsInQueue++; } finally { lock.unlock(); @@ -154,18 +146,10 @@ public void addCommand(final UUID datanodeUuid, final SCMCommand * Class that stores commands for a datanode. */ private static class Commands { - private long updateTime; - private long readTime; - private List commands; - - /** - * Constructs a Commands class. - */ - Commands() { - commands = new ArrayList<>(); - updateTime = 0; - readTime = 0; - } + private long updateTime = 0; + private long readTime = 0; + private List commands = new ArrayList<>(); + private final Map summary = new HashMap<>(); /** * Gets the last time the commands for this node was updated. @@ -190,9 +174,15 @@ public long getReadTime() { */ public void add(SCMCommand command) { this.commands.add(command); + summary.put(command.getType(), + summary.getOrDefault(command.getType(), 0) + 1); updateTime = Time.monotonicNow(); } + public int getCommandSummary(SCMCommandProto.Type commandType) { + return summary.getOrDefault(commandType, 0); + } + /** * Returns the commands for this datanode. * @return command list. @@ -200,6 +190,7 @@ public void add(SCMCommand command) { public List getCommands() { List temp = this.commands; this.commands = new ArrayList<>(); + summary.clear(); readTime = Time.monotonicNow(); return temp; } From 7394f2c2ae88f257f7b6038b4d5241979455656f Mon Sep 17 00:00:00 2001 From: Symious Date: Fri, 28 Oct 2022 18:19:30 +0800 Subject: [PATCH 44/48] HDDS-7361. Add general metrics for queues in Datanode (#3863) --- hadoop-hdds/container-service/pom.xml | 4 + .../statemachine/DatanodeQueueMetrics.java | 179 ++++++++++++++++++ .../statemachine/DatanodeStateMachine.java | 11 ++ .../common/statemachine/StateContext.java | 23 +++ .../metrics/TestDatanodeQueueMetrics.java | 111 +++++++++++ 5 files changed, 328 insertions(+) create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeQueueMetrics.java create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestDatanodeQueueMetrics.java diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml index ce01db47b597..9bfd9433bbb2 100644 --- a/hadoop-hdds/container-service/pom.xml +++ b/hadoop-hdds/container-service/pom.xml @@ -127,6 +127,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> io.netty netty-handler + + org.apache.commons + commons-text + org.mockito mockito-junit-jupiter diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeQueueMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeQueueMetrics.java new file mode 100644 index 000000000000..29eaa835ea7a --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeQueueMetrics.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.common.statemachine; + +import com.google.common.base.CaseFormat; +import org.apache.commons.text.WordUtils; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.ozone.OzoneConsts; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.hadoop.metrics2.lib.Interns.info; + +/** + * Class contains metrics related to Datanode queues. + */ +@Metrics(about = "Datanode Queue Metrics", context = OzoneConsts.OZONE) +public final class DatanodeQueueMetrics implements MetricsSource { + + private static final Logger LOG = + LoggerFactory.getLogger(DatanodeQueueMetrics.class); + + public static final String METRICS_SOURCE_NAME = + DatanodeQueueMetrics.class.getSimpleName(); + + public static final String STATE_CONTEXT_COMMAND_QUEUE_PREFIX = + "StateContextCommandQueue"; + public static final String COMMAND_DISPATCHER_QUEUE_PREFIX = + "CommandDispatcherCommandQueue"; + public static final String INCREMENTAL_REPORT_QUEUE_PREFIX = + "IncrementalReportQueue"; + public static final String CONTAINER_ACTION_QUEUE_PREFIX = + "ContainerActionQueue"; + public static final String PIPELINE_ACTION_QUEUE_PREFIX = + "PipelineActionQueue"; + + private MetricsRegistry registry; + + private DatanodeStateMachine datanodeStateMachine; + private static DatanodeQueueMetrics instance; + + private Map stateContextCommandQueueMap; + private Map commandDispatcherQueueMap; + private Map incrementalReportsQueueMap; + private Map containerActionQueueMap; + private Map pipelineActionQueueMap; + + public DatanodeQueueMetrics(DatanodeStateMachine datanodeStateMachine) { + this.registry = new MetricsRegistry(METRICS_SOURCE_NAME); + this.datanodeStateMachine = datanodeStateMachine; + + initializeQueues(); + } + + public static synchronized DatanodeQueueMetrics create(DatanodeStateMachine + datanodeStateMachine) { + if (instance != null) { + return instance; + } + instance = DefaultMetricsSystem.instance().register(METRICS_SOURCE_NAME, + "Queue metrics in Datanode", + new DatanodeQueueMetrics(datanodeStateMachine)); + return instance; + } + + private void initializeQueues() { + // Add queue from StateContext.commandQueue + stateContextCommandQueueMap = new HashMap<>(); + for (SCMCommandProto.Type type: SCMCommandProto.Type.values()) { + stateContextCommandQueueMap.put(type, getMetricsInfo( + STATE_CONTEXT_COMMAND_QUEUE_PREFIX, String.valueOf(type))); + } + + // Add queue from DatanodeStateMachine.commandDispatcher + commandDispatcherQueueMap = new HashMap<>(); + for (SCMCommandProto.Type type: SCMCommandProto.Type.values()) { + commandDispatcherQueueMap.put(type, getMetricsInfo( + COMMAND_DISPATCHER_QUEUE_PREFIX, String.valueOf(type))); + } + + // Initialize queue for StateContext.incrementalReportQueue, + // containerActionQueue, pipelineActionQueue + incrementalReportsQueueMap = new HashMap<>(); + containerActionQueueMap = new HashMap<>(); + pipelineActionQueueMap = new HashMap<>(); + } + + @Override + public void getMetrics(MetricsCollector collector, boolean b) { + MetricsRecordBuilder builder = collector.addRecord(METRICS_SOURCE_NAME); + + Map tmpMap = + datanodeStateMachine.getContext().getCommandQueueSummary(); + for (Map.Entry entry: + stateContextCommandQueueMap.entrySet()) { + builder.addGauge(entry.getValue(), + (long) tmpMap.getOrDefault(entry.getKey(), 0)); + } + + tmpMap = datanodeStateMachine.getCommandDispatcher() + .getQueuedCommandCount(); + for (Map.Entry entry: + commandDispatcherQueueMap.entrySet()) { + builder.addGauge(entry.getValue(), + (long) tmpMap.getOrDefault(entry.getKey(), 0)); + } + + for (Map.Entry entry: + incrementalReportsQueueMap.entrySet()) { + builder.addGauge(entry.getValue(), + datanodeStateMachine.getContext() + .getIncrementalReportQueueSize().getOrDefault(entry.getKey(), 0)); + } + for (Map.Entry entry: + containerActionQueueMap.entrySet()) { + builder.addGauge(entry.getValue(), + datanodeStateMachine.getContext() + .getContainerActionQueueSize().getOrDefault(entry.getKey(), 0)); + } + for (Map.Entry entry: + pipelineActionQueueMap.entrySet()) { + builder.addGauge(entry.getValue(), + datanodeStateMachine.getContext().getPipelineActionQueueSize() + .getOrDefault(entry.getKey(), 0)); + } + } + + public static synchronized void unRegister() { + instance = null; + DefaultMetricsSystem.instance().unregisterSource(METRICS_SOURCE_NAME); + } + + public void addEndpoint(InetSocketAddress endpoint) { + incrementalReportsQueueMap.computeIfAbsent(endpoint, + k -> getMetricsInfo(INCREMENTAL_REPORT_QUEUE_PREFIX, + CaseFormat.UPPER_UNDERSCORE + .to(CaseFormat.UPPER_CAMEL, k.getHostName()))); + containerActionQueueMap.computeIfAbsent(endpoint, + k -> getMetricsInfo(CONTAINER_ACTION_QUEUE_PREFIX, + CaseFormat.UPPER_UNDERSCORE + .to(CaseFormat.UPPER_CAMEL, k.getHostName()))); + pipelineActionQueueMap.computeIfAbsent(endpoint, + k -> getMetricsInfo(PIPELINE_ACTION_QUEUE_PREFIX, + CaseFormat.UPPER_UNDERSCORE + .to(CaseFormat.UPPER_CAMEL, k.getHostName()))); + } + + private MetricsInfo getMetricsInfo(String prefix, String metricName) { + String metric = prefix + WordUtils.capitalize(metricName) + "Size"; + String description = "Queue size of " + metricName + " from " + prefix; + return info(metric, description); + } +} \ No newline at end of file diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index 31864e2868c8..019577a07ef7 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -119,6 +119,7 @@ public class DatanodeStateMachine implements Closeable { private final ReplicationSupervisorMetrics replicationSupervisorMetrics; private final ECReconstructionMetrics ecReconstructionMetrics; + private final DatanodeQueueMetrics queueMetrics; /** * Constructs a datanode state machine. * @param datanodeDetails - DatanodeDetails used to identify a datanode @@ -225,6 +226,8 @@ public DatanodeStateMachine(DatanodeDetails datanodeDetails, .addPublisherFor(PipelineReportsProto.class) .addPublisherFor(CRLStatusReport.class) .build(); + + queueMetrics = DatanodeQueueMetrics.create(this); } private int getEndPointTaskThreadPoolSize() { @@ -418,6 +421,10 @@ public void close() throws IOException { if (commandDispatcher != null) { commandDispatcher.stop(); } + + if (queueMetrics != null) { + DatanodeQueueMetrics.unRegister(); + } } /** @@ -710,4 +717,8 @@ public UpgradeFinalizer getUpgradeFinalizer() { public ConfigurationSource getConf() { return conf; } + + public DatanodeQueueMetrics getQueueMetrics() { + return queueMetrics; + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java index 7a7f7926138e..eb44b3b8b630 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java @@ -40,6 +40,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; +import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.Descriptors.Descriptor; @@ -886,6 +887,9 @@ public void addEndpoint(InetSocketAddress endpoint) { mp.putIfAbsent(e, new AtomicBoolean(true)); }); this.isFullReportReadyToBeSent.putIfAbsent(endpoint, mp); + if (getQueueMetrics() != null) { + getQueueMetrics().addEndpoint(endpoint); + } } } @@ -919,4 +923,23 @@ public void configureReconHeartbeatFrequency() { public long getReconHeartbeatFrequency() { return reconHeartbeatFrequency.get(); } + + public Map getPipelineActionQueueSize() { + return pipelineActions.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().size())); + } + + public Map getContainerActionQueueSize() { + return containerActions.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().size())); + } + + public Map getIncrementalReportQueueSize() { + return incrementalReportsQueue.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().size())); + } + + public DatanodeQueueMetrics getQueueMetrics() { + return parentDatanodeStateMachine.getQueueMetrics(); + } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestDatanodeQueueMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestDatanodeQueueMetrics.java new file mode 100644 index 000000000000..b41bd15a970a --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestDatanodeQueueMetrics.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.metrics; + +import org.apache.commons.text.WordUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeQueueMetrics; +import org.junit.Rule; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.UUID; + +import static org.apache.hadoop.ozone.container.common.statemachine.DatanodeQueueMetrics.COMMAND_DISPATCHER_QUEUE_PREFIX; +import static org.apache.hadoop.ozone.container.common.statemachine.DatanodeQueueMetrics.STATE_CONTEXT_COMMAND_QUEUE_PREFIX; +import static org.apache.hadoop.test.MetricsAsserts.getLongGauge; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; + +/** + * Test for queue metrics of datanodes. + */ +public class TestDatanodeQueueMetrics { + + private MiniOzoneHAClusterImpl cluster = null; + private OzoneConfiguration conf; + private String clusterId; + private String scmId; + private String omServiceId; + private static int numOfOMs = 3; + private String scmServiceId; + private static int numOfSCMs = 3; + + private static final Logger LOG = LoggerFactory + .getLogger(TestDatanodeQueueMetrics.class); + + @Rule + public Timeout timeout = new Timeout(300_000); + + /** + * Create a MiniDFSCluster for testing. + *

+ * Ozone is made active by setting OZONE_ENABLED = true + * + * @throws IOException + */ + @BeforeEach + public void init() throws Exception { + conf = new OzoneConfiguration(); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s"); + clusterId = UUID.randomUUID().toString(); + scmId = UUID.randomUUID().toString(); + omServiceId = "om-service-test1"; + scmServiceId = "scm-service-test1"; + cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf) + .setClusterId(clusterId) + .setScmId(scmId) + .setOMServiceId(omServiceId) + .setSCMServiceId(scmServiceId) + .setNumOfStorageContainerManagers(numOfSCMs) + .setNumOfOzoneManagers(numOfOMs) + .setNumDatanodes(1) + .build(); + cluster.waitForClusterToBeReady(); + } + /** + * Set a timeout for each test. + */ + + @Test + public void testQueueMetrics() { + + for (SCMCommandProto.Type type: SCMCommandProto.Type.values()) { + Assertions.assertTrue( + getGauge(STATE_CONTEXT_COMMAND_QUEUE_PREFIX + + WordUtils.capitalize(String.valueOf(type)) + "Size") >= 0); + Assertions.assertTrue( + getGauge(COMMAND_DISPATCHER_QUEUE_PREFIX + + WordUtils.capitalize(String.valueOf(type)) + "Size") >= 0); + } + + } + + private long getGauge(String metricName) { + return getLongGauge(metricName, + getMetrics(DatanodeQueueMetrics.METRICS_SOURCE_NAME)); + } +} \ No newline at end of file From c187a8ff655cc33464ea8c20bc0541c3cfb8afd0 Mon Sep 17 00:00:00 2001 From: Jie Yao Date: Fri, 28 Oct 2022 20:11:44 +0800 Subject: [PATCH 45/48] HDDS-7384. EC: ReplicationManager - implement deleting container handler (#3881) --- .../replication/ReplicationManager.java | 20 ++ .../health/DeletingContainerHandler.java | 94 +++++++ .../health/EmptyContainerHandler.java | 19 +- .../health/TestDeletingContainerHandler.java | 238 ++++++++++++++++++ .../health/TestEmptyContainerHandler.java | 20 +- 5 files changed, 363 insertions(+), 28 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java index 77142848b692..b7befbc720a2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport; import org.apache.hadoop.hdds.scm.container.replication.health.ClosedWithMismatchedReplicasHandler; import org.apache.hadoop.hdds.scm.container.replication.health.ClosingContainerHandler; +import org.apache.hadoop.hdds.scm.container.replication.health.DeletingContainerHandler; import org.apache.hadoop.hdds.scm.container.replication.health.ECReplicationCheckHandler; import org.apache.hadoop.hdds.scm.container.replication.health.HealthCheck; import org.apache.hadoop.hdds.scm.container.replication.health.OpenContainerHandler; @@ -51,6 +52,7 @@ import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.DeleteContainerCommand; @@ -220,6 +222,7 @@ public ReplicationManager(final ConfigurationSource conf, .addNext(new ClosingContainerHandler(this)) .addNext(new QuasiClosedContainerHandler(this)) .addNext(new ClosedWithMismatchedReplicasHandler(this)) + .addNext(new DeletingContainerHandler(this)) .addNext(ecReplicationCheckHandler) .addNext(ratisReplicationCheckHandler); start(); @@ -403,6 +406,23 @@ public void sendDeleteCommand(final ContainerInfo container, int replicaIndex, } } + /** + * update container state. + * + * @param containerID Container to be updated + * @param event the event to update the container + */ + public void updateContainerState(ContainerID containerID, + HddsProtos.LifeCycleEvent event) { + try { + containerManager.updateContainerState(containerID, event); + } catch (IOException | InvalidStateTransitionException | + TimeoutException e) { + LOG.error("Failed to update the state of container {}, update Event {}", + containerID, event, e); + } + } + /** * Add an under replicated container back to the queue if it was unable to diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java new file mode 100644 index 000000000000..a31e5ecb161f --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.replication.health; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.replication.ContainerCheckRequest; +import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaOp; +import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager; +import org.apache.ratis.protocol.exceptions.NotLeaderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Class used in Replication Manager to handle the + * replicas of containers in DELETING State. + */ +public class DeletingContainerHandler extends AbstractCheck { + private final ReplicationManager replicationManager; + + public static final Logger LOG = + LoggerFactory.getLogger(DeletingContainerHandler.class); + + public DeletingContainerHandler(ReplicationManager replicationManager) { + this.replicationManager = replicationManager; + } + + /** + * If the number of replicas of the container is 0, change the state + * of the container to Deleted, otherwise resend delete command if needed. + * @param request ContainerCheckRequest object representing the container + * @return false if the specified container is not in DELETING state, + * otherwise true. + */ + @Override + public boolean handle(ContainerCheckRequest request) { + ContainerInfo containerInfo = request.getContainerInfo(); + ContainerID cID = containerInfo.containerID(); + HddsProtos.LifeCycleState containerState = containerInfo.getState(); + + if (containerState == HddsProtos.LifeCycleState.DELETED) { + return true; + } + + if (containerState != HddsProtos.LifeCycleState.DELETING) { + return false; + } + + if (request.getContainerReplicas().size() == 0) { + replicationManager.updateContainerState( + cID, HddsProtos.LifeCycleEvent.CLEANUP); + return true; + } + + Set pendingDelete = request.getPendingOps().stream() + .filter(o -> o.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) + .map(o -> o.getTarget()).collect(Collectors.toSet()); + //resend deleteCommand if needed + request.getContainerReplicas().stream() + .filter(r -> !pendingDelete.contains(r.getDatanodeDetails())) + .forEach(rp -> { + try { + replicationManager.sendDeleteCommand( + containerInfo, rp.getReplicaIndex(), rp.getDatanodeDetails()); + } catch (NotLeaderException e) { + LOG.warn("Failed to delete empty replica with index {} for " + + "container {} on datanode {}", rp.getReplicaIndex(), + cID, rp.getDatanodeDetails().getUuidString(), e); + } + }); + return true; + } +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java index 47ac50ce3a02..19c8d0a93aaa 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java @@ -20,20 +20,16 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.container.ContainerInfo; -import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport; import org.apache.hadoop.hdds.scm.container.replication.ContainerCheckRequest; import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager; -import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; import org.apache.ratis.protocol.exceptions.NotLeaderException; import org.apache.ratis.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.Set; -import java.util.concurrent.TimeoutException; /** * This handler deletes a container if it's closed and empty (0 key count) @@ -44,12 +40,9 @@ public class EmptyContainerHandler extends AbstractCheck { LoggerFactory.getLogger(EmptyContainerHandler.class); private final ReplicationManager replicationManager; - private final ContainerManager containerManager; - public EmptyContainerHandler(ReplicationManager replicationManager, - ContainerManager containerManager) { + public EmptyContainerHandler(ReplicationManager replicationManager) { this.replicationManager = replicationManager; - this.containerManager = containerManager; } /** @@ -72,14 +65,8 @@ public boolean handle(ContainerCheckRequest request) { deleteContainerReplicas(containerInfo, replicas); // Update the container's state - try { - containerManager.updateContainerState(containerInfo.containerID(), - HddsProtos.LifeCycleEvent.DELETE); - } catch (IOException | InvalidStateTransitionException | - TimeoutException e) { - LOG.error("Failed to delete empty container {}", - request.getContainerInfo(), e); - } + replicationManager.updateContainerState( + containerInfo.containerID(), HddsProtos.LifeCycleEvent.DELETE); return true; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java new file mode 100644 index 000000000000..e2a147ffd2f6 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.replication.health; + +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport; +import org.apache.hadoop.hdds.scm.container.replication.ContainerCheckRequest; +import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaOp; +import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager; +import org.apache.hadoop.hdds.scm.container.replication.ReplicationTestUtil; +import org.apache.ratis.protocol.exceptions.NotLeaderException; +import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.DELETING; + +/** + * Tests for {@link DeletingContainerHandler}. + */ +public class TestDeletingContainerHandler { + private ReplicationManager replicationManager; + private DeletingContainerHandler deletingContainerHandler; + private ECReplicationConfig ecReplicationConfig; + private RatisReplicationConfig ratisReplicationConfig; + + + @BeforeEach + public void setup() throws IOException { + + ecReplicationConfig = new ECReplicationConfig(3, 2); + ratisReplicationConfig = RatisReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.THREE); + replicationManager = Mockito.mock(ReplicationManager.class); + + Mockito.doNothing().when(replicationManager) + .updateContainerState(Mockito.any(ContainerID.class), + Mockito.any(HddsProtos.LifeCycleEvent.class)); + + deletingContainerHandler = + new DeletingContainerHandler(replicationManager); + } + + /** + * If a container is not in Deleting state, it should not be handled by + * DeletingContainerHandler. It should return false so the request can be + * passed to the next handler in the chain. + */ + @Test + public void testNonDeletingContainerReturnsFalse() { + ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( + ecReplicationConfig, 1, CLOSED); + Set containerReplicas = ReplicationTestUtil + .createReplicas(containerInfo.containerID(), + ContainerReplicaProto.State.CLOSING, 1, 2, 3, 4, 5); + + ContainerCheckRequest request = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.EMPTY_LIST) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .build(); + + Assert.assertFalse(deletingContainerHandler.handle(request)); + } + + @Test + public void testNonDeletingRatisContainerReturnsFalse() { + ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( + ratisReplicationConfig, 1, CLOSED); + Set containerReplicas = ReplicationTestUtil + .createReplicas(containerInfo.containerID(), + ContainerReplicaProto.State.CLOSING, 0, 0, 0); + + ContainerCheckRequest request = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.EMPTY_LIST) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .build(); + + Assert.assertFalse(deletingContainerHandler.handle(request)); + } + + /** + * If a container is in Deleting state and no replica exists, + * change the state of the container to DELETED. + */ + @Test + public void testCleanupIfNoReplicaExist() { + //ratis container + cleanupIfNoReplicaExist(RatisReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.THREE), 1); + + //ec container + //since updateContainerState is called once when testing + //ratis container, so here should be 1+1 = 2 times + cleanupIfNoReplicaExist(ecReplicationConfig, 2); + } + + + private void cleanupIfNoReplicaExist( + ReplicationConfig replicationConfig, int times) { + ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( + replicationConfig, 1, DELETING); + + Set containerReplicas = new HashSet<>(); + ContainerCheckRequest request = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.EMPTY_LIST) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .build(); + + Assert.assertTrue(deletingContainerHandler.handle(request)); + Mockito.verify(replicationManager, Mockito.times(times)) + .updateContainerState(Mockito.any(ContainerID.class), + Mockito.any(HddsProtos.LifeCycleEvent.class)); + } + + /** + * If a container is in Deleting state , some replicas exist and + * for each replica there is a pending delete, then do nothing. + */ + @Test + public void testNoNeedResendDeleteCommand() throws NotLeaderException { + //ratis container + ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( + ratisReplicationConfig, 1, DELETING); + Set containerReplicas = ReplicationTestUtil + .createReplicas(containerInfo.containerID(), + ContainerReplicaProto.State.CLOSED, 0, 0, 0); + List pendingOps = new ArrayList<>(); + containerReplicas.forEach(r -> pendingOps.add( + ContainerReplicaOp.create(ContainerReplicaOp.PendingOpType.DELETE, + r.getDatanodeDetails(), r.getReplicaIndex()))); + verifyDeleteCommandCount(containerInfo, containerReplicas, pendingOps, 0); + + //EC container + containerInfo = ReplicationTestUtil.createContainerInfo( + ecReplicationConfig, 1, DELETING); + containerReplicas = ReplicationTestUtil + .createReplicas(containerInfo.containerID(), + ContainerReplicaProto.State.CLOSED, 1, 2, 3, 4, 5); + pendingOps.clear(); + containerReplicas.forEach(r -> pendingOps.add( + ContainerReplicaOp.create(ContainerReplicaOp.PendingOpType.DELETE, + r.getDatanodeDetails(), r.getReplicaIndex()))); + verifyDeleteCommandCount(containerInfo, containerReplicas, pendingOps, 0); + + } + + /** + * If a container is in Deleting state , some replicas exist and + * for some replica there is no pending delete, then resending delete + * command. + */ + @Test + public void testResendDeleteCommand() throws NotLeaderException { + //ratis container + ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( + ratisReplicationConfig, 1, DELETING); + Set containerReplicas = ReplicationTestUtil + .createReplicas(containerInfo.containerID(), + ContainerReplicaProto.State.CLOSED, 0, 0, 0); + List pendingOps = new ArrayList<>(); + containerReplicas.stream().limit(2).forEach(replica -> pendingOps.add( + ContainerReplicaOp.create(ContainerReplicaOp.PendingOpType.DELETE, + replica.getDatanodeDetails(), replica.getReplicaIndex()))); + verifyDeleteCommandCount(containerInfo, containerReplicas, pendingOps, 1); + + //EC container + containerInfo = ReplicationTestUtil.createContainerInfo( + ecReplicationConfig, 1, DELETING); + containerReplicas = ReplicationTestUtil + .createReplicas(containerInfo.containerID(), + ContainerReplicaProto.State.CLOSED, 1, 2, 3, 4, 5); + pendingOps.clear(); + containerReplicas.stream().limit(3).forEach(replica -> pendingOps.add( + ContainerReplicaOp.create(ContainerReplicaOp.PendingOpType.DELETE, + replica.getDatanodeDetails(), replica.getReplicaIndex()))); + //since one delete command is end when testing ratis container, so + //here should be 1+2 = 3 times + verifyDeleteCommandCount(containerInfo, containerReplicas, pendingOps, 3); + + } + + private void verifyDeleteCommandCount(ContainerInfo containerInfo, + Set containerReplicas, + List pendingOps, + int times) throws NotLeaderException { + ContainerCheckRequest request = new ContainerCheckRequest.Builder() + .setPendingOps(pendingOps) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .build(); + + Assert.assertTrue(deletingContainerHandler.handle(request)); + + Mockito.verify(replicationManager, Mockito.times(times)) + .sendDeleteCommand(Mockito.any(ContainerInfo.class), Mockito.anyInt(), + Mockito.any(DatanodeDetails.class)); + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java index ac746900f544..bac90b4c390e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; -import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport; import org.apache.hadoop.hdds.scm.container.replication.ContainerCheckRequest; @@ -50,7 +49,6 @@ */ public class TestEmptyContainerHandler { private ReplicationManager replicationManager; - private ContainerManager containerManager; private EmptyContainerHandler emptyContainerHandler; private ECReplicationConfig ecReplicationConfig; private RatisReplicationConfig ratisReplicationConfig; @@ -62,10 +60,8 @@ public void setup() ratisReplicationConfig = RatisReplicationConfig.getInstance( HddsProtos.ReplicationFactor.THREE); replicationManager = Mockito.mock(ReplicationManager.class); - containerManager = Mockito.mock(ContainerManager.class); - emptyContainerHandler = - new EmptyContainerHandler(replicationManager, containerManager); + new EmptyContainerHandler(replicationManager); } /** @@ -74,7 +70,7 @@ public void setup() */ @Test public void testEmptyAndClosedECContainerReturnsTrue() - throws InvalidStateTransitionException, IOException, TimeoutException { + throws IOException { long keyCount = 0L; long bytesUsed = 123L; ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( @@ -96,7 +92,7 @@ public void testEmptyAndClosedECContainerReturnsTrue() @Test public void testEmptyAndClosedRatisContainerReturnsTrue() - throws InvalidStateTransitionException, IOException, TimeoutException { + throws IOException { long keyCount = 0L; long bytesUsed = 123L; ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( @@ -122,7 +118,7 @@ public void testEmptyAndClosedRatisContainerReturnsTrue() */ @Test public void testEmptyAndNonClosedECContainerReturnsFalse() - throws InvalidStateTransitionException, IOException, TimeoutException { + throws IOException { long keyCount = 0L; long bytesUsed = 123L; ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( @@ -150,7 +146,7 @@ public void testEmptyAndNonClosedECContainerReturnsFalse() */ @Test public void testNonEmptyRatisContainerReturnsFalse() - throws InvalidStateTransitionException, IOException, TimeoutException { + throws IOException { long keyCount = 5L; long bytesUsed = 123L; ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( @@ -177,7 +173,7 @@ public void testNonEmptyRatisContainerReturnsFalse() */ @Test public void testEmptyECContainerWithNonEmptyReplicaReturnsFalse() - throws InvalidStateTransitionException, IOException, TimeoutException { + throws IOException { ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( ecReplicationConfig, 1, CLOSED, 0L, 0L); Set containerReplicas = ReplicationTestUtil @@ -215,7 +211,7 @@ public void testEmptyECContainerWithNonEmptyReplicaReturnsFalse() */ private void assertAndVerify(ContainerCheckRequest request, boolean assertion, int times, long numEmptyExpected) - throws IOException, InvalidStateTransitionException, TimeoutException { + throws IOException { Assertions.assertEquals(assertion, emptyContainerHandler.handle(request)); Mockito.verify(replicationManager, Mockito.times(times)) .sendDeleteCommand(Mockito.any(ContainerInfo.class), Mockito.anyInt(), @@ -224,7 +220,7 @@ private void assertAndVerify(ContainerCheckRequest request, ReplicationManagerReport.HealthState.EMPTY)); if (times > 0) { - Mockito.verify(containerManager, Mockito.times(1)) + Mockito.verify(replicationManager, Mockito.times(1)) .updateContainerState(Mockito.any(ContainerID.class), Mockito.any(HddsProtos.LifeCycleEvent.class)); } From 3294d28cc97edb3cc0eb92c0ed47a5bd69d82f90 Mon Sep 17 00:00:00 2001 From: Duong Nguyen Date: Fri, 28 Oct 2022 11:11:43 -0700 Subject: [PATCH 46/48] HDDS-7231. Integrate the GetKeyInfo API to key read flows (#3800) --- .../hdds/scm/storage/BlockInputStream.java | 45 +- .../hdds/scm/storage/ChunkInputStream.java | 19 +- .../scm/storage/TestBlockInputStream.java | 133 ++-- .../hadoop/ozone/OzoneManagerVersion.java | 2 + .../hadoop/ozone/client/rpc/RpcClient.java | 121 ++-- .../hadoop/ozone/client/MockOmTransport.java | 14 + .../om/protocol/OzoneManagerProtocol.java | 2 + .../client/rpc/read/TestKeyInputStream.java | 4 +- .../om/TestOmContainerLocationCache.java | 674 ++++++++++++++++++ 9 files changed, 845 insertions(+), 169 deletions(-) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java index a6af1ba3f784..4626f580f5e4 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java @@ -46,6 +46,7 @@ import org.apache.hadoop.security.token.Token; import com.google.common.annotations.VisibleForTesting; +import org.apache.ratis.thirdparty.io.grpc.Status; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -140,11 +141,12 @@ public synchronized void initialize() throws IOException { IOException catchEx = null; do { try { - // If refresh returns new pipeline, retry with it. - // If we get IOException due to connectivity issue, - // retry according to retry policy. chunks = getChunkInfos(); break; + // If we get a StorageContainerException or an IOException due to + // datanodes are not reachable, refresh to get the latest pipeline + // info and retry. + // Otherwise, just retry according to the retry policy. } catch (SCMSecurityException ex) { throw ex; } catch (StorageContainerException ex) { @@ -152,6 +154,9 @@ public synchronized void initialize() throws IOException { catchEx = ex; } catch (IOException ex) { LOG.debug("Retry to get chunk info fail", ex); + if (isConnectivityIssue(ex)) { + refreshPipeline(ex); + } catchEx = ex; } } while (shouldRetryRead(catchEx)); @@ -187,19 +192,19 @@ public synchronized void initialize() throws IOException { } } + /** + * Check if this exception is because datanodes are not reachable. + */ + private boolean isConnectivityIssue(IOException ex) { + return Status.fromThrowable(ex).getCode() == Status.UNAVAILABLE.getCode(); + } + private void refreshPipeline(IOException cause) throws IOException { LOG.info("Unable to read information for block {} from pipeline {}: {}", blockID, pipeline.getId(), cause.getMessage()); if (refreshPipelineFunction != null) { LOG.debug("Re-fetching pipeline for block {}", blockID); - Pipeline newPipeline = refreshPipelineFunction.apply(blockID); - if (newPipeline == null || newPipeline.sameDatanodes(pipeline)) { - LOG.warn("No new pipeline for block {}", blockID); - throw cause; - } else { - LOG.debug("New pipeline got for block {}", blockID); - this.pipeline = newPipeline; - } + this.pipeline = refreshPipelineFunction.apply(blockID); } else { throw cause; } @@ -301,7 +306,13 @@ protected synchronized int readWithStrategy(ByteReaderStrategy strategy) int numBytesRead; try { numBytesRead = strategy.readFromBlock(current, numBytesToRead); - retries = 0; // reset retries after successful read + retries = 0; + // If we get a StorageContainerException or an IOException due to + // datanodes are not reachable, refresh to get the latest pipeline + // info and retry. + // Otherwise, just retry according to the retry policy. + } catch (SCMSecurityException ex) { + throw ex; } catch (StorageContainerException e) { if (shouldRetryRead(e)) { handleReadError(e); @@ -309,13 +320,13 @@ protected synchronized int readWithStrategy(ByteReaderStrategy strategy) } else { throw e; } - } catch (SCMSecurityException ex) { - throw ex; } catch (IOException ex) { - // We got a IOException which might be due - // to DN down or connectivity issue. if (shouldRetryRead(ex)) { - current.releaseClient(); + if (isConnectivityIssue(ex)) { + handleReadError(ex); + } else { + current.releaseClient(); + } continue; } else { throw ex; diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java index b3d3a7e389f7..e30df34b85d9 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkResponseProto; import org.apache.hadoop.hdds.scm.XceiverClientFactory; import org.apache.hadoop.hdds.scm.XceiverClientSpi; -import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.ozone.common.Checksum; import org.apache.hadoop.ozone.common.ChecksumData; @@ -425,20 +424,12 @@ protected ByteBuffer[] readChunk(ChunkInfo readChunkInfo) throws IOException { ReadChunkResponseProto readChunkResponse; - try { - List validators = - ContainerProtocolCalls.getValidatorList(); - validators.add(validator); + List validators = + ContainerProtocolCalls.getValidatorList(); + validators.add(validator); - readChunkResponse = ContainerProtocolCalls.readChunk(xceiverClient, - readChunkInfo, blockID, validators, token); - - } catch (IOException e) { - if (e instanceof StorageContainerException) { - throw e; - } - throw new IOException("Unexpected OzoneException: " + e.toString(), e); - } + readChunkResponse = ContainerProtocolCalls.readChunk(xceiverClient, + readChunkInfo, blockID, validators, token); if (readChunkResponse.hasData()) { return readChunkResponse.getData().asReadOnlyByteBufferList() diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java index 7f0121ea6e9b..0bba24d81921 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java @@ -29,19 +29,20 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.hdds.scm.pipeline.MockPipeline; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.security.exception.SCMSecurityException; import org.apache.hadoop.ozone.common.Checksum; import org.apache.hadoop.ozone.common.OzoneChecksumException; -import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.LambdaTestUtils; +import org.apache.ratis.thirdparty.io.grpc.Status; +import org.apache.ratis.thirdparty.io.grpc.StatusException; import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; -import org.slf4j.LoggerFactory; -import org.slf4j.event.Level; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; import java.io.EOFException; import java.io.IOException; @@ -52,11 +53,12 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; +import java.util.stream.Stream; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_NOT_FOUND; -import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_UNHEALTHY; import static org.apache.hadoop.hdds.scm.storage.TestChunkInputStream.generateRandomData; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -68,7 +70,6 @@ /** * Tests for {@link BlockInputStream}'s functionality. */ -@RunWith(MockitoJUnitRunner.class) public class TestBlockInputStream { private static final int CHUNK_SIZE = 100; @@ -80,11 +81,12 @@ public class TestBlockInputStream { private List chunks; private Map chunkDataMap; - @Mock private Function refreshPipeline; - @Before + @BeforeEach + @SuppressWarnings("unchecked") public void setup() throws Exception { + refreshPipeline = Mockito.mock(Function.class); BlockID blockID = new BlockID(new ContainerBlockID(1, 1)); checksum = new Checksum(ChecksumType.NONE, CHUNK_SIZE); createChunkList(5); @@ -280,35 +282,9 @@ public void testRefreshPipelineFunction() throws Exception { } } - @Test - public void testGetBlockInfoFailWithIOException() throws Exception { - GenericTestUtils.setLogLevel(BlockInputStream.getLog(), Level.DEBUG); - GenericTestUtils.LogCapturer logCapturer = - GenericTestUtils.LogCapturer.captureLogs( - LoggerFactory.getLogger(BlockInputStream.class)); - BlockID blockID = new BlockID(new ContainerBlockID(1, 1)); - AtomicBoolean isRefreshed = new AtomicBoolean(); - createChunkList(5); - BlockInputStream blockInputStreamWithRetry = - new DummyBlockInputStreamWithRetry(blockID, blockSize, - MockPipeline.createSingleNodePipeline(), null, - false, null, chunks, chunkDataMap, isRefreshed, - new IOException("unavailable")); - try { - Assert.assertFalse(isRefreshed.get()); - byte[] b = new byte[200]; - blockInputStreamWithRetry.read(b, 0, 200); - // As in case of IOException we do not do do refresh. - Assert.assertFalse(isRefreshed.get()); - Assert.assertTrue(logCapturer.getOutput().contains( - "Retry to get chunk info fail")); - } finally { - blockInputStreamWithRetry.close(); - } - } - - @Test - public void testRefreshOnReadFailure() throws Exception { + @ParameterizedTest + @MethodSource("exceptionsTriggersRefresh") + public void testRefreshOnReadFailure(IOException ex) throws Exception { // GIVEN BlockID blockID = new BlockID(new ContainerBlockID(1, 1)); Pipeline pipeline = MockPipeline.createSingleNodePipeline(); @@ -317,7 +293,7 @@ public void testRefreshOnReadFailure() throws Exception { final int len = 200; final ChunkInputStream stream = mock(ChunkInputStream.class); when(stream.read(any(), anyInt(), anyInt())) - .thenThrow(new StorageContainerException("test", CONTAINER_NOT_FOUND)) + .thenThrow(ex) .thenReturn(len); when(stream.getRemaining()) .thenReturn((long) len); @@ -347,47 +323,17 @@ protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) { } } - @Test - public void testRefreshExitsIfPipelineHasSameNodes() throws Exception { - // GIVEN - BlockID blockID = new BlockID(new ContainerBlockID(1, 1)); - Pipeline pipeline = MockPipeline.createSingleNodePipeline(); - - final int len = 200; - final ChunkInputStream stream = mock(ChunkInputStream.class); - when(stream.read(any(), anyInt(), anyInt())) - .thenThrow(new StorageContainerException("test", CONTAINER_UNHEALTHY)); - when(stream.getRemaining()) - .thenReturn((long) len); - - when(refreshPipeline.apply(blockID)) - .thenAnswer(invocation -> samePipelineWithNewId(pipeline)); - - BlockInputStream subject = new DummyBlockInputStream(blockID, blockSize, - pipeline, null, false, null, refreshPipeline, chunks, null) { - @Override - protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) { - return stream; - } - }; - - try { - subject.initialize(); - - // WHEN - byte[] b = new byte[len]; - LambdaTestUtils.intercept(StorageContainerException.class, - () -> subject.read(b, 0, len)); - - // THEN - verify(refreshPipeline).apply(blockID); - } finally { - subject.close(); - } + private static Stream exceptionsNotTriggerRefresh() { + return Stream.of( + Arguments.of(new SCMSecurityException("Security problem")), + Arguments.of(new OzoneChecksumException("checksum missing")), + Arguments.of(new IOException("Some random exception.")) + ); } - - @Test - public void testReadNotRetriedOnOtherException() throws Exception { + @ParameterizedTest + @MethodSource("exceptionsNotTriggerRefresh") + public void testReadNotRetriedOnOtherException(IOException ex) + throws Exception { // GIVEN BlockID blockID = new BlockID(new ContainerBlockID(1, 1)); Pipeline pipeline = MockPipeline.createSingleNodePipeline(); @@ -395,7 +341,7 @@ public void testReadNotRetriedOnOtherException() throws Exception { final int len = 200; final ChunkInputStream stream = mock(ChunkInputStream.class); when(stream.read(any(), anyInt(), anyInt())) - .thenThrow(new OzoneChecksumException("checksum missing")); + .thenThrow(ex); when(stream.getRemaining()) .thenReturn((long) len); @@ -412,9 +358,8 @@ protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) { // WHEN byte[] b = new byte[len]; - LambdaTestUtils.intercept(OzoneChecksumException.class, + Assertions.assertThrows(ex.getClass(), () -> subject.read(b, 0, len)); - // THEN verify(refreshPipeline, never()).apply(blockID); } finally { @@ -428,8 +373,10 @@ private Pipeline samePipelineWithNewId(Pipeline pipeline) { return MockPipeline.createPipeline(reverseOrder); } - @Test - public void testRefreshOnReadFailureAfterUnbuffer() throws Exception { + @ParameterizedTest + @MethodSource("exceptionsTriggersRefresh") + public void testRefreshOnReadFailureAfterUnbuffer(IOException ex) + throws Exception { // GIVEN BlockID blockID = new BlockID(new ContainerBlockID(1, 1)); Pipeline pipeline = MockPipeline.createSingleNodePipeline(); @@ -442,7 +389,7 @@ public void testRefreshOnReadFailureAfterUnbuffer() throws Exception { final int len = 200; final ChunkInputStream stream = mock(ChunkInputStream.class); when(stream.read(any(), anyInt(), anyInt())) - .thenThrow(new StorageContainerException("test", CONTAINER_NOT_FOUND)) + .thenThrow(ex) .thenReturn(len); when(stream.getRemaining()) .thenReturn((long) len); @@ -481,4 +428,12 @@ protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) { subject.close(); } } + + private static Stream exceptionsTriggersRefresh() { + return Stream.of( + Arguments.of(new StorageContainerException(CONTAINER_NOT_FOUND)), + Arguments.of(new IOException(new ExecutionException( + new StatusException(Status.UNAVAILABLE)))) + ); + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index c6872f286afe..a91885e25dde 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -34,6 +34,8 @@ public enum OzoneManagerVersion implements ComponentVersion { "New S3G persistent connection support is present in OM."), ERASURE_CODED_STORAGE_SUPPORT(2, "OzoneManager version that supports" + "ECReplicationConfig"), + OPTIMIZED_GET_KEY_INFO(3, "OzoneManager version that supports optimized" + + " key lookups using cached container locations."), FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index f3f87c52a8cb..1f970ecac61c 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1238,19 +1238,12 @@ public OzoneInputStream getKey( verifyVolumeName(volumeName); verifyBucketName(bucketName); Preconditions.checkNotNull(keyName); - OmKeyArgs keyArgs = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setSortDatanodesInPipeline(topologyAwareReadEnabled) - .setLatestVersionLocation(getLatestVersionLocation) - .build(); - OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs); + OmKeyInfo keyInfo = getKeyInfo(volumeName, bucketName, keyName, false); return getInputStreamWithRetryFunction(keyInfo); } @Override - public Map< OmKeyLocationInfo, Map > + public Map > getKeysEveryReplicas(String volumeName, String bucketName, String keyName) throws IOException { @@ -1260,22 +1253,15 @@ public OzoneInputStream getKey( verifyVolumeName(volumeName); verifyBucketName(bucketName); - Preconditions.checkNotNull(keyName); - OmKeyArgs keyArgs = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setSortDatanodesInPipeline(topologyAwareReadEnabled) - .build(); - - OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs); + OmKeyInfo keyInfo = getKeyInfo(volumeName, bucketName, keyName, true); List keyLocationInfos = keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly(); - for (OmKeyLocationInfo keyLocationInfo : keyLocationInfos) { + for (OmKeyLocationInfo locationInfo : keyLocationInfos) { Map blocks = new HashMap<>(); - Pipeline pipelineBefore = keyLocationInfo.getPipeline(); + + Pipeline pipelineBefore = locationInfo.getPipeline(); List datanodes = pipelineBefore.getNodes(); for (DatanodeDetails dn : datanodes) { @@ -1284,22 +1270,47 @@ public OzoneInputStream getKey( Pipeline pipeline = new Pipeline.Builder(pipelineBefore).setNodes(nodes) .setId(PipelineID.randomId()).build(); - keyLocationInfo.setPipeline(pipeline); + OmKeyLocationInfo dnKeyLocation = new OmKeyLocationInfo.Builder() + .setBlockID(locationInfo.getBlockID()) + .setLength(locationInfo.getLength()) + .setOffset(locationInfo.getOffset()) + .setToken(locationInfo.getToken()) + .setPartNumber(locationInfo.getPartNumber()) + .setCreateVersion(locationInfo.getCreateVersion()) + .setPipeline(pipeline) + .build(); - List keyLocationInfoList = new ArrayList<>(); - keyLocationInfoList.add(keyLocationInfo); + List keyLocationInfoList = + Collections.singletonList(dnKeyLocation); OmKeyLocationInfoGroup keyLocationInfoGroup = new OmKeyLocationInfoGroup(0, keyLocationInfoList); - List keyLocationInfoGroups = new ArrayList<>(); - keyLocationInfoGroups.add(keyLocationInfoGroup); + List keyLocationInfoGroups = + Collections.singletonList(keyLocationInfoGroup); keyInfo.setKeyLocationVersions(keyLocationInfoGroups); - OzoneInputStream is = createInputStream(keyInfo, Function.identity()); + OmKeyInfo dnKeyInfo = new OmKeyInfo.Builder() + .setVolumeName(keyInfo.getVolumeName()) + .setBucketName(keyInfo.getBucketName()) + .setKeyName(keyInfo.getKeyName()) + .setOmKeyLocationInfos(keyInfo.getKeyLocationVersions()) + .setDataSize(keyInfo.getDataSize()) + .setCreationTime(keyInfo.getCreationTime()) + .setModificationTime(keyInfo.getModificationTime()) + .setReplicationConfig(keyInfo.getReplicationConfig()) + .setFileEncryptionInfo(keyInfo.getFileEncryptionInfo()) + .setAcls(keyInfo.getAcls()) + .setObjectID(keyInfo.getObjectID()) + .setUpdateID(keyInfo.getUpdateID()) + .setParentObjectID(keyInfo.getParentObjectID()) + .setFileChecksum(keyInfo.getFileChecksum()) + .build(); + dnKeyInfo.setMetadata(keyInfo.getMetadata()); + dnKeyInfo.setKeyLocationVersions(keyLocationInfoGroups); - blocks.put(dn, is); + blocks.put(dn, createInputStream(dnKeyInfo, Function.identity())); } - result.put(keyLocationInfo, blocks); + result.put(locationInfo, blocks); } return result; @@ -1403,17 +1414,8 @@ public boolean recoverTrash(String volumeName, String bucketName, public OzoneKeyDetails getKeyDetails( String volumeName, String bucketName, String keyName) throws IOException { - Preconditions.checkNotNull(volumeName); - Preconditions.checkNotNull(bucketName); - Preconditions.checkNotNull(keyName); - OmKeyArgs keyArgs = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setSortDatanodesInPipeline(topologyAwareReadEnabled) - .setLatestVersionLocation(getLatestVersionLocation) - .build(); - OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs); + OmKeyInfo keyInfo = + getKeyInfo(volumeName, bucketName, keyName, false); List ozoneKeyLocations = new ArrayList<>(); long lastKeyOffset = 0L; @@ -1437,6 +1439,34 @@ public OzoneKeyDetails getKeyDetails( getInputStream); } + private OmKeyInfo getKeyInfo( + String volumeName, String bucketName, String keyName, + boolean forceUpdateContainerCache) throws IOException { + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + Preconditions.checkNotNull(keyName); + OmKeyArgs keyArgs = new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setSortDatanodesInPipeline(topologyAwareReadEnabled) + .setLatestVersionLocation(getLatestVersionLocation) + .setForceUpdateContainerCacheFromSCM(forceUpdateContainerCache) + .build(); + return getKeyInfo(keyArgs); + } + + private OmKeyInfo getKeyInfo(OmKeyArgs keyArgs) throws IOException { + final OmKeyInfo keyInfo; + if (omVersion.compareTo(OzoneManagerVersion.OPTIMIZED_GET_KEY_INFO) >= 0) { + keyInfo = ozoneManagerClient.getKeyInfo(keyArgs, false) + .getKeyInfo(); + } else { + keyInfo = ozoneManagerClient.lookupKey(keyArgs); + } + return keyInfo; + } + @Override public void close() throws IOException { if (ecReconstructExecutor != null) { @@ -1694,14 +1724,8 @@ private OzoneInputStream getInputStreamWithRetryFunction( OmKeyInfo keyInfo) throws IOException { return createInputStream(keyInfo, omKeyInfo -> { try { - OmKeyArgs omKeyArgs = new OmKeyArgs.Builder() - .setVolumeName(omKeyInfo.getVolumeName()) - .setBucketName(omKeyInfo.getBucketName()) - .setKeyName(omKeyInfo.getKeyName()) - .setSortDatanodesInPipeline(topologyAwareReadEnabled) - .setLatestVersionLocation(getLatestVersionLocation) - .build(); - return ozoneManagerClient.lookupKey(omKeyArgs); + return getKeyInfo(omKeyInfo.getVolumeName(), omKeyInfo.getBucketName(), + omKeyInfo.getKeyName(), true); } catch (IOException e) { LOG.error("Unable to lookup key {} on retry.", keyInfo.getKeyName(), e); return null; @@ -1999,8 +2023,9 @@ public OzoneKey headObject(String volumeName, String bucketName, .setKeyName(keyName) .setLatestVersionLocation(true) .setHeadOp(true) + .setForceUpdateContainerCacheFromSCM(false) .build(); - OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs); + OmKeyInfo keyInfo = getKeyInfo(keyArgs); return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(), diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockOmTransport.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockOmTransport.java index 90ed563b98ab..3eae9d1d7d0c 100644 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockOmTransport.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockOmTransport.java @@ -39,6 +39,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateVolumeResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteVolumeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteVolumeResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetKeyInfoRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetKeyInfoResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoBucketRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoBucketResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoVolumeRequest; @@ -129,6 +131,9 @@ public OMResponse submitRequest(OMRequest payload) throws IOException { case AllocateBlock: return response(payload, r -> r.setAllocateBlockResponse( allocateBlock(payload.getAllocateBlockRequest()))); + case GetKeyInfo: + return response(payload, r -> r.setGetKeyInfoResponse( + getKeyInfo(payload.getGetKeyInfoRequest()))); default: throw new IllegalArgumentException( "Mock version of om call " + payload.getCmdType() @@ -168,6 +173,15 @@ private LookupKeyResponse lookupKey(LookupKeyRequest lookupKeyRequest) { .build(); } + private GetKeyInfoResponse getKeyInfo(GetKeyInfoRequest request) { + final KeyArgs keyArgs = request.getKeyArgs(); + return GetKeyInfoResponse.newBuilder() + .setKeyInfo( + keys.get(keyArgs.getVolumeName()).get(keyArgs.getBucketName()) + .get(keyArgs.getKeyName())) + .build(); + } + private CommitKeyResponse commitKey(CommitKeyRequest commitKeyRequest) { final KeyArgs keyArgs = commitKeyRequest.getKeyArgs(); final KeyInfo openKey = diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 40f515adc448..2bd30eee721c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -270,7 +270,9 @@ default OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID, * @param args the args of the key. * @return OmKeyInfo instance that client uses to talk to container. * @throws IOException + * @deprecated use {@link OzoneManagerProtocol#getKeyInfo} instead. */ + @Deprecated OmKeyInfo lookupKey(OmKeyArgs args) throws IOException; /** diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStream.java index 442cbfe7604f..f0c684486b17 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStream.java @@ -410,7 +410,9 @@ private void testReadAfterReplication(boolean doUnbuffer) throws Exception { .setKeyName(keyName) .setReplicationConfig(RatisReplicationConfig.getInstance(THREE)) .build(); - OmKeyInfo keyInfo = getCluster().getOzoneManager().lookupKey(keyArgs); + OmKeyInfo keyInfo = getCluster().getOzoneManager() + .getKeyInfo(keyArgs, false) + .getKeyInfo(); OmKeyLocationInfoGroup locations = keyInfo.getLatestVersionLocations(); Assert.assertNotNull(locations); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java new file mode 100644 index 000000000000..50aa6ff7b612 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java @@ -0,0 +1,674 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements.  See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership.  The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License.  You may obtain a copy of the License at + * + *      http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.client.ContainerBlockID; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.BlockData; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.GetBlockResponseProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.GetCommittedBlockLengthResponseProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutBlockResponseProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkResponseProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.WriteChunkResponseProto; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.scm.XceiverClientFactory; +import org.apache.hadoop.hdds.scm.XceiverClientGrpc; +import org.apache.hadoop.hdds.scm.XceiverClientManager; +import org.apache.hadoop.hdds.scm.XceiverClientReply; +import org.apache.hadoop.hdds.scm.XceiverClientSpi; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; +import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; +import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; +import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.hdds.security.exception.SCMSecurityException; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneKeyDetails; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.client.rpc.RpcClient; +import org.apache.hadoop.ozone.common.Checksum; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.ozone.test.GenericTestUtils; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.io.grpc.Status; +import org.apache.ratis.thirdparty.io.grpc.StatusException; +import org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException; +import org.apache.ratis.util.ExitUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.Rule; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.rules.Timeout; +import org.mockito.ArgumentMatcher; +import org.mockito.Mockito; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.security.cert.X509Certificate; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; + +import static com.google.common.collect.Sets.newHashSet; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * This class includes the integration test-cases to verify the integration + * between client and OM to keep container location cache eventually + * consistent. For example, when clients facing particular errors reading data + * from datanodes, they should inform OM to refresh location cache and OM + * should in turn contact SCM to get the updated container location. + * + * This integration verifies clients and OM using mocked Datanode and SCM + * protocols. + */ +public class TestOmContainerLocationCache { + + /** + * Set a timeout for each test. + */ + @Rule + public Timeout timeout = Timeout.seconds(300); + private static ScmBlockLocationProtocol mockScmBlockLocationProtocol; + private static StorageContainerLocationProtocol mockScmContainerClient; + private static OzoneConfiguration conf; + private static OMMetadataManager metadataManager; + private static File dir; + private static final String BUCKET_NAME = "bucket1"; + private static final String VERSIONED_BUCKET_NAME = "versionedBucket1"; + private static final String VOLUME_NAME = "vol1"; + private static OzoneManager om; + private static RpcClient rpcClient; + private static ObjectStore objectStore; + private static XceiverClientGrpc mockDn1Protocol; + private static XceiverClientGrpc mockDn2Protocol; + private static final DatanodeDetails DN1 = + MockDatanodeDetails.createDatanodeDetails(UUID.randomUUID()); + private static final DatanodeDetails DN2 = + MockDatanodeDetails.createDatanodeDetails(UUID.randomUUID()); + private static long testContainerId = 1L; + + + @BeforeAll + public static void setUp() throws Exception { + ExitUtils.disableSystemExit(); + + conf = new OzoneConfiguration(); + conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "127.0.0.1:0"); + dir = GenericTestUtils.getRandomizedTestDir(); + conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.toString()); + conf.set(OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY, "true"); + conf.setLong(OZONE_KEY_PREALLOCATION_BLOCKS_MAX, 10); + + mockScmBlockLocationProtocol = mock(ScmBlockLocationProtocol.class); + mockScmContainerClient = + Mockito.mock(StorageContainerLocationProtocol.class); + + OmTestManagers omTestManagers = new OmTestManagers(conf, + mockScmBlockLocationProtocol, mockScmContainerClient); + om = omTestManagers.getOzoneManager(); + metadataManager = omTestManagers.getMetadataManager(); + + rpcClient = new RpcClient(conf, null) { + @NotNull + @Override + protected XceiverClientFactory createXceiverClientFactory( + List x509Certificates) throws IOException { + return mockDataNodeClientFactory(); + } + }; + + objectStore = new ObjectStore(conf, rpcClient); + + createVolume(VOLUME_NAME); + createBucket(VOLUME_NAME, BUCKET_NAME, false); + createBucket(VOLUME_NAME, VERSIONED_BUCKET_NAME, true); + } + + @AfterAll + public static void cleanup() throws Exception { + om.stop(); + FileUtils.deleteDirectory(dir); + } + + private static XceiverClientManager mockDataNodeClientFactory() + throws IOException { + mockDn1Protocol = spy(new XceiverClientGrpc(createPipeline(DN1), conf)); + mockDn2Protocol = spy(new XceiverClientGrpc(createPipeline(DN2), conf)); + XceiverClientManager manager = mock(XceiverClientManager.class); + when(manager.acquireClient(argThat(matchPipeline(DN1)))) + .thenReturn(mockDn1Protocol); + when(manager.acquireClientForReadData(argThat(matchPipeline(DN1)))) + .thenReturn(mockDn1Protocol); + + when(manager.acquireClient(argThat(matchPipeline(DN2)))) + .thenReturn(mockDn2Protocol); + when(manager.acquireClientForReadData(argThat(matchPipeline(DN2)))) + .thenReturn(mockDn2Protocol); + return manager; + } + + private static ArgumentMatcher matchPipeline(DatanodeDetails dn) { + return argument -> argument != null + && argument.getNodes().get(0).getUuid().equals(dn.getUuid()); + } + + private static void createBucket(String volumeName, String bucketName, + boolean isVersionEnabled) + throws IOException { + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setIsVersionEnabled(isVersionEnabled) + .build(); + + OMRequestTestUtils.addBucketToOM(metadataManager, bucketInfo); + } + + private static void createVolume(String volumeName) throws IOException { + OmVolumeArgs volumeArgs = OmVolumeArgs.newBuilder() + .setVolume(volumeName) + .setAdminName("bilbo") + .setOwnerName("bilbo") + .build(); + OMRequestTestUtils.addVolumeToOM(metadataManager, volumeArgs); + } + + @BeforeEach + @SuppressFBWarnings("ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD") + public void beforeEach() { + testContainerId++; + Mockito.reset(mockScmBlockLocationProtocol, mockScmContainerClient, + mockDn1Protocol, mockDn2Protocol); + when(mockDn1Protocol.getPipeline()).thenReturn(createPipeline(DN1)); + when(mockDn2Protocol.getPipeline()).thenReturn(createPipeline(DN2)); + } + + /** + * Verify that in a happy case, container location is cached and reused + * in OM. + */ + @Test + public void containerCachedInHappyCase() throws Exception { + byte[] data = "Test content".getBytes(UTF_8); + + mockScmAllocationOnDn1(testContainerId, 1L); + mockWriteChunkResponse(mockDn1Protocol); + mockPutBlockResponse(mockDn1Protocol, testContainerId, 1L, data); + + OzoneBucket bucket = objectStore.getVolume(VOLUME_NAME) + .getBucket(BUCKET_NAME); + + // Create keyName1. + String keyName1 = "key1"; + try (OzoneOutputStream os = bucket.createKey(keyName1, data.length)) { + IOUtils.write(data, os); + } + + mockScmGetContainerPipeline(testContainerId, DN1); + + // Read keyName1. + OzoneKeyDetails key1 = bucket.getKey(keyName1); + verify(mockScmContainerClient, times(1)) + .getContainerWithPipelineBatch(newHashSet(testContainerId)); + + mockGetBlock(mockDn1Protocol, testContainerId, 1L, data, null, null); + mockReadChunk(mockDn1Protocol, testContainerId, 1L, data, null, null); + try (InputStream is = key1.getContent()) { + byte[] read = new byte[(int) key1.getDataSize()]; + IOUtils.read(is, read); + Assertions.assertArrayEquals(data, read); + } + + // Create keyName2 in the same container to reuse the cache + String keyName2 = "key2"; + try (OzoneOutputStream os = bucket.createKey(keyName2, data.length)) { + IOUtils.write(data, os); + } + // Read keyName2. + OzoneKeyDetails key2 = bucket.getKey(keyName2); + try (InputStream is = key2.getContent()) { + byte[] read = new byte[(int) key2.getDataSize()]; + IOUtils.read(is, read); + Assertions.assertArrayEquals(data, read); + } + // Ensure SCM is not called once again. + verify(mockScmContainerClient, times(1)) + .getContainerWithPipelineBatch(newHashSet(testContainerId)); + } + + private static Stream errorsTriggerRefresh() { + return Stream.of( + Arguments.of(null, Result.CLOSED_CONTAINER_IO), + Arguments.of(null, Result.CONTAINER_NOT_FOUND), + Arguments.of(new StatusException(Status.UNAVAILABLE), null), + Arguments.of(new StatusRuntimeException(Status.UNAVAILABLE), null) + ); + } + + private static Stream errorsNotTriggerRefresh() { + return Stream.of( + Arguments.of(new StatusException(Status.UNAUTHENTICATED), null, + SCMSecurityException.class), + Arguments.of(new IOException("Any random IO exception."), null, + IOException.class) + ); + } + + /** + * Verify that in case a client got errors calling datanodes GetBlock, + * the client correctly requests OM to refresh relevant container location + * from SCM. + */ + @ParameterizedTest + @MethodSource("errorsTriggerRefresh") + public void containerRefreshedAfterDatanodeGetBlockError( + Exception dnException, Result dnResponseCode) throws Exception { + byte[] data = "Test content".getBytes(UTF_8); + + mockScmAllocationOnDn1(testContainerId, 1L); + mockWriteChunkResponse(mockDn1Protocol); + mockPutBlockResponse(mockDn1Protocol, testContainerId, 1L, data); + + OzoneBucket bucket = objectStore.getVolume(VOLUME_NAME) + .getBucket(BUCKET_NAME); + + String keyName = "key"; + try (OzoneOutputStream os = bucket.createKey(keyName, data.length)) { + IOUtils.write(data, os); + } + + mockScmGetContainerPipeline(testContainerId, DN1); + + OzoneKeyDetails key1 = bucket.getKey(keyName); + + verify(mockScmContainerClient, times(1)) + .getContainerWithPipelineBatch(newHashSet(testContainerId)); + + try (InputStream is = key1.getContent()) { + // Simulate dn1 got errors, and the container's moved to dn2. + mockGetBlock(mockDn1Protocol, testContainerId, 1L, null, + dnException, dnResponseCode); + mockScmGetContainerPipeline(testContainerId, DN2); + mockGetBlock(mockDn2Protocol, testContainerId, 1L, data, null, null); + mockReadChunk(mockDn2Protocol, testContainerId, 1L, data, null, null); + + byte[] read = new byte[(int) key1.getDataSize()]; + IOUtils.read(is, read); + Assertions.assertArrayEquals(data, read); + } + + // verify SCM is called one more time to refresh. + verify(mockScmContainerClient, times(2)) + .getContainerWithPipelineBatch(newHashSet(testContainerId)); + } + + /** + * Verify that in case a client got errors datanodes ReadChunk,the client + * correctly requests OM to refresh relevant container location from + * SCM. + */ + @ParameterizedTest + @MethodSource("errorsTriggerRefresh") + public void containerRefreshedAfterDatanodeReadChunkError( + Exception dnException, Result dnResponseCode) throws Exception { + byte[] data = "Test content".getBytes(UTF_8); + + mockScmAllocationOnDn1(testContainerId, 1L); + mockWriteChunkResponse(mockDn1Protocol); + mockPutBlockResponse(mockDn1Protocol, testContainerId, 1L, data); + + OzoneBucket bucket = objectStore.getVolume(VOLUME_NAME) + .getBucket(BUCKET_NAME); + + String keyName = "key"; + try (OzoneOutputStream os = bucket.createKey(keyName, data.length)) { + IOUtils.write(data, os); + } + + mockScmGetContainerPipeline(testContainerId, DN1); + + OzoneKeyDetails key1 = bucket.getKey(keyName); + + verify(mockScmContainerClient, times(1)) + .getContainerWithPipelineBatch(newHashSet(testContainerId)); + + try (InputStream is = key1.getContent()) { + // simulate dn1 goes down, the container's to dn2. + mockGetBlock(mockDn1Protocol, testContainerId, 1L, data, null, null); + mockReadChunk(mockDn1Protocol, testContainerId, 1L, null, + dnException, dnResponseCode); + mockScmGetContainerPipeline(testContainerId, DN2); + mockGetBlock(mockDn2Protocol, testContainerId, 1L, data, null, null); + mockReadChunk(mockDn2Protocol, testContainerId, 1L, data, null, null); + + byte[] read = new byte[(int) key1.getDataSize()]; + IOUtils.read(is, read); + Assertions.assertArrayEquals(data, read); + } + + // verify SCM is called one more time to refresh. + verify(mockScmContainerClient, times(2)) + .getContainerWithPipelineBatch(newHashSet(testContainerId)); + } + + /** + * Verify that in case a client got particular errors datanodes GetBlock, + * the client fails correctly fast and don't invoke cache refresh. + */ + @ParameterizedTest + @MethodSource("errorsNotTriggerRefresh") + public void containerNotRefreshedAfterDatanodeGetBlockError( + Exception ex, Result errorCode, Class expectedEx) + throws Exception { + byte[] data = "Test content".getBytes(UTF_8); + + mockScmAllocationOnDn1(testContainerId, 1L); + mockWriteChunkResponse(mockDn1Protocol); + mockPutBlockResponse(mockDn1Protocol, testContainerId, 1L, data); + + OzoneBucket bucket = objectStore.getVolume(VOLUME_NAME) + .getBucket(BUCKET_NAME); + + String keyName = "key"; + try (OzoneOutputStream os = bucket.createKey(keyName, data.length)) { + IOUtils.write(data, os); + } + + mockScmGetContainerPipeline(testContainerId, DN1); + + OzoneKeyDetails key1 = bucket.getKey(keyName); + + verify(mockScmContainerClient, times(1)) + .getContainerWithPipelineBatch(newHashSet(testContainerId)); + + try (InputStream is = key1.getContent()) { + // simulate dn1 got errors, and the container's moved to dn2. + mockGetBlock(mockDn1Protocol, testContainerId, 1L, null, ex, errorCode); + + assertThrows(expectedEx, + () -> IOUtils.read(is, new byte[(int) key1.getDataSize()])); + } + + // verify SCM is called one more time to refresh. + verify(mockScmContainerClient, times(1)) + .getContainerWithPipelineBatch(newHashSet(testContainerId)); + } + + /** + * Verify that in case a client got particular errors datanodes ReadChunk, + * the client fails correctly fast and don't invoke cache refresh. + */ + @ParameterizedTest + @MethodSource("errorsNotTriggerRefresh") + public void containerNotRefreshedAfterDatanodeReadChunkError( + Exception dnException, Result dnResponseCode, + Class expectedEx) throws Exception { + byte[] data = "Test content".getBytes(UTF_8); + + mockScmAllocationOnDn1(testContainerId, 1L); + mockWriteChunkResponse(mockDn1Protocol); + mockPutBlockResponse(mockDn1Protocol, testContainerId, 1L, data); + + OzoneBucket bucket = objectStore.getVolume(VOLUME_NAME) + .getBucket(BUCKET_NAME); + + String keyName = "key"; + try (OzoneOutputStream os = bucket.createKey(keyName, data.length)) { + IOUtils.write(data, os); + } + + mockScmGetContainerPipeline(testContainerId, DN1); + + OzoneKeyDetails key1 = bucket.getKey(keyName); + + verify(mockScmContainerClient, times(1)) + .getContainerWithPipelineBatch(newHashSet(testContainerId)); + + try (InputStream is = key1.getContent()) { + // simulate dn1 got errors, and the container's moved to dn2. + mockGetBlock(mockDn1Protocol, testContainerId, 1L, data, null, null); + mockReadChunk(mockDn1Protocol, testContainerId, 1L, null, + dnException, dnResponseCode); + + assertThrows(expectedEx, + () -> IOUtils.read(is, new byte[(int) key1.getDataSize()])); + } + + // verify SCM is called one more time to refresh. + verify(mockScmContainerClient, times(1)) + .getContainerWithPipelineBatch(newHashSet(testContainerId)); + } + + private void mockPutBlockResponse(XceiverClientSpi mockDnProtocol, + long containerId, long localId, + byte[] data) + throws IOException, ExecutionException, InterruptedException { + GetCommittedBlockLengthResponseProto build = + GetCommittedBlockLengthResponseProto.newBuilder() + .setBlockLength(8) + .setBlockID(createBlockId(containerId, localId)) + .build(); + ContainerCommandResponseProto putResponse = + ContainerCommandResponseProto.newBuilder() + .setPutBlock(PutBlockResponseProto.newBuilder() + .setCommittedBlockLength(build).build()) + .setResult(Result.SUCCESS) + .setCmdType(Type.PutBlock) + .build(); + doAnswer(invocation -> + new XceiverClientReply(completedFuture(putResponse))) + .when(mockDnProtocol) + .sendCommandAsync(argThat(matchCmd(Type.PutBlock))); + } + + @NotNull + private ContainerProtos.DatanodeBlockID createBlockId(long containerId, + long localId) { + return ContainerProtos.DatanodeBlockID.newBuilder() + .setContainerID(containerId) + .setLocalID(localId).build(); + } + + private void mockWriteChunkResponse(XceiverClientSpi mockDnProtocol) + throws IOException, ExecutionException, InterruptedException { + ContainerCommandResponseProto writeResponse = + ContainerCommandResponseProto.newBuilder() + .setWriteChunk(WriteChunkResponseProto.newBuilder().build()) + .setResult(Result.SUCCESS) + .setCmdType(Type.WriteChunk) + .build(); + doAnswer(invocation -> + new XceiverClientReply(completedFuture(writeResponse))) + .when(mockDnProtocol) + .sendCommandAsync(argThat(matchCmd(Type.WriteChunk))); + } + + private ArgumentMatcher matchCmd(Type type) { + return argument -> argument != null && argument.getCmdType() == type; + } + + private void mockScmAllocationOnDn1(long containerID, + long localId) throws IOException { + ContainerBlockID blockId = new ContainerBlockID(containerID, localId); + AllocatedBlock block = new AllocatedBlock.Builder() + .setPipeline(createPipeline(DN1)) + .setContainerBlockID(blockId) + .build(); + when(mockScmBlockLocationProtocol + .allocateBlock(Mockito.anyLong(), Mockito.anyInt(), + any(ReplicationConfig.class), + Mockito.anyString(), + any(ExcludeList.class))) + .thenReturn(Collections.singletonList(block)); + } + + private void mockScmGetContainerPipeline(long containerId, + DatanodeDetails dn) + throws IOException { + Pipeline pipeline = createPipeline(dn); + ContainerInfo containerInfo = new ContainerInfo.Builder() + .setContainerID(containerId) + .setPipelineID(pipeline.getId()).build(); + List containerWithPipelines = + Collections.singletonList( + new ContainerWithPipeline(containerInfo, pipeline)); + + when(mockScmContainerClient.getContainerWithPipelineBatch( + newHashSet(containerId))).thenReturn(containerWithPipelines); + } + + private void mockGetBlock(XceiverClientGrpc mockDnProtocol, + long containerId, long localId, + byte[] data, + Exception exception, + Result errorCode) throws Exception { + + final CompletableFuture response; + if (exception != null) { + response = new CompletableFuture<>(); + response.completeExceptionally(exception); + } else if (errorCode != null) { + ContainerCommandResponseProto getBlockResp = + ContainerCommandResponseProto.newBuilder() + .setResult(errorCode) + .setCmdType(Type.GetBlock) + .build(); + response = completedFuture(getBlockResp); + } else { + ContainerCommandResponseProto getBlockResp = + ContainerCommandResponseProto.newBuilder() + .setGetBlock(GetBlockResponseProto.newBuilder() + .setBlockData(BlockData.newBuilder() + .addChunks(createChunkInfo(data)) + .setBlockID(createBlockId(containerId, localId)) + .build()) + .build() + ) + .setResult(Result.SUCCESS) + .setCmdType(Type.GetBlock) + .build(); + response = completedFuture(getBlockResp); + } + doAnswer(invocation -> new XceiverClientReply(response)) + .when(mockDnProtocol) + .sendCommandAsync(argThat(matchCmd(Type.GetBlock)), any()); + } + + @NotNull + private ChunkInfo createChunkInfo(byte[] data) throws Exception { + Checksum checksum = new Checksum(ChecksumType.CRC32, 4); + return ChunkInfo.newBuilder() + .setOffset(0) + .setLen(data.length) + .setChunkName("chunk1") + .setChecksumData(checksum.computeChecksum(data).getProtoBufMessage()) + .build(); + } + + private void mockReadChunk(XceiverClientGrpc mockDnProtocol, + long containerId, long localId, + byte[] data, + Exception exception, + Result errorCode) throws Exception { + final CompletableFuture response; + if (exception != null) { + response = new CompletableFuture<>(); + response.completeExceptionally(exception); + } else if (errorCode != null) { + ContainerCommandResponseProto readChunkResp = + ContainerCommandResponseProto.newBuilder() + .setResult(errorCode) + .setCmdType(Type.ReadChunk) + .build(); + response = completedFuture(readChunkResp); + } else { + ContainerCommandResponseProto readChunkResp = + ContainerCommandResponseProto.newBuilder() + .setReadChunk(ReadChunkResponseProto.newBuilder() + .setBlockID(createBlockId(containerId, localId)) + .setChunkData(createChunkInfo(data)) + .setData(ByteString.copyFrom(data)) + .build() + ) + .setResult(Result.SUCCESS) + .setCmdType(Type.ReadChunk) + .build(); + response = completedFuture(readChunkResp); + } + + doAnswer(invocation -> new XceiverClientReply(response)) + .when(mockDnProtocol) + .sendCommandAsync(argThat(matchCmd(Type.ReadChunk)), any()); + + } + + private static Pipeline createPipeline(DatanodeDetails dn) { + return Pipeline.newBuilder() + .setState(Pipeline.PipelineState.OPEN) + .setId(PipelineID.randomId()) + .setReplicationConfig( + RatisReplicationConfig.getInstance(ReplicationFactor.THREE)) + .setNodes(Collections.singletonList(dn)) + .build(); + } +} \ No newline at end of file From 2050ebb0668d5283733b34608333f9628b448b9a Mon Sep 17 00:00:00 2001 From: Duong Nguyen Date: Wed, 26 Oct 2022 18:06:12 -0700 Subject: [PATCH 47/48] HDDS-7419. Integrate the GetKeyInfo API to OFS --- .../hadoop/ozone/client/rpc/RpcClient.java | 12 +++++++- .../hadoop/ozone/om/helpers/OmKeyInfo.java | 30 +++++++++++++++++-- .../om/protocol/OzoneManagerProtocol.java | 2 ++ .../src/main/proto/OmClientProtocol.proto | 1 + .../hadoop/ozone/om/KeyManagerImpl.java | 23 ++++++++++++-- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 1 + .../BasicRootedOzoneClientAdapterImpl.java | 1 + 7 files changed, 65 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 1f970ecac61c..0bbe7d5dbdff 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1698,7 +1698,17 @@ public OzoneInputStream readFile(String volumeName, String bucketName, .setSortDatanodesInPipeline(topologyAwareReadEnabled) .setLatestVersionLocation(getLatestVersionLocation) .build(); - OmKeyInfo keyInfo = ozoneManagerClient.lookupFile(keyArgs); + final OmKeyInfo keyInfo; + if (omVersion.compareTo(OzoneManagerVersion.OPTIMIZED_GET_KEY_INFO) >= 0) { + keyInfo = ozoneManagerClient.getKeyInfo(keyArgs, false) + .getKeyInfo(); + if (!keyInfo.isFile()) { + throw new OMException(keyName + " is not a file.", + OMException.ResultCodes.NOT_A_FILE); + } + } else { + keyInfo = ozoneManagerClient.lookupFile(keyArgs); + } return getInputStreamWithRetryFunction(keyInfo); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index f8f589af2efc..209af9f7b25c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -61,6 +61,10 @@ public final class OmKeyInfo extends WithParentObjectId { private ReplicationConfig replicationConfig; private FileEncryptionInfo encInfo; private FileChecksum fileChecksum; + /** + * Support OSF use-case to identify if the key is a file or a directory. + */ + private boolean isFile; /** * Represents leaf node name. This also will be used when the keyName is @@ -106,12 +110,13 @@ public final class OmKeyInfo extends WithParentObjectId { Map metadata, FileEncryptionInfo encInfo, List acls, long parentObjectID, long objectID, long updateID, - FileChecksum fileChecksum) { + FileChecksum fileChecksum, boolean isFile) { this(volumeName, bucketName, keyName, versions, dataSize, creationTime, modificationTime, replicationConfig, metadata, encInfo, acls, objectID, updateID, fileChecksum); this.fileName = fileName; this.parentObjectID = parentObjectID; + this.isFile = isFile; } public String getVolumeName() { @@ -177,6 +182,14 @@ public void updateModifcationTime() { this.modificationTime = Time.monotonicNow(); } + public void setFile(boolean file) { + isFile = file; + } + + public boolean isFile() { + return isFile; + } + /** * updates the length of the each block in the list given. * This will be called when the key is being committed to OzoneManager. @@ -409,6 +422,8 @@ public static class Builder { private long parentObjectID; private FileChecksum fileChecksum; + private boolean isFile; + public Builder() { this.metadata = new HashMap<>(); omKeyLocationInfoGroups = new ArrayList<>(); @@ -520,12 +535,17 @@ public Builder setFileChecksum(FileChecksum checksum) { return this; } + public Builder setFile(boolean isAFile) { + this.isFile = isAFile; + return this; + } + public OmKeyInfo build() { return new OmKeyInfo( volumeName, bucketName, keyName, fileName, omKeyLocationInfoGroups, dataSize, creationTime, modificationTime, replicationConfig, metadata, encInfo, acls, - parentObjectID, objectID, updateID, fileChecksum); + parentObjectID, objectID, updateID, fileChecksum, isFile); } } @@ -627,6 +647,7 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, if (encInfo != null) { kb.setFileEncryptionInfo(OMPBHelper.convert(encInfo)); } + kb.setIsFile(isFile); return kb.build(); } @@ -669,6 +690,11 @@ public static OmKeyInfo getFromProtobuf(KeyInfo keyInfo) throws IOException { FileChecksum fileChecksum = OMPBHelper.convert(keyInfo.getFileChecksum()); builder.setFileChecksum(fileChecksum); } + + if (keyInfo.hasIsFile()) { + builder.setFile(keyInfo.getIsFile()); + } + // not persisted to DB. FileName will be filtered out from keyName builder.setFileName(OzoneFSUtils.getFileName(keyInfo.getKeyName())); return builder.build(); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 2bd30eee721c..d9a758127536 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -759,7 +759,9 @@ default OpenKeySession createFile(OmKeyArgs keyArgs, boolean overWrite, * if bucket does not exist * @throws IOException if there is error in the db * invalid arguments + * @deprecated use {@link OzoneManagerProtocol#getKeyInfo} instead. */ + @Deprecated OmKeyInfo lookupFile(OmKeyArgs keyArgs) throws IOException; /** diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 76f224b7d7ad..5ee314643576 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -953,6 +953,7 @@ message KeyInfo { optional uint64 parentID = 16; optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 17; optional FileChecksumProto fileChecksum = 18; + optional bool isFile = 19; } message DirectoryInfo { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index dd66c33a1e3b..2aec21c6ce27 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -367,7 +367,7 @@ private OmKeyInfo readKeyInfo(OmKeyArgs args) throws IOException { if (bucketLayout.isFileSystemOptimized()) { value = getOmKeyInfoFSO(volumeName, bucketName, keyName); } else { - value = getOmKeyInfo(volumeName, bucketName, keyName); + value = getOmKeyInfoDirectoryAware(volumeName, bucketName, keyName); } } catch (IOException ex) { if (ex instanceof OMException) { @@ -396,8 +396,26 @@ private OmKeyInfo readKeyInfo(OmKeyArgs args) throws IOException { return value; } + private OmKeyInfo getOmKeyInfoDirectoryAware(String volumeName, + String bucketName, String keyName) throws IOException { + OmKeyInfo keyInfo = getOmKeyInfo(volumeName, bucketName, keyName); + + // Check if the key is a directory. + if (keyInfo != null) { + keyInfo.setFile(true); + return keyInfo; + } + + String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); + OmKeyInfo dirKeyInfo = getOmKeyInfo(volumeName, bucketName, dirKey); + if (dirKeyInfo != null) { + dirKeyInfo.setFile(false); + } + return dirKeyInfo; + } + private OmKeyInfo getOmKeyInfo(String volumeName, String bucketName, - String keyName) throws IOException { + String keyName) throws IOException { String keyBytes = metadataManager.getOzoneKey(volumeName, bucketName, keyName); BucketLayout bucketLayout = getBucketLayout(metadataManager, volumeName, @@ -425,6 +443,7 @@ private OmKeyInfo getOmKeyInfoFSO(String volumeName, String bucketName, fileStatus.getKeyInfo().getKeyName()); fileStatus.getKeyInfo().setKeyName(keyPath); } + fileStatus.getKeyInfo().setFile(fileStatus.isFile()); return fileStatus.getKeyInfo(); } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index 25e4ffbb1874..f7a7326296d0 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@ -219,6 +219,7 @@ public InputStream readFile(String key) throws IOException { return bucket.readFile(key).getInputStream(); } catch (OMException ex) { if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND + || ex.getResult() == OMException.ResultCodes.KEY_NOT_FOUND || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) { throw new FileNotFoundException( ex.getResult().name() + ": " + ex.getMessage()); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 724a76bb786a..d04206213806 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -351,6 +351,7 @@ public InputStream readFile(String pathStr) throws IOException { return bucket.readFile(key).getInputStream(); } catch (OMException ex) { if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND + || ex.getResult() == OMException.ResultCodes.KEY_NOT_FOUND || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) { throw new FileNotFoundException( ex.getResult().name() + ": " + ex.getMessage()); From 42aed99b8ba658d58ed3b1be851426f0c9989c4e Mon Sep 17 00:00:00 2001 From: Duong Nguyen Date: Wed, 26 Oct 2022 20:55:29 -0700 Subject: [PATCH 48/48] Update integration test. --- .../org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index f3e8cf10bee3..58ffd4256e07 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -1005,10 +1005,10 @@ public void testSeekOnFileLength() throws IOException { Path fileNotExists = new Path("/file_notexist"); try { fs.open(fileNotExists); - Assert.fail("Should throw FILE_NOT_FOUND error as file doesn't exist!"); + Assert.fail("Should throw FileNotFoundException as file doesn't exist!"); } catch (FileNotFoundException fnfe) { - Assert.assertTrue("Expected FILE_NOT_FOUND error", - fnfe.getMessage().contains("FILE_NOT_FOUND")); + Assert.assertTrue("Expected KEY_NOT_FOUND error", + fnfe.getMessage().contains("KEY_NOT_FOUND")); } }