From 6c53f896d3f5a00d2e7cfea437a9cdfc7a5c72a2 Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Tue, 10 Jun 2025 12:35:33 +0530 Subject: [PATCH 01/10] HDDS-13092. Container scanner should trigger on-demand volume scan when marking a container unhealthy --- .../ozoneimpl/BackgroundContainerDataScanner.java | 4 ++++ .../BackgroundContainerMetadataScanner.java | 4 ++++ .../ozoneimpl/OnDemandContainerDataScanner.java | 4 ++++ .../TestBackgroundContainerDataScanner.java | 14 ++++++++++++++ .../TestBackgroundContainerMetadataScanner.java | 15 +++++++++++++++ .../ozoneimpl/TestContainerScannersAbstract.java | 9 +++++++++ .../TestOnDemandContainerDataScanner.java | 12 ++++++++++++ 7 files changed, 62 insertions(+) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java index 7edd99cdba1..9156f60700d 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java @@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.interfaces.Container; +import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -100,6 +101,9 @@ public void scanContainer(Container c) containerId, result.getException()); metrics.incNumUnHealthyContainers(); controller.markContainerUnhealthy(containerId, result); + LOG.info("Triggering a volume scan for volume [{}] as unhealthy container [{}] was on it.", + containerData.getVolume().getStorageDir().getPath(), containerId); + StorageVolumeUtil.onFailure(containerData.getVolume()); } metrics.incNumContainersScanned(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java index 5e2e26a37df..67d313aebb6 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java @@ -23,6 +23,7 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.interfaces.Container; +import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,6 +81,9 @@ public void scanContainer(Container container) containerID, result.getException()); metrics.incNumUnHealthyContainers(); controller.markContainerUnhealthy(containerID, result); + LOG.info("Triggering a volume scan for volume [{}] as unhealthy container [{}] was on it.", + containerVolume.getStorageDir().getPath(), containerID); + StorageVolumeUtil.onFailure(containerVolume); } // Do not update the scan timestamp after the scan since this was just a diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java index a5639922152..d119e5c2834 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java @@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult; +import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -149,6 +150,9 @@ private static void performOnDemandScan(Container container) { instance.metrics.incNumUnHealthyContainers(); instance.containerController.markContainerUnhealthy(containerId, result); + LOG.info("Triggering a volume scan for volume [{}] as unhealthy container [{}] was on it.", + containerData.getVolume().getStorageDir().getPath(), containerId); + StorageVolumeUtil.onFailure(containerData.getVolume()); } instance.metrics.incNumContainersScanned(); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java index 93f8a957e78..ea2314753d6 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java @@ -28,7 +28,9 @@ import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.atMostOnce; import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -41,9 +43,11 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult; +import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; @@ -126,6 +130,16 @@ public void testUnhealthyContainersDetected() throws Exception { verifyContainerMarkedUnhealthy(deletedContainer, never()); } + @Test + @Override + public void testUnhealthyContainersTriggersVolumeScan() throws Exception { + try (MockedStatic mockedStatic = mockStatic(StorageVolumeUtil.class)) { + scanner.runIteration(); + verifyContainerMarkedUnhealthy(corruptData, atLeastOnce()); + mockedStatic.verify(() -> StorageVolumeUtil.onFailure(corruptData.getContainerData().getVolume()), times(1)); + } + } + @Test public void testScanTimestampUpdated() throws Exception { scanner.runIteration(); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java index c4e82714ed3..2816ce80bed 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java @@ -27,7 +27,9 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.atMostOnce; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -40,9 +42,11 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult; +import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; @@ -125,6 +129,17 @@ public void testUnhealthyContainersDetected() throws Exception { verifyContainerMarkedUnhealthy(openContainer, never()); } + @Test + @Override + public void testUnhealthyContainersTriggersVolumeScan() throws Exception { + try (MockedStatic mockedStatic = mockStatic(StorageVolumeUtil.class)) { + scanner.runIteration(); + verifyContainerMarkedUnhealthy(openCorruptMetadata, atLeastOnce()); + mockedStatic.verify(() -> + StorageVolumeUtil.onFailure(openCorruptMetadata.getContainerData().getVolume()), times(1)); + } + } + @Test @Override public void testUnhealthyContainerNotRescanned() throws Exception { diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java index 90be5b24c78..0dd160ffb96 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java @@ -29,6 +29,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.File; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -122,6 +123,9 @@ public abstract void testPreviouslyScannedContainerIsScanned() @Test public abstract void testUnhealthyContainerNotRescanned() throws Exception; + @Test + public abstract void testUnhealthyContainersTriggersVolumeScan() throws Exception; + // HELPER METHODS protected void setScannedTimestampOld(Container container) { @@ -187,6 +191,11 @@ protected void setContainers(Container... containers) { } private ContainerController mockContainerController() { + + File volLocation = mock(File.class); + when(volLocation.getPath()).thenReturn("/temp/volume-testcontainerscanner"); + when(vol.getStorageDir()).thenReturn(volLocation); + // healthy container ContainerTestUtils.setupMockContainer(healthy, true, ScanResult.healthy(), ScanResult.healthy(), diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerDataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerDataScanner.java index 207b140edb4..f001426a7a8 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerDataScanner.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerDataScanner.java @@ -29,6 +29,7 @@ import static org.mockito.Mockito.atMostOnce; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -45,6 +46,7 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult; +import org.apache.ozone.test.GenericTestUtils.LogCapturer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -290,6 +292,16 @@ public void testUnhealthyContainerNotRescanned() throws Exception { assertEquals(0, metrics.getNumUnHealthyContainers()); } + @Test + @Override + public void testUnhealthyContainersTriggersVolumeScan() throws Exception { + LogCapturer logCapturer = LogCapturer.captureLogs( + OnDemandContainerDataScanner.class); + scanContainer(corruptData); + verifyContainerMarkedUnhealthy(corruptData, times(1)); + assertTrue(logCapturer.getOutput().contains("Triggering a volume scan for volume")); + } + private void scanContainer(Container container) throws Exception { OnDemandContainerDataScanner.init(conf, controller); Optional> scanFuture = From d75ab188cccf381a171881f9c26dea2fc7783598 Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Tue, 8 Jul 2025 11:14:07 +0530 Subject: [PATCH 02/10] fix implementation after merging master --- .../ozoneimpl/BackgroundContainerDataScanner.java | 1 - .../BackgroundContainerMetadataScanner.java | 1 + .../container/ozoneimpl/ContainerScanHelper.java | 14 ++++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java index a71c8e76913..8b1da664159 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hdfs.util.Canceler; import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.ozone.container.common.interfaces.Container; -import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java index cb57fc1929f..2a33a11e712 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java @@ -67,6 +67,7 @@ public void scanContainer(Container container) } if (result.hasErrors()) { scanHelper.handleUnhealthyScanResult(containerID, result); + scanHelper.triggerVolumeScan(container.getContainerData()); } // Do not update the scan timestamp after the scan since this was just a diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java index 6cd7b80f02f..32e8c98a6c7 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java @@ -26,6 +26,7 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.ScanResult; +import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.slf4j.Logger; @@ -76,6 +77,7 @@ public void scanData(Container container, DataTransferThrottler throttler, Ca } if (result.hasErrors()) { handleUnhealthyScanResult(containerId, result); + triggerVolumeScan(containerData); } metrics.incNumContainersScanned(); } @@ -107,6 +109,18 @@ public void handleUnhealthyScanResult(long containerID, ScanResult result) throw } } + public void triggerVolumeScan(ContainerData containerData) { + log.info("Triggering a volume scan for volume [{}] as unhealthy container [{}] was on it.", + containerData.getVolume().getStorageDir().getPath(), containerData.getContainerID()); + HddsVolume volume = containerData.getVolume(); + if (volume != null && !volume.isFailed()) { + StorageVolumeUtil.onFailure(volume); + } else { + log.warn("Cannot trigger volume scan for container {} since its volume is null or has failed.", + containerData.getContainerID()); + } + } + public boolean shouldScanMetadata(Container container) { if (container == null) { return false; From 5e01b91fb8bbe1c3ecddd9952a4fd1b5be3b5a59 Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Thu, 10 Jul 2025 15:27:20 +0530 Subject: [PATCH 03/10] Move null check to before log --- .../hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java index 32e8c98a6c7..ada5f92c735 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java @@ -110,10 +110,10 @@ public void handleUnhealthyScanResult(long containerID, ScanResult result) throw } public void triggerVolumeScan(ContainerData containerData) { - log.info("Triggering a volume scan for volume [{}] as unhealthy container [{}] was on it.", - containerData.getVolume().getStorageDir().getPath(), containerData.getContainerID()); HddsVolume volume = containerData.getVolume(); if (volume != null && !volume.isFailed()) { + log.info("Triggering a volume scan for volume [{}] as unhealthy container [{}] was on it.", + volume.getStorageDir().getPath(), containerData.getContainerID()); StorageVolumeUtil.onFailure(volume); } else { log.warn("Cannot trigger volume scan for container {} since its volume is null or has failed.", From 15bdc216a6ba7e22ca6b22f4644e2d66ed0c57ab Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Mon, 14 Jul 2025 11:22:14 +0530 Subject: [PATCH 04/10] rewrite test, fix logs and other comments --- .../BackgroundContainerMetadataScanner.java | 8 +++---- .../ozoneimpl/ContainerScanHelper.java | 20 +++++++++------- .../TestBackgroundContainerDataScanner.java | 24 ++++++++++++------- .../TestContainerScannersAbstract.java | 9 +++++++ 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java index 2a33a11e712..d902803c16f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.Iterator; +import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,16 +59,15 @@ public void scanContainer(Container container) return; } - long containerID = container.getContainerData().getContainerID(); + ContainerData containerData = container.getContainerData(); MetadataScanResult result = container.scanMetaData(); if (result.isDeleted()) { - LOG.debug("Container [{}] has been deleted during the metadata scan.", containerID); + LOG.debug("Container [{}] has been deleted during the metadata scan.", containerData.getContainerID()); return; } if (result.hasErrors()) { - scanHelper.handleUnhealthyScanResult(containerID, result); - scanHelper.triggerVolumeScan(container.getContainerData()); + scanHelper.handleUnhealthyScanResult(containerData, result); } // Do not update the scan timestamp after the scan since this was just a diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java index ada5f92c735..4d9c9b3f898 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java @@ -76,8 +76,7 @@ public void scanData(Container container, DataTransferThrottler throttler, Ca log.warn("Failed to update container checksum after scan of container {}", containerId, ex); } if (result.hasErrors()) { - handleUnhealthyScanResult(containerId, result); - triggerVolumeScan(containerData); + handleUnhealthyScanResult(containerData, result); } metrics.incNumContainersScanned(); } @@ -90,8 +89,8 @@ public void scanData(Container container, DataTransferThrottler throttler, Ca logScanCompleted(containerData, now); } - public void handleUnhealthyScanResult(long containerID, ScanResult result) throws IOException { - + public void handleUnhealthyScanResult(ContainerData containerData, ScanResult result) throws IOException { + long containerID = containerData.getContainerID(); log.error("Corruption detected in container [{}]. Marking it UNHEALTHY. {}", containerID, result); if (log.isDebugEnabled()) { StringBuilder allErrorString = new StringBuilder(); @@ -107,17 +106,22 @@ public void handleUnhealthyScanResult(long containerID, ScanResult result) throw if (containerMarkedUnhealthy) { metrics.incNumUnHealthyContainers(); } + // triggering a volume scan for the unhealthy container + triggerVolumeScan(containerData); } public void triggerVolumeScan(ContainerData containerData) { HddsVolume volume = containerData.getVolume(); if (volume != null && !volume.isFailed()) { - log.info("Triggering a volume scan for volume [{}] as unhealthy container [{}] was on it.", - volume.getStorageDir().getPath(), containerData.getContainerID()); + log.info("Triggering scan of volume [{}] with unhealthy container [{}]", + volume, containerData.getContainerID()); StorageVolumeUtil.onFailure(volume); - } else { - log.warn("Cannot trigger volume scan for container {} since its volume is null or has failed.", + } else if (volume == null) { + log.warn("Cannot trigger volume scan for container {} since its volume is null", containerData.getContainerID()); + } else { + log.debug("Skipping volume scan for container {} since its volume {} has failed.", + containerData.getContainerID(), volume); } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java index 835f50e544a..5e46fbd5b08 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java @@ -32,13 +32,15 @@ import static org.mockito.Mockito.atMostOnce; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.time.Duration; import java.util.Arrays; import java.util.Optional; @@ -49,11 +51,9 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.interfaces.Container; -import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; @@ -139,11 +139,19 @@ public void testUnhealthyContainersDetected() throws Exception { @Test @Override public void testUnhealthyContainersTriggersVolumeScan() throws Exception { - try (MockedStatic mockedStatic = mockStatic(StorageVolumeUtil.class)) { - scanner.runIteration(); - verifyContainerMarkedUnhealthy(corruptData, atLeastOnce()); - mockedStatic.verify(() -> StorageVolumeUtil.onFailure(corruptData.getContainerData().getVolume()), times(1)); - } + // Replace scanHelper with a spy to verify the unhealthy scan result handling. + Field field = BackgroundContainerDataScanner.class.getDeclaredField("scanHelper"); + field.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + ContainerScanHelper originalScanHelper = (ContainerScanHelper) field.get(scanner); + ContainerScanHelper spyScanHelper = spy(originalScanHelper); + field.set(scanner, spyScanHelper); + + scanner.runIteration(); + verifyContainerMarkedUnhealthy(corruptData, atLeastOnce()); + verify(spyScanHelper, atLeastOnce()).triggerVolumeScan(corruptData.getContainerData()); } @Test diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java index ed18fc77eff..c6133a31a14 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java @@ -40,6 +40,8 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; + +import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult; import org.apache.hadoop.ozone.container.common.ContainerTestUtils; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.interfaces.Container; @@ -206,6 +208,13 @@ private ContainerController mockContainerController() { File volLocation = mock(File.class); when(volLocation.getPath()).thenReturn("/temp/volume-testcontainerscanner"); when(vol.getStorageDir()).thenReturn(volLocation); + when(vol.isFailed()).thenReturn(false); + try { + when(vol.check(any())).thenReturn(VolumeCheckResult.HEALTHY); + } catch (Exception e) { + // This is a mock, so this should never throw an exception. + throw new RuntimeException("Unexpected exception in mock setup " + e, e); + } // healthy container ContainerTestUtils.setupMockContainer(healthy, From ae88309ca41cbf50c19cbbae98106e5a497bcb64 Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Mon, 14 Jul 2025 11:36:52 +0530 Subject: [PATCH 05/10] fix merge issues --- .../container/ozoneimpl/TestOnDemandContainerScanner.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java index 67ce8183e48..887a638bbee 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java @@ -351,11 +351,10 @@ public void testMerkleTreeWritten() throws Exception { @Test @Override public void testUnhealthyContainersTriggersVolumeScan() throws Exception { - LogCapturer logCapturer = LogCapturer.captureLogs( - OnDemandContainerDataScanner.class); + LogCapturer logCapturer = LogCapturer.captureLogs(OnDemandContainerScanner.class); scanContainer(corruptData); verifyContainerMarkedUnhealthy(corruptData, times(1)); - assertTrue(logCapturer.getOutput().contains("Triggering a volume scan for volume")); + assertTrue(logCapturer.getOutput().contains("Triggering scan of volume")); } private void scanContainer(Container container) throws Exception { From af87c8ec85717cc72eaa42d8339c595d3515603e Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Mon, 14 Jul 2025 11:51:42 +0530 Subject: [PATCH 06/10] fix tests --- .../TestBackgroundContainerDataScanner.java | 14 +------- ...estBackgroundContainerMetadataScanner.java | 14 +++----- .../TestContainerScannersAbstract.java | 33 ++++++++++++++----- 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java index 5e46fbd5b08..4425032a039 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java @@ -33,14 +33,11 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import java.time.Duration; import java.util.Arrays; import java.util.Optional; @@ -139,16 +136,7 @@ public void testUnhealthyContainersDetected() throws Exception { @Test @Override public void testUnhealthyContainersTriggersVolumeScan() throws Exception { - // Replace scanHelper with a spy to verify the unhealthy scan result handling. - Field field = BackgroundContainerDataScanner.class.getDeclaredField("scanHelper"); - field.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - ContainerScanHelper originalScanHelper = (ContainerScanHelper) field.get(scanner); - ContainerScanHelper spyScanHelper = spy(originalScanHelper); - field.set(scanner, spyScanHelper); - + ContainerScanHelper spyScanHelper = replaceScanHelperWithSpy(scanner, true); scanner.runIteration(); verifyContainerMarkedUnhealthy(corruptData, atLeastOnce()); verify(spyScanHelper, atLeastOnce()).triggerVolumeScan(corruptData.getContainerData()); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java index 1e0535cc413..9e02925f9fd 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java @@ -32,9 +32,7 @@ import static org.mockito.Mockito.atMost; import static org.mockito.Mockito.atMostOnce; import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -47,11 +45,9 @@ import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.ozone.container.common.interfaces.Container; -import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; @@ -137,12 +133,10 @@ public void testUnhealthyContainersDetected() throws Exception { @Test @Override public void testUnhealthyContainersTriggersVolumeScan() throws Exception { - try (MockedStatic mockedStatic = mockStatic(StorageVolumeUtil.class)) { - scanner.runIteration(); - verifyContainerMarkedUnhealthy(openCorruptMetadata, atLeastOnce()); - mockedStatic.verify(() -> - StorageVolumeUtil.onFailure(openCorruptMetadata.getContainerData().getVolume()), times(1)); - } + ContainerScanHelper spyScanHelper = replaceScanHelperWithSpy(scanner, false); + scanner.runIteration(); + verifyContainerMarkedUnhealthy(openCorruptMetadata, atLeastOnce()); + verify(spyScanHelper, atLeastOnce()).triggerVolumeScan(openCorruptMetadata.getContainerData()); } @Test diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java index c6133a31a14..d14bd22501e 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java @@ -28,10 +28,13 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.File; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -40,8 +43,6 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; - -import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult; import org.apache.hadoop.ozone.container.common.ContainerTestUtils; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.interfaces.Container; @@ -198,6 +199,27 @@ protected void setContainers(Container... containers) { when(controller.getContainers()).thenReturn(this.containers); } + /** + * Replace scanHelper with a spy to perform verifications. + */ + protected ContainerScanHelper replaceScanHelperWithSpy(Object scanner, boolean isData) + throws NoSuchFieldException, IllegalAccessException { + Field field ; + if (isData) { + field = BackgroundContainerDataScanner.class.getDeclaredField("scanHelper"); + } else { + field = BackgroundContainerMetadataScanner.class.getDeclaredField("scanHelper"); + } + field.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + ContainerScanHelper originalScanHelper = (ContainerScanHelper) field.get(scanner); + ContainerScanHelper spyScanHelper = spy(originalScanHelper); + field.set(scanner, spyScanHelper); + return spyScanHelper; + } + private ContainerController mockContainerController() { DataScanResult healthyData = getHealthyDataScanResult(); DataScanResult unhealthyData = getUnhealthyDataScanResult(); @@ -208,13 +230,6 @@ private ContainerController mockContainerController() { File volLocation = mock(File.class); when(volLocation.getPath()).thenReturn("/temp/volume-testcontainerscanner"); when(vol.getStorageDir()).thenReturn(volLocation); - when(vol.isFailed()).thenReturn(false); - try { - when(vol.check(any())).thenReturn(VolumeCheckResult.HEALTHY); - } catch (Exception e) { - // This is a mock, so this should never throw an exception. - throw new RuntimeException("Unexpected exception in mock setup " + e, e); - } // healthy container ContainerTestUtils.setupMockContainer(healthy, From 60ed84daf5346b529f6e76f152a66a234cbab538 Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Mon, 14 Jul 2025 11:54:17 +0530 Subject: [PATCH 07/10] fix tests --- .../container/ozoneimpl/TestContainerScannersAbstract.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java index d14bd22501e..9ce021f74f3 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java @@ -204,7 +204,7 @@ protected void setContainers(Container... containers) { */ protected ContainerScanHelper replaceScanHelperWithSpy(Object scanner, boolean isData) throws NoSuchFieldException, IllegalAccessException { - Field field ; + Field field; if (isData) { field = BackgroundContainerDataScanner.class.getDeclaredField("scanHelper"); } else { From a249fa904032821ea631d73129fc3b54ff773d13 Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Mon, 14 Jul 2025 13:01:46 +0530 Subject: [PATCH 08/10] remove final modifier changes in test --- .../container/ozoneimpl/TestContainerScannersAbstract.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java index 9ce021f74f3..67a850bbb2f 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java @@ -211,9 +211,6 @@ protected ContainerScanHelper replaceScanHelperWithSpy(Object scanner, boolean i field = BackgroundContainerMetadataScanner.class.getDeclaredField("scanHelper"); } field.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); ContainerScanHelper originalScanHelper = (ContainerScanHelper) field.get(scanner); ContainerScanHelper spyScanHelper = spy(originalScanHelper); field.set(scanner, spyScanHelper); From 2debf207039cb5a61124932233670b87e28ef441 Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Mon, 14 Jul 2025 13:45:22 +0530 Subject: [PATCH 09/10] remove final modifier changes in test --- .../ozone/container/ozoneimpl/TestContainerScannersAbstract.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java index 67a850bbb2f..f05ce4d84d9 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java @@ -34,7 +34,6 @@ import java.io.File; import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; From ad2e379929deb505dc3bf229441cc7ff5b383384 Mon Sep 17 00:00:00 2001 From: tejaskriya Date: Fri, 18 Jul 2025 13:59:04 +0530 Subject: [PATCH 10/10] revert test changes, move trigger of vol scan --- .../ozoneimpl/ContainerScanHelper.java | 4 ++-- .../TestBackgroundContainerDataScanner.java | 14 +++++++++---- ...estBackgroundContainerMetadataScanner.java | 16 ++++++++++---- .../TestContainerScannersAbstract.java | 21 ------------------- .../TestOnDemandContainerScanner.java | 1 + 5 files changed, 25 insertions(+), 31 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java index b75bb026649..4c4a45c55d4 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java @@ -131,9 +131,9 @@ public void handleUnhealthyScanResult(ContainerData containerData, ScanResult re boolean containerMarkedUnhealthy = controller.markContainerUnhealthy(containerID, result); if (containerMarkedUnhealthy) { metrics.incNumUnHealthyContainers(); + // triggering a volume scan for the unhealthy container + triggerVolumeScan(containerData); } - // triggering a volume scan for the unhealthy container - triggerVolumeScan(containerData); } public void triggerVolumeScan(ContainerData containerData) { diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java index 4425032a039..508c472a7c8 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerDataScanner.java @@ -32,6 +32,7 @@ import static org.mockito.Mockito.atMostOnce; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -48,9 +49,12 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.interfaces.Container; +import org.apache.hadoop.ozone.container.common.interfaces.ScanResult; +import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; @@ -136,10 +140,12 @@ public void testUnhealthyContainersDetected() throws Exception { @Test @Override public void testUnhealthyContainersTriggersVolumeScan() throws Exception { - ContainerScanHelper spyScanHelper = replaceScanHelperWithSpy(scanner, true); - scanner.runIteration(); - verifyContainerMarkedUnhealthy(corruptData, atLeastOnce()); - verify(spyScanHelper, atLeastOnce()).triggerVolumeScan(corruptData.getContainerData()); + when(controller.markContainerUnhealthy(anyLong(), any(ScanResult.class))).thenReturn(true); + try (MockedStatic mockedStatic = mockStatic(StorageVolumeUtil.class)) { + scanner.runIteration(); + verifyContainerMarkedUnhealthy(corruptData, atLeastOnce()); + mockedStatic.verify(() -> StorageVolumeUtil.onFailure(corruptData.getContainerData().getVolume()), times(1)); + } } @Test diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java index 9e02925f9fd..9b6c6aed3f0 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestBackgroundContainerMetadataScanner.java @@ -32,7 +32,9 @@ import static org.mockito.Mockito.atMost; import static org.mockito.Mockito.atMostOnce; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -45,9 +47,12 @@ import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.ozone.container.common.interfaces.Container; +import org.apache.hadoop.ozone.container.common.interfaces.ScanResult; +import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; @@ -133,10 +138,13 @@ public void testUnhealthyContainersDetected() throws Exception { @Test @Override public void testUnhealthyContainersTriggersVolumeScan() throws Exception { - ContainerScanHelper spyScanHelper = replaceScanHelperWithSpy(scanner, false); - scanner.runIteration(); - verifyContainerMarkedUnhealthy(openCorruptMetadata, atLeastOnce()); - verify(spyScanHelper, atLeastOnce()).triggerVolumeScan(openCorruptMetadata.getContainerData()); + when(controller.markContainerUnhealthy(anyLong(), any(ScanResult.class))).thenReturn(true); + try (MockedStatic mockedStatic = mockStatic(StorageVolumeUtil.class)) { + scanner.runIteration(); + verifyContainerMarkedUnhealthy(openCorruptMetadata, atLeastOnce()); + mockedStatic.verify(() -> + StorageVolumeUtil.onFailure(openCorruptMetadata.getContainerData().getVolume()), times(1)); + } } @Test diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java index f05ce4d84d9..7bd45c3b503 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScannersAbstract.java @@ -28,12 +28,10 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.File; -import java.lang.reflect.Field; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -198,31 +196,12 @@ protected void setContainers(Container... containers) { when(controller.getContainers()).thenReturn(this.containers); } - /** - * Replace scanHelper with a spy to perform verifications. - */ - protected ContainerScanHelper replaceScanHelperWithSpy(Object scanner, boolean isData) - throws NoSuchFieldException, IllegalAccessException { - Field field; - if (isData) { - field = BackgroundContainerDataScanner.class.getDeclaredField("scanHelper"); - } else { - field = BackgroundContainerMetadataScanner.class.getDeclaredField("scanHelper"); - } - field.setAccessible(true); - ContainerScanHelper originalScanHelper = (ContainerScanHelper) field.get(scanner); - ContainerScanHelper spyScanHelper = spy(originalScanHelper); - field.set(scanner, spyScanHelper); - return spyScanHelper; - } - private ContainerController mockContainerController() { DataScanResult healthyData = getHealthyDataScanResult(); DataScanResult unhealthyData = getUnhealthyDataScanResult(); MetadataScanResult healthyMetadata = getHealthyMetadataScanResult(); MetadataScanResult unhealthyMetadata = getUnhealthyMetadataScanResult(); - File volLocation = mock(File.class); when(volLocation.getPath()).thenReturn("/temp/volume-testcontainerscanner"); when(vol.getStorageDir()).thenReturn(volLocation); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java index 887a638bbee..8bd8f2060ba 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java @@ -351,6 +351,7 @@ public void testMerkleTreeWritten() throws Exception { @Test @Override public void testUnhealthyContainersTriggersVolumeScan() throws Exception { + when(controller.markContainerUnhealthy(anyLong(), any(ScanResult.class))).thenReturn(true); LogCapturer logCapturer = LogCapturer.captureLogs(OnDemandContainerScanner.class); scanContainer(corruptData); verifyContainerMarkedUnhealthy(corruptData, times(1));