diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java index 5fe8a657cf12..5990435b6c0e 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java @@ -21,11 +21,17 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.recon.api.types.ContainerDiscrepancyInfo; import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata; import org.apache.hadoop.ozone.recon.api.types.ContainersResponse; @@ -92,6 +98,7 @@ public class ContainerEndpoint { private ReconOMMetadataManager omMetadataManager; private final ReconContainerManager containerManager; + private final PipelineManager pipelineManager; private final ContainerHealthSchemaManager containerHealthSchemaManager; private final ReconNamespaceSummaryManager reconNamespaceSummaryManager; private final OzoneStorageContainerManager reconSCM; @@ -105,6 +112,7 @@ public ContainerEndpoint(OzoneStorageContainerManager reconSCM, ReconNamespaceSummaryManager reconNamespaceSummaryManager) { this.containerManager = (ReconContainerManager) reconSCM.getContainerManager(); + this.pipelineManager = reconSCM.getPipelineManager(); this.containerHealthSchemaManager = containerHealthSchemaManager; this.reconNamespaceSummaryManager = reconNamespaceSummaryManager; this.reconSCM = reconSCM; @@ -481,4 +489,81 @@ private List getBlocks( return blockIds; } + @GET + @Path("/mismatch") + public Response getContainerMisMatchInsights() { + List containerDiscrepancyInfoList = + new ArrayList<>(); + try { + Map omContainers = + reconContainerMetadataManager.getContainers(-1, -1); + List scmNonDeletedContainers = + containerManager.getContainers().stream() + .filter(containerInfo -> !(containerInfo.getState() == + HddsProtos.LifeCycleState.DELETED)) + .map(containerInfo -> containerInfo.getContainerID()).collect( + Collectors.toList()); + + // Filter list of container Ids which are present in OM but not in SCM. + List> notSCMContainers = + omContainers.entrySet().stream().filter(containerMetadataEntry -> + !(scmNonDeletedContainers.contains( + containerMetadataEntry.getKey()))) + .collect( + Collectors.toList()); + + notSCMContainers.forEach(nonSCMContainer -> { + ContainerDiscrepancyInfo containerDiscrepancyInfo = + new ContainerDiscrepancyInfo(); + containerDiscrepancyInfo.setContainerID(nonSCMContainer.getKey()); + containerDiscrepancyInfo.setNumberOfKeys( + nonSCMContainer.getValue().getNumberOfKeys()); + containerDiscrepancyInfo.setPipelines(nonSCMContainer.getValue() + .getPipelines()); + containerDiscrepancyInfo.setExistsAt("OM"); + containerDiscrepancyInfoList.add(containerDiscrepancyInfo); + }); + + // Filter list of container Ids which are present in SCM but not in OM. + List nonOMContainers = scmNonDeletedContainers.stream() + .filter(containerId -> !omContainers.containsKey(containerId)) + .collect(Collectors.toList()); + + List pipelines = new ArrayList<>(); + nonOMContainers.forEach(nonOMContainerId -> { + ContainerDiscrepancyInfo containerDiscrepancyInfo = + new ContainerDiscrepancyInfo(); + containerDiscrepancyInfo.setContainerID(nonOMContainerId); + containerDiscrepancyInfo.setNumberOfKeys(0); + PipelineID pipelineID = null; + try { + pipelineID = containerManager.getContainer( + ContainerID.valueOf(nonOMContainerId)) + .getPipelineID(); + + if (null != pipelineID) { + pipelines.add(pipelineManager.getPipeline(pipelineID)); + } + } catch (ContainerNotFoundException e) { + LOG.warn("Container {} not found in SCM: {}", nonOMContainerId, e); + } catch (PipelineNotFoundException e) { + LOG.debug("Pipeline not found for container: {} and pipelineId: {}", + nonOMContainerId, pipelineID, e); + } + containerDiscrepancyInfo.setPipelines(pipelines); + containerDiscrepancyInfo.setExistsAt("SCM"); + containerDiscrepancyInfoList.add(containerDiscrepancyInfo); + }); + + } catch (IOException ex) { + throw new WebApplicationException(ex, + Response.Status.INTERNAL_SERVER_ERROR); + } catch (IllegalArgumentException e) { + throw new WebApplicationException(e, Response.Status.BAD_REQUEST); + } catch (Exception ex) { + throw new WebApplicationException(ex, + Response.Status.INTERNAL_SERVER_ERROR); + } + return Response.ok(containerDiscrepancyInfoList).build(); + } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerDiscrepancyInfo.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerDiscrepancyInfo.java new file mode 100644 index 000000000000..33702bf3a050 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerDiscrepancyInfo.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.recon.api.types; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; + +import java.util.List; + +/** + * Metadata object that represents a Container Discrepancy Info. + */ +public class ContainerDiscrepancyInfo { + + @JsonProperty("containerId") + private long containerID; + + @JsonProperty("numberOfKeys") + private long numberOfKeys; + + @JsonProperty("pipelines") + private List pipelines; + + @JsonProperty("existsAt") + private String existsAt; + + public ContainerDiscrepancyInfo() { + + } + + public long getContainerID() { + return containerID; + } + + public void setContainerID(long containerID) { + this.containerID = containerID; + } + + public long getNumberOfKeys() { + return numberOfKeys; + } + + public void setNumberOfKeys(long numberOfKeys) { + this.numberOfKeys = numberOfKeys; + } + + public List getPipelines() { + return pipelines; + } + + public void setPipelines( + List pipelines) { + this.pipelines = pipelines; + } + + public String getExistsAt() { + return existsAt; + } + + public void setExistsAt(String existsAt) { + this.existsAt = existsAt; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerMetadata.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerMetadata.java index d4111fb5a05b..be91bd13cffd 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerMetadata.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerMetadata.java @@ -17,9 +17,13 @@ */ package org.apache.hadoop.ozone.recon.api.types; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; + import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; +import java.util.List; /** * Metadata object that represents a Container. @@ -33,6 +37,9 @@ public class ContainerMetadata { @XmlElement(name = "NumberOfKeys") private long numberOfKeys; + @JsonProperty("pipelines") + private List pipelines; + public ContainerMetadata(long containerID) { this.containerID = containerID; } @@ -53,4 +60,12 @@ public void setNumberOfKeys(long numberOfKeys) { this.numberOfKeys = numberOfKeys; } + public List getPipelines() { + return pipelines; + } + + public void setPipelines( + List pipelines) { + this.pipelines = pipelines; + } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java index 6c40afc0fdd9..6c623c70041f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java @@ -39,12 +39,16 @@ import javax.inject.Singleton; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.recon.ReconUtils; import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata; import org.apache.hadoop.ozone.recon.api.types.KeyPrefixContainer; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.scm.ContainerReplicaHistory; import org.apache.hadoop.ozone.recon.scm.ContainerReplicaHistoryList; import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; @@ -54,6 +58,7 @@ import org.apache.hadoop.hdds.utils.db.TableIterator; import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao; import org.hadoop.ozone.recon.schema.tables.pojos.GlobalStats; +import org.jetbrains.annotations.NotNull; import org.jooq.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,6 +85,9 @@ public class ReconContainerMetadataManagerImpl @Inject private Configuration sqlConfiguration; + @Inject + private ReconOMMetadataManager omMetadataManager; + @Inject public ReconContainerMetadataManagerImpl(ReconDBProvider reconDBProvider, Configuration sqlConfiguration) { @@ -455,6 +463,8 @@ public Map getContainers(int limit, ContainerKeyPrefix containerKeyPrefix = keyValue.getKey(); Long containerID = containerKeyPrefix.getContainerId(); Integer numberOfKeys = keyValue.getValue(); + List pipelines = + getPipelines(containerKeyPrefix); // break the loop if limit has been reached // and one more new entity needs to be added to the containers map @@ -469,12 +479,34 @@ public Map getContainers(int limit, ContainerMetadata containerMetadata = containers.get(containerID); containerMetadata.setNumberOfKeys(containerMetadata.getNumberOfKeys() + numberOfKeys); + containerMetadata.setPipelines(pipelines); containers.put(containerID, containerMetadata); } } return containers; } + @NotNull + private List getPipelines(ContainerKeyPrefix containerKeyPrefix) + throws IOException { + OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable(BucketLayout.LEGACY) + .getSkipCache(containerKeyPrefix.getKeyPrefix()); + if (null == omKeyInfo) { + omKeyInfo = + omMetadataManager.getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .getSkipCache(containerKeyPrefix.getKeyPrefix()); + } + List pipelines = new ArrayList<>(); + if (null != omKeyInfo) { + omKeyInfo.getKeyLocationVersions().stream().map( + omKeyLocationInfoGroup -> + omKeyLocationInfoGroup.getLocationList() + .stream().map(omKeyLocationInfo -> pipelines.add( + omKeyLocationInfo.getPipeline()))); + } + return pipelines; + } + @Override public void deleteContainerMapping(ContainerKeyPrefix containerKeyPrefix) throws IOException { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java index ee6f2c544412..96c68a710190 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -41,6 +42,8 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.api.types.ContainerDiscrepancyInfo; +import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata; import org.apache.hadoop.ozone.recon.api.types.ContainersResponse; import org.apache.hadoop.ozone.recon.api.types.DeletedContainerInfo; @@ -69,6 +72,8 @@ import org.junit.Test; import org.junit.jupiter.api.Assertions; import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; @@ -108,6 +113,10 @@ public class TestContainerEndpoint { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static final Logger LOG = + LoggerFactory.getLogger(TestContainerEndpoint.class); + private OzoneStorageContainerManager ozoneStorageContainerManager; private ReconContainerManager reconContainerManager; private ContainerStateManager containerStateManager; @@ -1163,4 +1172,53 @@ public void testGetSCMDeletedContainersPrevKeyParam() throws Exception { Assertions.assertEquals("DELETED", deletedContainerInfo.getContainerState()); } + + @Test + public void testGetContainerInsightsNonSCMContainers() + throws IOException, TimeoutException { + Map omContainers = + reconContainerMetadataManager.getContainers(-1, 0); + putContainerInfos(2); + List scmContainers = reconContainerManager.getContainers(); + assertEquals(omContainers.size(), scmContainers.size()); + // delete container Id 1 from SCM + reconContainerManager.deleteContainer(ContainerID.valueOf(1)); + Response containerInsights = + containerEndpoint.getContainerMisMatchInsights(); + List containerDiscrepancyInfoList = + (List) containerInsights.getEntity(); + ContainerDiscrepancyInfo containerDiscrepancyInfo = + containerDiscrepancyInfoList.get(0); + assertEquals(1, containerDiscrepancyInfo.getContainerID()); + assertEquals(1, containerDiscrepancyInfoList.size()); + assertEquals("OM", containerDiscrepancyInfo.getExistsAt()); + } + + @Test + public void testGetContainerInsightsNonOMContainers() + throws IOException, TimeoutException { + putContainerInfos(2); + List deletedContainerKeyList = + reconContainerMetadataManager.getKeyPrefixesForContainer(2).entrySet() + .stream().map(entry -> entry.getKey()).collect( + Collectors.toList()); + deletedContainerKeyList.forEach((ContainerKeyPrefix key) -> { + try (RDBBatchOperation rdbBatchOperation = new RDBBatchOperation()) { + reconContainerMetadataManager + .batchDeleteContainerMapping(rdbBatchOperation, key); + reconContainerMetadataManager.commitBatchOperation(rdbBatchOperation); + } catch (IOException e) { + LOG.error("Unable to write Container Key Prefix data in Recon DB.", e); + } + }); + Response containerInsights = + containerEndpoint.getContainerMisMatchInsights(); + List containerDiscrepancyInfoList = + (List) containerInsights.getEntity(); + ContainerDiscrepancyInfo containerDiscrepancyInfo = + containerDiscrepancyInfoList.get(0); + assertEquals(2, containerDiscrepancyInfo.getContainerID()); + assertEquals(1, containerDiscrepancyInfoList.size()); + assertEquals("SCM", containerDiscrepancyInfo.getExistsAt()); + } } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java index d0184df9bee0..1c744409b9ff 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java @@ -18,6 +18,8 @@ package org.apache.hadoop.ozone.recon.spi.impl; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -31,6 +33,7 @@ import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata; import org.apache.hadoop.ozone.recon.api.types.KeyPrefixContainer; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.junit.Assert; import org.junit.Before; @@ -47,6 +50,7 @@ public class TestReconContainerMetadataManagerImpl { @ClassRule public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder(); private static ReconContainerMetadataManager reconContainerMetadataManager; + private static ReconOMMetadataManager reconOMMetadataManager; private String keyPrefix1 = "V3/B1/K1"; private String keyPrefix2 = "V3/B1/K2"; @@ -54,10 +58,14 @@ public class TestReconContainerMetadataManagerImpl { @BeforeClass public static void setupOnce() throws Exception { + reconOMMetadataManager = getTestReconOmMetadataManager( + initializeNewOmMetadataManager(TEMP_FOLDER.newFolder()), + TEMP_FOLDER.newFolder()); ReconTestInjector reconTestInjector = new ReconTestInjector.Builder(TEMP_FOLDER) .withReconSqlDb() .withContainerDB() + .withReconOm(reconOMMetadataManager) .build(); reconContainerMetadataManager = reconTestInjector.getInstance(ReconContainerMetadataManager.class); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java index 8b10ab59b91a..60f52bab228f 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; import org.apache.hadoop.ozone.recon.ReconTestInjector; import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -35,6 +36,9 @@ import java.util.Map; import java.util.Set; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager; + /** * Test for NSSummary manager. */ @@ -42,15 +46,20 @@ public class TestReconNamespaceSummaryManagerImpl { @ClassRule public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder(); private static ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager; + private static ReconOMMetadataManager reconOMMetadataManager; private static int[] testBucket; private static final Set TEST_CHILD_DIR = new HashSet<>(Arrays.asList(new Long[]{1L, 2L, 3L})); @BeforeClass public static void setupOnce() throws Exception { + reconOMMetadataManager = getTestReconOmMetadataManager( + initializeNewOmMetadataManager(TEMP_FOLDER.newFolder()), + TEMP_FOLDER.newFolder()); ReconTestInjector reconTestInjector = new ReconTestInjector.Builder(TEMP_FOLDER) .withReconSqlDb() + .withReconOm(reconOMMetadataManager) .withContainerDB() .build(); reconNamespaceSummaryManager = reconTestInjector.getInstance(