HDDS-8326. Container Level Info - OM DB Insights.

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java

@@ -17,48 +17,34 @@
  */
 package org.apache.hadoop.ozone.recon.api;
 
-import java.io.IOException;
-import java.time.Instant;
-import java.util.Collection;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import javax.inject.Inject;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
 import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.recon.api.types.ContainerDiscrepancyInfo;
 import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
 import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
 import org.apache.hadoop.ozone.recon.api.types.ContainersResponse;
 import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
+import org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
 import org.apache.hadoop.ozone.recon.api.types.KeysResponse;
 import org.apache.hadoop.ozone.recon.api.types.MissingContainerMetadata;
 import org.apache.hadoop.ozone.recon.api.types.MissingContainersResponse;
 import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata;
-import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersSummary;
 import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse;
-import org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
-import org.apache.hadoop.ozone.recon.persistence.ContainerHistory;
+import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersSummary;
 import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager;
+import org.apache.hadoop.ozone.recon.persistence.ContainerHistory;
 import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
 import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
 import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager;
@@ -68,6 +54,26 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.inject.Inject;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
 import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_BATCH_NUMBER;
 import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_FETCH_COUNT;
 import static org.apache.hadoop.ozone.recon.ReconConstants.PREV_CONTAINER_ID_DEFAULT_VALUE;
@@ -91,6 +97,7 @@ public class ContainerEndpoint {
   private ReconOMMetadataManager omMetadataManager;
 
   private final ReconContainerManager containerManager;
+  private final PipelineManager pipelineManager;
   private final ContainerHealthSchemaManager containerHealthSchemaManager;
   private final ReconNamespaceSummaryManager reconNamespaceSummaryManager;
   private final OzoneStorageContainerManager reconSCM;
@@ -104,6 +111,7 @@ public ContainerEndpoint(OzoneStorageContainerManager reconSCM,
                ReconNamespaceSummaryManager reconNamespaceSummaryManager) {
     this.containerManager =
         (ReconContainerManager) reconSCM.getContainerManager();
+    this.pipelineManager = reconSCM.getPipelineManager();
     this.containerHealthSchemaManager = containerHealthSchemaManager;
     this.reconNamespaceSummaryManager = reconNamespaceSummaryManager;
     this.reconSCM = reconSCM;
@@ -417,4 +425,86 @@ private List<ContainerBlockMetadata> getBlocks(
   private BucketLayout getBucketLayout() {
     return BucketLayout.DEFAULT;
   }
+
+  @GET
+  @Path("/mismatch")
+  public Response getContainerMisMatchInsights() {
+    List<ContainerDiscrepancyInfo> containerDiscrepancyInfoList =
+        new ArrayList<>();
+    try {
+      Map<Long, ContainerMetadata> omContainers =
+          reconContainerMetadataManager.getContainers(-1, -1);
+      List<Long> scmAllContainers = containerManager.getContainers().stream()
+          .filter(containerInfo -> !(containerInfo.getState() ==
+              HddsProtos.LifeCycleState.DELETED))
+          .map(containerInfo -> containerInfo.getContainerID()).collect(
+              Collectors.toList());
+      List<Long> scmNonDeletedContainers =
+          containerManager.getContainers().stream()
+              .filter(containerInfo -> !(containerInfo.getState() ==
+                  HddsProtos.LifeCycleState.DELETED))
+              .map(containerInfo -> containerInfo.getContainerID()).collect(
+                  Collectors.toList());
+
+      // Filter list of container Ids which are present in OM but not in SCM.
+      List<Map.Entry<Long, ContainerMetadata>> notSCMContainers =
+          omContainers.entrySet().stream().filter(containerMetadataEntry ->
+                  !(scmAllContainers.contains(containerMetadataEntry.getKey())))
+              .collect(
+                  Collectors.toList());
+
+      notSCMContainers.forEach(nonSCMContainer -> {
+        ContainerDiscrepancyInfo containerDiscrepancyInfo =
+            new ContainerDiscrepancyInfo();
+        containerDiscrepancyInfo.setContainerID(nonSCMContainer.getKey());
+        containerDiscrepancyInfo.setNumberOfKeys(
+            nonSCMContainer.getValue().getNumberOfKeys());
+        containerDiscrepancyInfo.setPipelines(nonSCMContainer.getValue()
+            .getPipelines());
+        containerDiscrepancyInfo.setExistsAt("OM");
+        containerDiscrepancyInfoList.add(containerDiscrepancyInfo);
+      });
+
+      // Filter list of container Ids which are present in SCM but not in OM.
+      List<Long> nonOMContainers = scmNonDeletedContainers.stream()
+          .filter(containerId -> !omContainers.containsKey(containerId))
+          .collect(Collectors.toList());
+
+      List<Pipeline> pipelines = new ArrayList<>();
+      nonOMContainers.forEach(nonOMContainerId -> {
+        ContainerDiscrepancyInfo containerDiscrepancyInfo =
+            new ContainerDiscrepancyInfo();
+        containerDiscrepancyInfo.setContainerID(nonOMContainerId);
+        containerDiscrepancyInfo.setNumberOfKeys(0);
+        PipelineID pipelineID = null;
+        try {
+          pipelineID = containerManager.getContainer(
+                  ContainerID.valueOf(nonOMContainerId))
+              .getPipelineID();
+
+          if (null != pipelineID) {
+            pipelines.add(pipelineManager.getPipeline(pipelineID));
+          }
+        } catch (ContainerNotFoundException e) {
+          LOG.warn("Container {} not found in SCM: {}", nonOMContainerId, e);
+        } catch (PipelineNotFoundException e) {
+          LOG.debug("Pipeline not found for container: {} and pipelineId: {}",
+              nonOMContainerId, pipelineID, e);
+        }
+        containerDiscrepancyInfo.setPipelines(pipelines);
+        containerDiscrepancyInfo.setExistsAt("SCM");
+        containerDiscrepancyInfoList.add(containerDiscrepancyInfo);
+      });
+
+    } catch (IOException ex) {
+      throw new WebApplicationException(ex,
+          Response.Status.INTERNAL_SERVER_ERROR);
+    } catch (IllegalArgumentException e) {
+      throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
+    } catch (Exception ex) {
+      throw new WebApplicationException(ex,
+          Response.Status.INTERNAL_SERVER_ERROR);
+    }
+    return Response.ok(containerDiscrepancyInfoList).build();
+  }
 }

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerDiscrepancyInfo.java

@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.api.types;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+
+import java.util.List;
+
+/**
+ * Metadata object that represents a Container Discrepancy Info.
+ */
+public class ContainerDiscrepancyInfo {
+
+  @JsonProperty("containerId")
+  private long containerID;
+
+  @JsonProperty("omContainerState")
+  @JsonInclude(JsonInclude.Include.NON_EMPTY)
+  private long omContainerState;
+
+  @JsonProperty("scmContainerState")
+  @JsonInclude(JsonInclude.Include.NON_EMPTY)
+  private long scmContainerState;
+
+  @JsonProperty("numberOfKeys")
+  private long numberOfKeys;
+
+  @JsonProperty("pipelines")
+  private List<Pipeline> pipelines;
+
+  @JsonProperty("existsAt")
+  private String existsAt;
+
+  public ContainerDiscrepancyInfo() {
+
+  }
+
+  public long getContainerID() {
+    return containerID;
+  }
+
+  public void setContainerID(long containerID) {
+    this.containerID = containerID;
+  }
+
+  public long getNumberOfKeys() {
+    return numberOfKeys;
+  }
+
+  public void setNumberOfKeys(long numberOfKeys) {
+    this.numberOfKeys = numberOfKeys;
+  }
+
+  public long getOmContainerState() {
+    return omContainerState;
+  }
+
+  public void setOmContainerState(long omContainerState) {
+    this.omContainerState = omContainerState;
+  }
+
+  public long getScmContainerState() {
+    return scmContainerState;
+  }
+
+  public void setScmContainerState(long scmContainerState) {
+    this.scmContainerState = scmContainerState;
+  }
+
+  public List<Pipeline> getPipelines() {
+    return pipelines;
+  }
+
+  public void setPipelines(
+      List<Pipeline> pipelines) {
+    this.pipelines = pipelines;
+  }
+
+  public String getExistsAt() {
+    return existsAt;
+  }
+
+  public void setExistsAt(String existsAt) {
+    this.existsAt = existsAt;
+  }
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerMetadata.java

@@ -17,9 +17,13 @@
  */
 package org.apache.hadoop.ozone.recon.api.types;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
+import java.util.List;
 
 /**
  * Metadata object that represents a Container.
@@ -33,6 +37,9 @@ public class ContainerMetadata {
   @XmlElement(name = "NumberOfKeys")
   private long numberOfKeys;
 
+  @JsonProperty("pipelines")
+  private List<Pipeline> pipelines;
+
   public ContainerMetadata(long containerID) {
     this.containerID = containerID;
   }
@@ -53,4 +60,12 @@ public void setNumberOfKeys(long numberOfKeys) {
     this.numberOfKeys = numberOfKeys;
   }
 
+  public List<Pipeline> getPipelines() {
+    return pipelines;
+  }
+
+  public void setPipelines(
+      List<Pipeline> pipelines) {
+    this.pipelines = pipelines;
+  }
 }