HDDS-8326. Container Level Info - OM DB Insights. #4509
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,11 +21,17 @@ | |
| import org.apache.hadoop.hdds.protocol.proto.HddsProtos; | ||
| import org.apache.hadoop.hdds.scm.container.ContainerID; | ||
| import org.apache.hadoop.hdds.scm.container.ContainerInfo; | ||
| import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; | ||
| import org.apache.hadoop.hdds.scm.pipeline.Pipeline; | ||
| import org.apache.hadoop.hdds.scm.pipeline.PipelineID; | ||
| import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; | ||
| import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException; | ||
| import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; | ||
| import org.apache.hadoop.ozone.om.helpers.BucketLayout; | ||
| import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; | ||
| import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; | ||
| import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; | ||
| import org.apache.hadoop.ozone.recon.api.types.ContainerDiscrepancyInfo; | ||
| import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; | ||
| import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata; | ||
| import org.apache.hadoop.ozone.recon.api.types.ContainersResponse; | ||
|
|
@@ -92,6 +98,7 @@ public class ContainerEndpoint { | |
| private ReconOMMetadataManager omMetadataManager; | ||
|
|
||
| private final ReconContainerManager containerManager; | ||
| private final PipelineManager pipelineManager; | ||
| private final ContainerHealthSchemaManager containerHealthSchemaManager; | ||
| private final ReconNamespaceSummaryManager reconNamespaceSummaryManager; | ||
| private final OzoneStorageContainerManager reconSCM; | ||
|
|
@@ -105,6 +112,7 @@ public ContainerEndpoint(OzoneStorageContainerManager reconSCM, | |
| ReconNamespaceSummaryManager reconNamespaceSummaryManager) { | ||
| this.containerManager = | ||
| (ReconContainerManager) reconSCM.getContainerManager(); | ||
| this.pipelineManager = reconSCM.getPipelineManager(); | ||
| this.containerHealthSchemaManager = containerHealthSchemaManager; | ||
| this.reconNamespaceSummaryManager = reconNamespaceSummaryManager; | ||
| this.reconSCM = reconSCM; | ||
|
|
@@ -481,4 +489,83 @@ private List<ContainerBlockMetadata> getBlocks( | |
| return blockIds; | ||
| } | ||
|
|
||
| @GET | ||
| @Path("/mismatch") | ||
| public Response getContainerMisMatchInsights() { | ||
| List<ContainerDiscrepancyInfo> containerDiscrepancyInfoList = | ||
| new ArrayList<>(); | ||
| try { | ||
| Map<Long, ContainerMetadata> omContainers = | ||
| reconContainerMetadataManager.getContainers(-1, -1); | ||
| List<Long> scmAllContainers = containerManager.getContainers().stream() | ||
| .map(containerInfo -> containerInfo.getContainerID()).collect( | ||
| Collectors.toList()); | ||
| List<Long> scmNonDeletedContainers = | ||
| containerManager.getContainers().stream() | ||
| .filter(containerInfo -> !(containerInfo.getState() == | ||
| HddsProtos.LifeCycleState.DELETED)) | ||
| .map(containerInfo -> containerInfo.getContainerID()).collect( | ||
| Collectors.toList()); | ||
|
|
||
| // Filter list of container Ids which are present in OM but not in SCM. | ||
| List<Map.Entry<Long, ContainerMetadata>> notSCMContainers = | ||
| omContainers.entrySet().stream().filter(containerMetadataEntry -> | ||
| !(scmAllContainers.contains(containerMetadataEntry.getKey()))) | ||
|
There was a problem hiding this comment. This still looks incorrect. If a container has got Deleted in SCM and still present in OM, it will never get flagged as scmAllContainers has Deleted SCM containers as well. so no discrepancy will ever get reported.
Contributor
Author
There was a problem hiding this comment.
IMO, we want to identify mismatch of container referred in OM but present in SCM, now why the container is in deleted state in SCM, that is different issue. But any container which cannot be queried for metadata by OM to SCM is a data loss situation. @sumitagrawl - I think your 1st point is what we discussed. Let us know what discussion happened with @errose28
Contributor
There was a problem hiding this comment. @devmadhuu We should not count deleted SCM container for comparing to OM also, deleted container is just a bookmark, to avoid re-created same container at DN due to issue with sync at DN or Old DN. This needs to be removed in comparison here tool. |
||
| .collect( | ||
| Collectors.toList()); | ||
|
|
||
| notSCMContainers.forEach(nonSCMContainer -> { | ||
|
There was a problem hiding this comment. notSCMContainers List will be empty in normal scenario and only if some thing went wrong will have it populated.
Contributor
Author
There was a problem hiding this comment. foreach will not execute if list is empty, so no check required. |
||
| ContainerDiscrepancyInfo containerDiscrepancyInfo = | ||
| new ContainerDiscrepancyInfo(); | ||
| containerDiscrepancyInfo.setContainerID(nonSCMContainer.getKey()); | ||
| containerDiscrepancyInfo.setNumberOfKeys( | ||
| nonSCMContainer.getValue().getNumberOfKeys()); | ||
| containerDiscrepancyInfo.setPipelines(nonSCMContainer.getValue() | ||
| .getPipelines()); | ||
| containerDiscrepancyInfo.setExistsAt("OM"); | ||
| containerDiscrepancyInfoList.add(containerDiscrepancyInfo); | ||
| }); | ||
|
|
||
| // Filter list of container Ids which are present in SCM but not in OM. | ||
| List<Long> nonOMContainers = scmNonDeletedContainers.stream() | ||
| .filter(containerId -> !omContainers.containsKey(containerId)) | ||
| .collect(Collectors.toList()); | ||
|
|
||
| List<Pipeline> pipelines = new ArrayList<>(); | ||
| nonOMContainers.forEach(nonOMContainerId -> { | ||
|
There was a problem hiding this comment. Here again need a check if nonOMContainers empty
Contributor
Author
There was a problem hiding this comment. foreach will not execute if list is empty, so no check required. |
||
| ContainerDiscrepancyInfo containerDiscrepancyInfo = | ||
| new ContainerDiscrepancyInfo(); | ||
| containerDiscrepancyInfo.setContainerID(nonOMContainerId); | ||
| containerDiscrepancyInfo.setNumberOfKeys(0); | ||
| PipelineID pipelineID = null; | ||
| try { | ||
| pipelineID = containerManager.getContainer( | ||
| ContainerID.valueOf(nonOMContainerId)) | ||
| .getPipelineID(); | ||
|
|
||
| if (null != pipelineID) { | ||
| pipelines.add(pipelineManager.getPipeline(pipelineID)); | ||
| } | ||
| } catch (ContainerNotFoundException e) { | ||
| LOG.warn("Container {} not found in SCM: {}", nonOMContainerId, e); | ||
| } catch (PipelineNotFoundException e) { | ||
| LOG.debug("Pipeline not found for container: {} and pipelineId: {}", | ||
| nonOMContainerId, pipelineID, e); | ||
| } | ||
| containerDiscrepancyInfo.setPipelines(pipelines); | ||
| containerDiscrepancyInfo.setExistsAt("SCM"); | ||
| containerDiscrepancyInfoList.add(containerDiscrepancyInfo); | ||
| }); | ||
|
|
||
| } catch (IOException ex) { | ||
| throw new WebApplicationException(ex, | ||
| Response.Status.INTERNAL_SERVER_ERROR); | ||
| } catch (IllegalArgumentException e) { | ||
| throw new WebApplicationException(e, Response.Status.BAD_REQUEST); | ||
| } catch (Exception ex) { | ||
| throw new WebApplicationException(ex, | ||
| Response.Status.INTERNAL_SERVER_ERROR); | ||
| } | ||
| return Response.ok(containerDiscrepancyInfoList).build(); | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * <p> | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * <p> | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.hadoop.ozone.recon.api.types; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonInclude; | ||
| import com.fasterxml.jackson.annotation.JsonProperty; | ||
| import org.apache.hadoop.hdds.scm.pipeline.Pipeline; | ||
|
|
||
| import java.util.List; | ||
|
|
||
| /** | ||
| * Metadata object that represents a Container Discrepancy Info. | ||
| */ | ||
| public class ContainerDiscrepancyInfo { | ||
|
|
||
| @JsonProperty("containerId") | ||
| private long containerID; | ||
|
|
||
| @JsonProperty("omContainerState") | ||
| @JsonInclude(JsonInclude.Include.NON_EMPTY) | ||
| private long omContainerState; | ||
|
|
||
| @JsonProperty("scmContainerState") | ||
| @JsonInclude(JsonInclude.Include.NON_EMPTY) | ||
| private long scmContainerState; | ||
|
Contributor
There was a problem hiding this comment. It looks like scmContainerState and omContainerState are not used in anywhere.
Contributor
Author
There was a problem hiding this comment.
Thanks for pointing out Sammi, I have removed them, earlier I was thinking to use, but I don't have a mechanism to for states of mismatched containers. Pls review again. |
||
|
|
||
| @JsonProperty("numberOfKeys") | ||
| private long numberOfKeys; | ||
|
|
||
| @JsonProperty("pipelines") | ||
| private List<Pipeline> pipelines; | ||
|
|
||
| @JsonProperty("existsAt") | ||
| private String existsAt; | ||
|
|
||
| public ContainerDiscrepancyInfo() { | ||
|
|
||
| } | ||
|
|
||
| public long getContainerID() { | ||
| return containerID; | ||
| } | ||
|
|
||
| public void setContainerID(long containerID) { | ||
| this.containerID = containerID; | ||
| } | ||
|
|
||
| public long getNumberOfKeys() { | ||
| return numberOfKeys; | ||
| } | ||
|
|
||
| public void setNumberOfKeys(long numberOfKeys) { | ||
| this.numberOfKeys = numberOfKeys; | ||
| } | ||
|
|
||
| public long getOmContainerState() { | ||
| return omContainerState; | ||
| } | ||
|
|
||
| public void setOmContainerState(long omContainerState) { | ||
| this.omContainerState = omContainerState; | ||
| } | ||
|
|
||
| public long getScmContainerState() { | ||
| return scmContainerState; | ||
| } | ||
|
|
||
| public void setScmContainerState(long scmContainerState) { | ||
| this.scmContainerState = scmContainerState; | ||
| } | ||
|
|
||
| public List<Pipeline> getPipelines() { | ||
| return pipelines; | ||
| } | ||
|
|
||
| public void setPipelines( | ||
| List<Pipeline> pipelines) { | ||
| this.pipelines = pipelines; | ||
| } | ||
|
|
||
| public String getExistsAt() { | ||
| return existsAt; | ||
| } | ||
|
|
||
| public void setExistsAt(String existsAt) { | ||
| this.existsAt = existsAt; | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
scmNonDeletedContainers and scmAllContainers are same, duplicate
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Its fixed. Pls re-review.