HDDS-4177. SCM Container DB bootstrap on Recon startup

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java

@@ -365,4 +365,6 @@ StatusAndMessages queryUpgradeFinalizationProgress(
    * commands operating on {@code containerID}.
    */
   Token<?> getContainerToken(ContainerID containerID) throws IOException;
+
+  long getContainerCount() throws IOException;
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -89,7 +89,7 @@ public final class OzoneConsts {
   // OM Http server endpoints
   public static final String OZONE_OM_SERVICE_LIST_HTTP_ENDPOINT =
       "/serviceList";
-  public static final String OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT =
+  public static final String OZONE_DB_CHECKPOINT_HTTP_ENDPOINT =
       "/dbCheckpoint";
 
   // Ozone File System scheme

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -2546,6 +2546,24 @@
       OM snapshot.
     </description>
   </property>
+  <property>
+    <name>ozone.recon.scm.connection.request.timeout</name>
+    <value>5s</value>
+    <tag>OZONE, RECON, SCM</tag>
+    <description>
+      Connection request timeout in milliseconds for HTTP call made by Recon to
+      request SCM DB snapshot.
+    </description>
+  </property>
+  <property>
+    <name>ozone.recon.scm.connection.timeout</name>
+    <value>5s</value>
+    <tag>OZONE, RECON, SCM</tag>
+    <description>
+      Connection timeout for HTTP call in milliseconds made by Recon to request
+      SCM snapshot.
+    </description>
+  </property>
   <property>
     <name>ozone.recon.om.socket.timeout</name>
     <value>5s</value>
@@ -2579,6 +2597,23 @@
       Request to flush the OM DB before taking checkpoint snapshot.
     </description>
   </property>
+  <property>
+    <name>ozone.recon.scm.container.threshold</name>
+    <value>100</value>
+    <tag>OZONE, RECON, SCM</tag>
+    <description>
+      Threshold value for the difference in number of containers
+      in SCM and RECON.
+    </description>
+  </property>
+  <property>
+    <name>ozone.recon.scm.snapshot.enabled</name>
+    <value>false</value>
+    <tag>OZONE, RECON, SCM</tag>
+    <description>
+      If enabled, SCM DB Snapshot is taken by Recon.
+    </description>
+  </property>
   <property>
     <name>hdds.tracing.enabled</name>
     <value>false</value>

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java

@@ -52,6 +52,8 @@
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetExistContainerWithPipelinesInBatchRequestProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineRequestProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerCountRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerCountResponseProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.InSafeModeRequestProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ListPipelineRequestProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ListPipelineResponseProto;
@@ -931,6 +933,18 @@ public Token<?> getContainerToken(
     return OzonePBHelper.tokenFromProto(response.getToken());
   }
 
+  @Override
+  public long getContainerCount() throws IOException {
+    GetContainerCountRequestProto request =
+        GetContainerCountRequestProto.newBuilder().build();
+
+    GetContainerCountResponseProto response =
+        submitRequest(Type.GetContainerCount,
+          builder -> builder.setGetContainerCountRequest(request))
+        .getGetContainerCountResponse();
+    return response.getContainerCount();
+  }
+
   @Override
   public Object getUnderlyingProxyObject() {
     return rpcProxy;

hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto

@@ -74,6 +74,7 @@ message ScmContainerLocationRequest {
   optional ContainerBalancerStatusRequestProto containerBalancerStatusRequest = 35;
   optional FinalizeScmUpgradeRequestProto finalizeScmUpgradeRequest = 36;
   optional QueryUpgradeFinalizationProgressRequestProto queryUpgradeFinalizationProgressRequest = 37;
+  optional GetContainerCountRequestProto getContainerCountRequest = 38;
 }
 
 message ScmContainerLocationResponse {
@@ -119,6 +120,7 @@ message ScmContainerLocationResponse {
   optional ContainerBalancerStatusResponseProto containerBalancerStatusResponse = 35;
   optional FinalizeScmUpgradeResponseProto finalizeScmUpgradeResponse = 36;
   optional QueryUpgradeFinalizationProgressResponseProto queryUpgradeFinalizationProgressResponse = 37;
+  optional GetContainerCountResponseProto getContainerCountResponse = 38;
 
   enum Status {
     OK = 1;
@@ -162,6 +164,7 @@ enum Type {
   GetContainerBalancerStatus = 30;
   FinalizeScmUpgrade = 31;
   QueryUpgradeFinalizationProgress = 32;
+  GetContainerCount = 33;
 }
 
 /**
@@ -383,6 +386,13 @@ message GetPipelineResponseProto {
   required Pipeline pipeline = 1;
 }
 
+message GetContainerCountRequestProto {
+}
+
+message GetContainerCountResponseProto {
+  required int64 containerCount = 1;
+}
+
 message ActivatePipelineRequestProto {
   required PipelineID pipelineID = 1;
   optional string traceID = 2;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java

@@ -53,6 +53,7 @@
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineResponseProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetExistContainerWithPipelinesInBatchRequestProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetExistContainerWithPipelinesInBatchResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerCountResponseProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineRequestProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetPipelineResponseProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetSafeModeRuleStatusesRequestProto;
@@ -397,6 +398,13 @@ public ScmContainerLocationResponse processRequest(
             .setDatanodeUsageInfoResponse(getDatanodeUsageInfo(
                 request.getDatanodeUsageInfoRequest()))
             .build();
+      case GetContainerCount:
+        return ScmContainerLocationResponse.newBuilder()
+          .setCmdType(request.getCmdType())
+          .setStatus(Status.OK)
+          .setGetContainerCountResponse(getContainerCount(
+                  request.getGetContainerCountRequest()))
+          .build();
       default:
         throw new IllegalArgumentException(
             "Unknown command type: " + request.getCmdType());
@@ -831,4 +839,12 @@ public DatanodeUsageInfoResponseProto getDatanodeUsageInfo(
         .build();
   }
 
+  public GetContainerCountResponseProto getContainerCount(
+      StorageContainerLocationProtocolProtos.GetContainerCountRequestProto
+      request) throws IOException {
+
+    return GetContainerCountResponseProto.newBuilder()
+      .setContainerCount(impl.getContainerCount())
+      .build();
+  }
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java

@@ -945,6 +945,11 @@ public Token<?> getContainerToken(ContainerID containerID)
         .generateToken(remoteUser.getUserName(), containerID);
   }
 
+  @Override
+  public long getContainerCount() throws IOException {
+    return scm.getContainerManager().getContainers().size();
+  }
+
   /**
    * Queries a list of Node that match a set of statuses.
    *

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java

@@ -23,7 +23,7 @@
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.server.http.BaseHttpServer;
 import org.apache.hadoop.ozone.OzoneConsts;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
 
 /**
  * HttpServer2 wrapper for the Ozone Storage Container Manager.
@@ -34,7 +34,7 @@ public StorageContainerManagerHttpServer(MutableConfigurationSource conf,
                                            StorageContainerManager scm)
       throws IOException {
     super(conf, "scm");
-    addServlet("dbCheckpoint", OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT,
+    addServlet("dbCheckpoint", OZONE_DB_CHECKPOINT_HTTP_ENDPOINT,
         SCMDBCheckpointServlet.class);
     getWebAppContext().setAttribute(OzoneConsts.SCM_CONTEXT_ATTRIBUTE, scm);
   }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java

@@ -30,7 +30,7 @@
 import java.net.InetSocketAddress;
 
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_PORT_DEFAULT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_PORT_KEY;
@@ -153,13 +153,13 @@ public String getOMDBCheckpointEnpointUrl(boolean isHttpPolicy) {
     if (isHttpPolicy) {
       if (StringUtils.isNotEmpty(getHttpAddress())) {
         return "http://" + getHttpAddress() +
-            OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT +
+            OZONE_DB_CHECKPOINT_HTTP_ENDPOINT +
             "?" + OZONE_DB_CHECKPOINT_REQUEST_FLUSH + "=true";
       }
     } else {
       if (StringUtils.isNotEmpty(getHttpsAddress())) {
         return "https://" + getHttpsAddress() +
-            OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT +
+            OZONE_DB_CHECKPOINT_HTTP_ENDPOINT +
             "?" + OZONE_DB_CHECKPOINT_REQUEST_FLUSH + "=true";
       }
     }

hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config

@@ -21,6 +21,7 @@ OZONE-SITE.XML_ozone.csi.socket=/tmp/csi.sock
 
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.http-address=scm:9876
 OZONE-SITE.XML_ozone.scm.container.size=1GB
 OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
 OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s

hadoop-ozone/dist/src/main/compose/ozone/docker-config

@@ -19,6 +19,7 @@ CORE-SITE.XML_fs.trash.interval=1
 
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.http-address=scm:9876
 OZONE-SITE.XML_ozone.scm.container.size=1GB
 OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
 OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s

hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config

@@ -16,6 +16,7 @@
 
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.http-address=scm:9876
 OZONE-SITE.XML_ozone.scm.container.size=1GB
 OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
 OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s

hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config

@@ -19,6 +19,7 @@ CORE-SITE.XML_fs.trash.interval=1
 
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.http-address=scm:9876
 OZONE-SITE.XML_ozone.scm.container.size=1GB
 OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
 OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java

@@ -18,7 +18,7 @@
 package org.apache.hadoop.ozone.recon;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
 
 import java.util.HashMap;
 import java.util.UUID;
@@ -114,7 +114,7 @@ public void testReconGetsSnapshotFromLeader() throws Exception {
     String expectedUrl = "http://" +
         (hostname.equals("0.0.0.0") ? "localhost" : hostname) + ":" +
         ozoneManager.get().getHttpServer().getHttpAddress().getPort() +
-        OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+        OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
     String snapshotUrl = impl.getOzoneManagerSnapshotUrl();
     Assert.assertEquals("OM Snapshot should be requested from the leader.",
         expectedUrl, snapshotUrl);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java

@@ -23,7 +23,7 @@
 import org.apache.hadoop.hdds.server.http.BaseHttpServer;
 import org.apache.hadoop.ozone.OzoneConsts;
 
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OM_SERVICE_LIST_HTTP_ENDPOINT;
 
 /**
@@ -36,7 +36,7 @@ public OzoneManagerHttpServer(MutableConfigurationSource conf,
     super(conf, "ozoneManager");
     addServlet("serviceList", OZONE_OM_SERVICE_LIST_HTTP_ENDPOINT,
         ServiceListJSONServlet.class);
-    addServlet("dbCheckpoint", OZONE_OM_DB_CHECKPOINT_HTTP_ENDPOINT,
+    addServlet("dbCheckpoint", OZONE_DB_CHECKPOINT_HTTP_ENDPOINT,
         OMDBCheckpointServlet.class);
     getWebAppContext().setAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE, om);
   }

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java

@@ -41,6 +41,8 @@ private ReconConstants() {
   public static final String CONTAINER_KEY_COUNT_TABLE =
       "containerKeyCountTable";
 
+  public static final String RECON_SCM_SNAPSHOT_DB = "scm.snapshot.db";
+
   // By default, limit the number of results returned
   public static final String DEFAULT_FETCH_COUNT = "1000";
   public static final String DEFAULT_BATCH_NUMBER = "1";

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java

@@ -124,6 +124,22 @@ public final class  ReconServerConfigKeys {
   public static final String
       OZONE_RECON_METRICS_HTTP_CONNECTION_REQUEST_TIMEOUT_DEFAULT = "10s";
 
+  public static final String OZONE_RECON_SCM_CONTAINER_THRESHOLD =
+      "ozone.recon.scm.container.threshold";
+  public static final int OZONE_RECON_SCM_CONTAINER_THRESHOLD_DEFAULT = 100;
+
+  public static final String OZONE_RECON_SCM_SNAPSHOT_ENABLED =
+      "ozone.recon.scm.snapshot.enabled";
+  public static final boolean OZONE_RECON_SCM_SNAPSHOT_ENABLED_DEFAULT = false;
+
+  public static final String OZONE_RECON_SCM_CONNECTION_TIMEOUT =
+      "ozone.recon.scm.connection.timeout";
+  public static final String OZONE_RECON_SCM_CONNECTION_TIMEOUT_DEFAULT = "5s";
+
+  public static final String OZONE_RECON_SCM_CONNECTION_REQUEST_TIMEOUT =
+      "ozone.recon.scm.connection.request.timeout";
+  public static final String
+      OZONE_RECON_SCM_CONNECTION_REQUEST_TIMEOUT_DEFAULT = "5s";
   /**
    * Private constructor for utility class.
    */

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java

@@ -81,6 +81,7 @@ public ContainerHealthTask(
   public synchronized void run() {
     try {
       while (canRun()) {
+        wait(interval);
         long start = Time.monotonicNow();
         long currentTime = System.currentTimeMillis();
         long existingCount = processExistingDBRecords(currentTime);
@@ -97,7 +98,6 @@ public synchronized void run() {
                 " processing {} containers.", Time.monotonicNow() - start,
             containers.size());
         processedContainers.clear();
-        wait(interval);
       }
     } catch (Throwable t) {
       LOG.error("Exception in Missing Container task Thread.", t);

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java

@@ -294,4 +294,9 @@ private boolean needUpdate(DatanodeDetails datanodeDetails,
     return currentTime - getLastHeartbeat(datanodeDetails) >=
         reconDatanodeOutdatedTime;
   }
+
+  public void reinitialize(Table<UUID, DatanodeDetails> nodeTable) {
+    this.nodeDB = nodeTable;
+    loadExistingNodes();
+  }
 }