HDFS-15855.Solve the problem of incorrect EC progress when loading FsImage.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
 import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -484,9 +485,26 @@ private void loadPolicy(ErasureCodingPolicyInfo info) {
   public synchronized void loadPolicies(
       List<ErasureCodingPolicyInfo> ecPolicies, Configuration conf)
       throws IOException{
+    loadPolicies(ecPolicies, conf, null);
+  }
+
+  /**
+   * Reload the erasure coding strategy from fsImage and use
+   * a counter to record.
+   *
+   * @param ecPolicies contains ErasureCodingPolicy list
+   *
+   */
+  public synchronized void loadPolicies(
+          List<ErasureCodingPolicyInfo> ecPolicies, Configuration conf,
+          StartupProgress.Counter counter)
+          throws IOException{
     Preconditions.checkNotNull(ecPolicies);
     for (ErasureCodingPolicyInfo p : ecPolicies) {
       loadPolicy(p);
+      if (counter != null) {
+        counter.increment();
+      }
     }
     enableDefaultPolicy(conf);
     updatePolicies();

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java

@@ -484,7 +484,7 @@ public int compare(FileSummary.Section s1, FileSummary.Section s2) {
         case ERASURE_CODING:
           Step step = new Step(StepType.ERASURE_CODING_POLICIES);
           prog.beginStep(Phase.LOADING_FSIMAGE, step);
-          loadErasureCodingSection(in);
+          loadErasureCodingSection(in, prog, step);
           prog.endStep(Phase.LOADING_FSIMAGE, step);
           break;
         default:
@@ -569,7 +569,8 @@ private void loadCacheManagerSection(InputStream in, StartupProgress prog,
           new CacheManager.PersistState(s, pools, directives));
     }
 
-    private void loadErasureCodingSection(InputStream in)
+    private void loadErasureCodingSection(InputStream in,
+        StartupProgress prog, Step currentStep)
         throws IOException {
       ErasureCodingSection s = ErasureCodingSection.parseDelimitedFrom(in);
       List<ErasureCodingPolicyInfo> ecPolicies = Lists
@@ -578,7 +579,11 @@ private void loadErasureCodingSection(InputStream in)
         ecPolicies.add(PBHelperClient.convertErasureCodingPolicyInfo(
             s.getPolicies(i)));
       }
-      fsn.getErasureCodingPolicyManager().loadPolicies(ecPolicies, conf);
+
+      prog.setTotal(Phase.LOADING_FSIMAGE, currentStep, ecPolicies.size());
+      Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, currentStep);
+      fsn.getErasureCodingPolicyManager().loadPolicies(ecPolicies, conf,
+          counter);
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java

@@ -24,6 +24,7 @@
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
 import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.*;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Rule;
@@ -189,6 +190,45 @@ public void testChangeDefaultPolicy() throws Exception {
     assertAllPoliciesAreDisabled(getPoliciesResult);
   }
 
+  @Test
+  public void testLoadPolicyWithCounter() throws Exception {
+    final HdfsConfiguration conf = new HdfsConfiguration();
+    final String testPolicy = "RS-3-2-1024k";
+    final String defaultPolicy = conf.getTrimmed(
+            DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
+            DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
+    assertNotEquals("The default policy and the next default policy " +
+            "should not be the same!", testPolicy, defaultPolicy);
+
+    StartupProgress prog = NameNode.getStartupProgress();
+    Step step = new Step(StepType.ERASURE_CODING_POLICIES);
+    prog.beginStep(Phase.LOADING_FSIMAGE, step);
+    StartupProgress.Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE,
+        step);
+    StartupProgressView progView = prog.createView();
+    long count = progView.getCount(Phase.LOADING_FSIMAGE, step);
+    assertEquals(count, 0);
+
+    ErasureCodingPolicyManager manager =
+            ErasureCodingPolicyManager.getInstance();
+    // Change the default policy to a new one
+    conf.set(
+            DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
+            testPolicy);
+    manager.init(conf);
+    // Load policies similar to when fsimage is loaded at namenode startup
+    manager.loadPolicies(constructAllDisabledInitialPolicies(), conf, counter);
+
+    ErasureCodingPolicyInfo[] getPoliciesResult = manager.getPolicies();
+    boolean isEnabled = isPolicyEnabled(testPolicy, getPoliciesResult);
+    assertTrue("The new default policy should be " +
+            "in enabled state!", isEnabled);
+
+    progView = prog.createView();
+    count = progView.getCount(Phase.LOADING_FSIMAGE, step);
+    assertEquals(count, getPoliciesResult.length);
+  }
+
   private void testGetPolicies(ErasureCodingPolicy[] enabledPolicies)
       throws Exception {
     HdfsConfiguration conf = new HdfsConfiguration();