HDDS-1234. Testing xml-to-md workflow

.github/workflows/ci.yml

@@ -142,7 +142,7 @@ jobs:
           distribution: 'temurin'
           java-version: ${{ matrix.java }}
       - name: Run a full build
-        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc ${{ inputs.ratis_args }}
+        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Store binaries for tests
@@ -218,7 +218,7 @@ jobs:
           distribution: 'temurin'
           java-version: ${{ matrix.java }}
       - name: Compile Ozone using Java ${{ matrix.java }}
-        run: hadoop-ozone/dev-support/checks/build.sh -Dskip.npx -Dskip.installnpx -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }}
+        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Dskip.npx -Dskip.installnpx -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }}
         env:
           OZONE_WITH_COVERAGE: false
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
@@ -428,7 +428,6 @@ jobs:
           mkdir -p hadoop-ozone/dist/target
           tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
           rm ozone*.tar.gz
-          sudo chmod -R a+rwX hadoop-ozone/dist/target
       - name: Execute tests
         run: |
           pushd hadoop-ozone/dist/target/ozone-*

.github/workflows/intermittent-test-check.yml

@@ -115,7 +115,7 @@ jobs:
           java-version: 8
       - name: Build (most) of Ozone
         run: |
-          args="-Dskip.npx -Dskip.installnpx -DskipShade"
+          args="-Dskip.npx -Dskip.installnpx -DskipShade -Dmaven.javadoc.skip=true"
           if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then
             args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}"
             args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}"

.github/workflows/repeat-acceptance.yml

@@ -108,7 +108,7 @@ jobs:
           distribution: 'temurin'
           java-version: ${{ env.JAVA_VERSION }}
       - name: Run a full build
-        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc
+        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc -Dmaven.javadoc.skip=true
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Store binaries for tests

hadoop-hdds/annotations/pom.xml

@@ -34,4 +34,16 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <properties>
     <maven.test.skip>true</maven.test.skip> <!-- no tests in this module so far -->
   </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <proc>none</proc>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
 </project>

hadoop-hdds/client/pom.xml

@@ -70,6 +70,43 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
           <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <annotationProcessorPaths>
+            <path>
+              <groupId>org.apache.ozone</groupId>
+              <artifactId>hdds-config</artifactId>
+              <version>${hdds.version}</version>
+            </path>
+          </annotationProcessorPaths>
+          <annotationProcessors>
+            <annotationProcessor>org.apache.hadoop.hdds.conf.ConfigFileGenerator</annotationProcessor>
+          </annotationProcessors>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>ban-annotations</id> <!-- override default restriction from root POM -->
+            <configuration>
+              <rules>
+                <restrictImports>
+                  <reason>Only selected annotation processors are enabled, see configuration of maven-compiler-plugin.</reason>
+                  <bannedImports>
+                    <bannedImport>org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator</bannedImport>
+                    <bannedImport>org.apache.hadoop.hdds.scm.metadata.Replicate</bannedImport>
+                    <bannedImport>org.kohsuke.MetaInfServices</bannedImport>
+                  </bannedImports>
+                </restrictImports>
+              </rules>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java

@@ -285,6 +285,7 @@ public ContainerCommandResponseProto sendCommand(
     }
     for (DatanodeDetails dn : datanodeList) {
       try {
+        request = reconstructRequestIfNeeded(request, dn);
         futureHashMap.put(dn, sendCommandAsync(request, dn).getResponse());
       } catch (InterruptedException e) {
         LOG.error("Command execution was interrupted.");
@@ -316,6 +317,29 @@ public ContainerCommandResponseProto sendCommand(
     return responseProtoHashMap;
   }
 
+  /**
+   * @param request
+   * @param dn
+   * @param pipeline
+   * In case of getBlock for EC keys, it is required to set replicaIndex for
+   * every request with the replicaIndex for that DN for which the request is
+   * sent to. This method unpacks proto and reconstructs request after setting
+   * the replicaIndex field.
+   * @return new updated Request
+   */
+  private ContainerCommandRequestProto reconstructRequestIfNeeded(
+      ContainerCommandRequestProto request, DatanodeDetails dn) {
+    boolean isEcRequest = pipeline.getReplicationConfig()
+        .getReplicationType() == HddsProtos.ReplicationType.EC;
+    if (request.hasGetBlock() && isEcRequest) {
+      ContainerProtos.GetBlockRequestProto gbr = request.getGetBlock();
+      request = request.toBuilder().setGetBlock(gbr.toBuilder().setBlockID(
+          gbr.getBlockID().toBuilder().setReplicaIndex(
+              pipeline.getReplicaIndex(dn)).build()).build()).build();
+    }
+    return request;
+  }
+
   @Override
   public ContainerCommandResponseProto sendCommand(
       ContainerCommandRequestProto request, List<Validator> validators)

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java

@@ -67,27 +67,26 @@ private HddsClientUtils() {
           .add(NotReplicatedException.class)
           .build();
 
-  private static void doNameChecks(String resName) {
+  private static void doNameChecks(String resName, String resType) {
     if (resName == null) {
-      throw new IllegalArgumentException("Bucket or Volume name is null");
+      throw new IllegalArgumentException(resType + " name is null");
     }
 
     if (resName.length() < OzoneConsts.OZONE_MIN_BUCKET_NAME_LENGTH ||
         resName.length() > OzoneConsts.OZONE_MAX_BUCKET_NAME_LENGTH) {
-      throw new IllegalArgumentException(
-          "Bucket or Volume length is illegal, "
-              + "valid length is 3-63 characters");
+      throw new IllegalArgumentException(resType +
+          " length is illegal, " + "valid length is 3-63 characters");
     }
 
     if (resName.charAt(0) == '.' || resName.charAt(0) == '-') {
-      throw new IllegalArgumentException(
-          "Bucket or Volume name cannot start with a period or dash");
+      throw new IllegalArgumentException(resType +
+          " name cannot start with a period or dash");
     }
 
     if (resName.charAt(resName.length() - 1) == '.' ||
         resName.charAt(resName.length() - 1) == '-') {
-      throw new IllegalArgumentException("Bucket or Volume name "
-          + "cannot end with a period or dash");
+      throw new IllegalArgumentException(resType +
+          " name cannot end with a period or dash");
     }
   }
 
@@ -108,27 +107,27 @@ private static boolean isSupportedCharacter(char c, boolean isStrictS3) {
     return false;
   }
 
-  private static void doCharacterChecks(char currChar, char prev,
+  private static void doCharacterChecks(char currChar, char prev, String resType,
       boolean isStrictS3) {
     if (Character.isUpperCase(currChar)) {
-      throw new IllegalArgumentException(
-          "Bucket or Volume name does not support uppercase characters");
+      throw new IllegalArgumentException(resType +
+          " name does not support uppercase characters");
     }
     if (!isSupportedCharacter(currChar, isStrictS3)) {
-      throw new IllegalArgumentException("Bucket or Volume name has an " +
-          "unsupported character : " + currChar);
+      throw new IllegalArgumentException(resType +
+          " name has an unsupported character : " + currChar);
     }
     if (prev == '.' && currChar == '.') {
-      throw new IllegalArgumentException("Bucket or Volume name should not " +
-          "have two contiguous periods");
+      throw new IllegalArgumentException(resType +
+          " name should not have two contiguous periods");
     }
     if (prev == '-' && currChar == '.') {
-      throw new IllegalArgumentException(
-          "Bucket or Volume name should not have period after dash");
+      throw new IllegalArgumentException(resType +
+          " name should not have period after dash");
     }
     if (prev == '.' && currChar == '-') {
-      throw new IllegalArgumentException(
-          "Bucket or Volume name should not have dash after period");
+      throw new IllegalArgumentException(resType +
+          " name should not have dash after period");
     }
   }
 
@@ -140,7 +139,11 @@ private static void doCharacterChecks(char currChar, char prev,
    * @throws IllegalArgumentException
    */
   public static void verifyResourceName(String resName) {
-    verifyResourceName(resName, true);
+    verifyResourceName(resName, "resource", true);
+  }
+
+  public static void verifyResourceName(String resName, String resType) {
+    verifyResourceName(resName, resType, true);
   }
 
   /**
@@ -150,9 +153,9 @@ public static void verifyResourceName(String resName) {
    *
    * @throws IllegalArgumentException
    */
-  public static void verifyResourceName(String resName, boolean isStrictS3) {
+  public static void verifyResourceName(String resName, String resType, boolean isStrictS3) {
 
-    doNameChecks(resName);
+    doNameChecks(resName, resType);
 
     boolean isIPv4 = true;
     char prev = (char) 0;
@@ -162,13 +165,13 @@ public static void verifyResourceName(String resName, boolean isStrictS3) {
       if (currChar != '.') {
         isIPv4 = ((currChar >= '0') && (currChar <= '9')) && isIPv4;
       }
-      doCharacterChecks(currChar, prev, isStrictS3);
+      doCharacterChecks(currChar, prev, resType, isStrictS3);
       prev = currChar;
     }
 
     if (isIPv4) {
-      throw new IllegalArgumentException(
-          "Bucket or Volume name cannot be an IPv4 address or all numeric");
+      throw new IllegalArgumentException(resType +
+          " name cannot be an IPv4 address or all numeric");
     }
   }
 

hadoop-hdds/common/pom.xml

@@ -257,6 +257,43 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
           <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <annotationProcessorPaths>
+            <path>
+              <groupId>org.apache.ozone</groupId>
+              <artifactId>hdds-config</artifactId>
+              <version>${hdds.version}</version>
+            </path>
+          </annotationProcessorPaths>
+          <annotationProcessors>
+            <annotationProcessor>org.apache.hadoop.hdds.conf.ConfigFileGenerator</annotationProcessor>
+          </annotationProcessors>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>ban-annotations</id> <!-- override default restriction from root POM -->
+            <configuration>
+              <rules>
+                <restrictImports>
+                  <reason>Only selected annotation processors are enabled, see configuration of maven-compiler-plugin.</reason>
+                  <bannedImports>
+                    <bannedImport>org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator</bannedImport>
+                    <bannedImport>org.apache.hadoop.hdds.scm.metadata.Replicate</bannedImport>
+                    <bannedImport>org.kohsuke.MetaInfServices</bannedImport>
+                  </bannedImports>
+                </restrictImports>
+              </rules>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/DecommissionUtils.java

@@ -126,7 +126,9 @@ public static Map<String, Object> getCountsMap(DatanodeDetails datanode, JsonNod
                                                  Map<String, Object> countsMap, String errMsg)
       throws IOException {
     for (int i = 1; i <= numDecomNodes; i++) {
-      if (datanode.getHostName().equals(counts.get("tag.datanode." + i).asText())) {
+      String datanodeHostName =
+          (counts.get("tag.datanode." + i) != null) ? (counts.get("tag.datanode." + i).asText()) : "";
+      if (datanode.getHostName().equals(datanodeHostName)) {
         JsonNode pipelinesDN = counts.get("PipelinesWaitingToCloseDN." + i);
         JsonNode underReplicatedDN = counts.get("UnderReplicatedDN." + i);
         JsonNode unclosedDN = counts.get("UnclosedContainersDN." + i);