Test branch 2

.github/workflows/doc.yml

@@ -17,10 +17,86 @@ name: xml-to-md
 on:
   push:
     branches:
-      - HDDS-11072
+      - test-branch-2
 
 jobs:
+  build:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        java: [ 8 ]
+      fail-fast: false
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+
+      - name: Cache for npm dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.pnpm-store
+            **/node_modules
+          key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-
+
+      - name: Cache for maven dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
+
+      - name: Download Ratis repo
+        if: ${{ inputs.ratis_args != '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ratis-jars
+          path: |
+            ~/.m2/repository/org/apache/ratis
+
+      - name: Setup java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+
+      - name: Run a full build
+        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
+        env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+
+      - name: Store binaries for tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: ozone-bin
+          path: |
+            hadoop-ozone/dist/target/ozone-*.tar.gz
+            !hadoop-ozone/dist/target/ozone-*-src.tar.gz
+          retention-days: 1
+
+      - name: Store source tarball for compilation
+        uses: actions/upload-artifact@v4
+        with:
+          name: ozone-src
+          path: hadoop-ozone/dist/target/ozone-*-src.tar.gz
+          retention-days: 1
+
+      - name: Store Maven repo for tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: ozone-repo
+          path: |
+            ~/.m2/repository/org/apache/ozone
+          retention-days: 1
+
   xml-to-md:
+    needs:
+      - build
     runs-on: ubuntu-20.04
 
     steps:
@@ -33,18 +109,29 @@ jobs:
           python-version: '3.x'
 
       - name: Install dependencies
-        run: sudo apt-get update && sudo apt-get install -y python3-pip
+        run: sudo apt-get update && sudo apt-get install -y python3-pip unzip
+
+      - name: Download the source tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: ozone-src
+          path: .
+
+      - name: Extract the source tarball
+        run: |
+          mkdir -p ozone-src
+          tar -xzf hadoop-ozone/dist/target/ozone-*.tar.gz -C ozone-src
 
       - name: Run the Python script to convert XML properties into Markdown
-        run: python3 dev_support/ci/xml_to_md.py
+        run: python3 dev-support/ci/xml_to_md.py
 
       - name: Upload the markdown file
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Configurations.md
           path: hadoop-hdds/docs/content/tools/Configurations.md
 
       - name: Download the markdown file
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: Configurations.md

.github/workflows/post-commit.yml

@@ -24,4 +24,4 @@ jobs:
   CI:
     if: github.event_name == 'pull_request' || !startsWith(github.ref_name, 'dependabot')
     uses: ./.github/workflows/ci.yml
-    secrets: inherit
+    secrets: inherit

dev-support/ci/xml_to_md.py

@@ -1,97 +1,77 @@
 import os
 import xml.etree.ElementTree as ET
-from collections import namedtuple
-from pathlib import Path
+import tarfile
+import fnmatch
 
-# Define the Property namedtuple to hold property details
-Property = namedtuple('Property', ['name', 'value', 'tag', 'description'])
+class Property:
+    def __init__(self, name, value, tag, description):
+        self.name = name
+        self.value = value
+        self.tag = tag
+        self.description = description
 
-def parse_xml_file(file_path):
-    """
-    Parse the given XML file and extract properties.
-
-    :param file_path: Path to the XML file
-    :return: Dictionary of properties with property names as keys
-    """
-    tree = ET.parse(file_path)
-    root = tree.getroot()
+def parse_xml_file(xml_file):
     properties = {}
+    tree = ET.parse(xml_file)
+    root = tree.getroot()
     for prop in root.findall('property'):
-        name = prop.find('name').text if prop.find('name') is not None else ''
-        value = prop.find('value').text if prop.find('value') is not None else ''
-        tag = prop.find('tag').text if prop.find('tag') is not None else ''
-        description = prop.find('description').text if prop.find('description') is not None else ''
-        description = ' '.join(description.split()).strip()  # Clean up whitespace
+        name = prop.find('name').text
+        value = prop.find('value').text if prop.find('value') is not None else ""
+        tag = prop.find('tag').text if prop.find('tag') is not None else ""
+        description = prop.find('description').text if prop.find('description') is not None else ""
         properties[name] = Property(name, value, tag, description)
     return properties
 
-def generate_markdown(properties):
-    """
-    Generate Markdown content from properties.
+def write_markdown(properties, output_file):
+    with open(output_file, 'w') as f:
+        f.write("<!--\n")
+        f.write("Licensed to the Apache Software Foundation (ASF) under one or more\n")
+        f.write("contributor license agreements.  See the NOTICE file distributed with\n")
+        f.write("this work for additional information regarding copyright ownership.\n")
+        f.write("The ASF licenses this file to You under the Apache License, Version 2.0\n")
+        f.write("(the \"License\"); you may not use this file except in compliance with\n")
+        f.write("the License.  You may obtain a copy of the License at\n\n")
+        f.write("   http://www.apache.org/licenses/LICENSE-2.0\n\n")
+        f.write("Unless required by applicable law or agreed to in writing, software\n")
+        f.write("distributed under the License is distributed on an \"AS IS\" BASIS,\n")
+        f.write("WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n")
+        f.write("See the License for the specific language governing permissions and\n")
+        f.write("limitations under the License.\n")
+        f.write("-->\n\n")
 
-    :param properties: Dictionary of properties
-    :return: Markdown string
-    """
-    markdown = []
-    markdown.append("<!--\n")
-    markdown.append("Licensed to the Apache Software Foundation (ASF) under one or more\n")
-    markdown.append("contributor license agreements.  See the NOTICE file distributed with\n")
-    markdown.append("this work for additional information regarding copyright ownership.\n")
-    markdown.append("The ASF licenses this file to You under the Apache License, Version 2.0\n")
-    markdown.append("(the \"License\"); you may not use this file except in compliance with\n")
-    markdown.append("the License.  You may obtain a copy of the License at\n\n")
-    markdown.append("   http://www.apache.org/licenses/LICENSE-2.0\n\n")
-    markdown.append("Unless required by applicable law or agreed to in writing, software\n")
-    markdown.append("distributed under the License is distributed on an \"AS IS\" BASIS,\n")
-    markdown.append("WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n")
-    markdown.append("See the License for the specific language governing permissions and\n")
-    markdown.append("limitations under the License.\n")
-    markdown.append("-->\n\n")
+        for prop in sorted(properties.values(), key=lambda p: p.name):
+            f.write(f"| **Name**        | `{prop.name}` |\n")
+            f.write(f"|:----------------|:----------------------------|\n")
+            f.write(f"| **Value**       | {prop.value} |\n")
+            f.write(f"| **Tag**         | {prop.tag} |\n")
+            f.write(f"| **Description** | {prop.description} |\n")
+            f.write("--------------------------------------------------------------------------------\n")
 
-    for prop in sorted(properties.values(), key=lambda p: p.name):
-        markdown.append(f"| **Name**        | `{prop.name}` |\n")
-        markdown.append("|:----------------|:----------------------------|\n")
-        markdown.append(f"| **Value**       | {prop.value} |\n")
-        markdown.append(f"| **Tag**         | {prop.tag} |\n")
-        markdown.append(f"| **Description** | {prop.description} |\n")
-        markdown.append("--------------------------------------------------------------------------------\n")
-
-    return ''.join(markdown)
+def find_xml_files(directory):
+    xml_files = []
+    for root, _, files in os.walk(directory):
+        for file in files:
+            if file == 'ozone-default-generated.xml' and 'test-classes' not in root:
+                xml_files.append(os.path.join(root, file))
+    return xml_files
 
 def main():
-    """
-    Main function to parse XML files and generate Markdown documentation.
-    """
-    xml_files = [
-        "hadoop-hdds/client/target/classes/ozone-default-generated.xml",
-        "hadoop-hdds/common/target/classes/ozone-default-generated.xml",
-        "hadoop-hdds/container-service/target/classes/ozone-default-generated.xml",
-        "hadoop-hdds/framework/target/classes/ozone-default-generated.xml",
-        "hadoop-hdds/server-scm/target/classes/ozone-default-generated.xml",
-        "hadoop-ozone/common/target/classes/ozone-default-generated.xml",
-        "hadoop-ozone/csi/target/classes/ozone-default-generated.xml",
-        "hadoop-ozone/ozone-manager/target/classes/ozone-default-generated.xml",
-        "hadoop-ozone/recon-codegen/target/classes/ozone-default-generated.xml",
-        "hadoop-ozone/recon/target/classes/ozone-default-generated.xml",
-    ]
+    tar_file = 'hadoop-ozone/dist/target/ozone-*.tar.gz'
+    extract_dir = 'ozone-src'
 
-    property_map = {}
+    # Extract the tar file
+    with tarfile.open(tar_file, 'r:gz') as tar_ref:
+        tar_ref.extractall(extract_dir)
 
-    for xml_file in xml_files:
-        if not os.path.exists(xml_file):
-            print(f"File not found: {xml_file}")
-            continue
+    # Find all XML files
+    xml_files = find_xml_files(extract_dir)
 
-        properties = parse_xml_file(xml_file)
-        property_map.update(properties)
-
-    markdown_content = generate_markdown(property_map)
-
-    output_path = Path("hadoop-hdds/docs/content/tools/Configurations.md")
-    output_path.parent.mkdir(parents=True, exist_ok=True)
+    properties = {}
+    for xml_file in xml_files:
+        properties.update(parse_xml_file(xml_file))
 
-    with output_path.open('w', encoding='utf-8') as file:
-        file.write(markdown_content)
+    output_file = "hadoop-hdds/docs/content/tools/Configurations.md"
+    write_markdown(properties, output_file)
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

hadoop-ozone/dev-support/checks/docs.sh

@@ -40,3 +40,5 @@ if [[ -s "${REPORT_FILE}" ]]; then
 fi
 
 exit ${rc}
+
+

hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml

@@ -33,3 +33,4 @@ data:
   LOG4J.PROPERTIES_log4j.appender.stdout: "org.apache.log4j.ConsoleAppender"
   LOG4J.PROPERTIES_log4j.appender.stdout.layout: "org.apache.log4j.PatternLayout"
   LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n"
+