Test branch 2

.github/workflows/ci.yml

@@ -166,6 +166,54 @@ jobs:
           path: |
             ~/.m2/repository/org/apache/ozone
           retention-days: 1
+  build-config-doc:
+    needs:
+      - build
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: Download the source artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ozone-bin
+          path: ozone-bin
+      - name: Extract the source tarball
+        run: |
+          mkdir -p ozone-bin/extracted
+          tar -xzf ozone-bin/ozone-*-SNAPSHOT.tar.gz -C ozone-bin/extracted
+      - name: Run the Python script to convert XML properties into Markdown
+        run: python3 dev-support/ci/xml_to_md.py ozone-bin/extracted > hadoop-hdds/docs/content/tools/Configurations.md
+      - name: Create Pull Request in apache/ozone
+        if: github.ref == 'refs/heads/master'
+        uses: peter-evans/create-pull-request@v4
+        with:
+          commit-message: "[Auto] Update Configurations.md from $GITHUB_SHA"
+          title: "[Auto] Update Configurations.md from $GITHUB_SHA"
+          body: "This is an Automated pull request generated from build-config-doc workflow."
+      - name: Checkout ozone-site repository
+        if: github.ref == 'refs/heads/master'
+        uses: actions/checkout@v4
+        with:
+          repository: apache/ozone-site
+          ref: 'HDDS-9225-website-v2'
+          token: ${{ secrets.GITHUB_TOKEN }}
+          path: ozone-site
+      - name: Copy MD file to ozone-site repository
+        if: github.ref == 'refs/heads/master'
+        run: |
+          cp hadoop-hdds/docs/content/tools/Configurations.md ozone-site/docs/*-administrative-guide/*-configuration/configurations.md
+      - name: Create Pull Request in apache/ozone-site
+        if: github.ref == 'refs/heads/master'
+        uses: peter-evans/create-pull-request@v4
+        with:
+          commit-message: "[Auto] Update configurations.md page from ozone $GITHUB_SHA"
+          title: "[Auto] Update configurations.md page from ozone $GITHUB_SHA"
+          body: "This is an Automated pull request generated from ozone."
   compile:
     needs:
       - build-info

dev-support/ci/xml_to_md.py

@@ -1,77 +1,113 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
+import re
+import zipfile
 import xml.etree.ElementTree as ET
-import tarfile
-import fnmatch
+from collections import namedtuple
+from pathlib import Path
+import sys
+
+Property = namedtuple('Property', ['name', 'value', 'tag', 'description'])
 
-class Property:
-    def __init__(self, name, value, tag, description):
-        self.name = name
-        self.value = value
-        self.tag = tag
-        self.description = description
+def extract_xml_from_jar(jar_path, xml_filename):
+    xml_files = []
+    with zipfile.ZipFile(jar_path, 'r') as jar:
+        for file_info in jar.infolist():
+            if file_info.filename.endswith(xml_filename):
+                with jar.open(file_info.filename) as xml_file:
+                    xml_files.append(xml_file.read())
+    return xml_files
 
-def parse_xml_file(xml_file):
+def parse_xml_file(xml_content):
+    root = ET.fromstring(xml_content)
     properties = {}
-    tree = ET.parse(xml_file)
-    root = tree.getroot()
     for prop in root.findall('property'):
-        name = prop.find('name').text
-        value = prop.find('value').text if prop.find('value') is not None else ""
-        tag = prop.find('tag').text if prop.find('tag') is not None else ""
-        description = prop.find('description').text if prop.find('description') is not None else ""
-        properties[name] = Property(name, value, tag, description)
+        name = prop.findtext('name')
+        if not name:
+            raise ValueError("Property 'name' is required but missing in XML.")
+        description = prop.findtext('description', '')
+        if not description:
+            raise ValueError(f"Property '{name}' is missing a description.")
+        properties[name] = Property(
+            name=name,
+            value=prop.findtext('value', ''),
+            tag=prop.findtext('tag', ''),
+            description=' '.join(description.split()).strip()
+        )
     return properties
 
-def write_markdown(properties, output_file):
-    with open(output_file, 'w') as f:
-        f.write("<!--\n")
-        f.write("Licensed to the Apache Software Foundation (ASF) under one or more\n")
-        f.write("contributor license agreements.  See the NOTICE file distributed with\n")
-        f.write("this work for additional information regarding copyright ownership.\n")
-        f.write("The ASF licenses this file to You under the Apache License, Version 2.0\n")
-        f.write("(the \"License\"); you may not use this file except in compliance with\n")
-        f.write("the License.  You may obtain a copy of the License at\n\n")
-        f.write("   http://www.apache.org/licenses/LICENSE-2.0\n\n")
-        f.write("Unless required by applicable law or agreed to in writing, software\n")
-        f.write("distributed under the License is distributed on an \"AS IS\" BASIS,\n")
-        f.write("WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n")
-        f.write("See the License for the specific language governing permissions and\n")
-        f.write("limitations under the License.\n")
-        f.write("-->\n\n")
+def generate_markdown(properties):
+    markdown = f"""
+        ## Ozone Configuration Keys
+        This page provides a comprehensive overview of the configuration keys available in Ozone.
+        ### Configuration Keys
+        """
 
-        for prop in sorted(properties.values(), key=lambda p: p.name):
-            f.write(f"| **Name**        | `{prop.name}` |\n")
-            f.write(f"|:----------------|:----------------------------|\n")
-            f.write(f"| **Value**       | {prop.value} |\n")
-            f.write(f"| **Tag**         | {prop.tag} |\n")
-            f.write(f"| **Description** | {prop.description} |\n")
-            f.write("--------------------------------------------------------------------------------\n")
-
-def find_xml_files(directory):
-    xml_files = []
-    for root, _, files in os.walk(directory):
-        for file in files:
-            if file == 'ozone-default-generated.xml' and 'test-classes' not in root:
-                xml_files.append(os.path.join(root, file))
-    return xml_files
+    for prop in sorted(properties.values(), key=lambda p: p.name):
+        markdown += f"""
+            | **Name**        | `{prop.name}` |
+            |:----------------|:----------------------------|
+            | **Value**       | {prop.value} |
+            | **Tag**         | {prop.tag} |
+            | **Description** | {prop.description} |
+            --------------------------------------------------------------------------------
+            """
+    return markdown
 
 def main():
-    tar_file = 'hadoop-ozone/dist/target/ozone-*.tar.gz'
-    extract_dir = 'ozone-src'
+    if len(sys.argv) != 3:
+        print("Usage: python3 xml_to_md.py <base_path> <output_path>")
+        sys.exit(1)
 
-    # Extract the tar file
-    with tarfile.open(tar_file, 'r:gz') as tar_ref:
-        tar_ref.extractall(extract_dir)
+    base_path = sys.argv[1]
+    output_path = sys.argv[2]
 
-    # Find all XML files
-    xml_files = find_xml_files(extract_dir)
+    # Find ozone SNAPSHOT directory dynamically using regex
+    snapshot_dir = next(
+        (os.path.join(base_path, d) for d in os.listdir(base_path) if re.match(r'ozone-[\d.]+\d-SNAPSHOT', d)),
+        None
+    )
 
-    properties = {}
-    for xml_file in xml_files:
-        properties.update(parse_xml_file(xml_file))
+    if not snapshot_dir:
+        raise ValueError("Snapshot directory not found in the specified base path.")
+
+    extract_path = os.path.join(snapshot_dir, 'share', 'ozone', 'lib')
+    xml_filename = 'ozone-default-generated.xml'
+
+    property_map = {}
+    for file_name in os.listdir(extract_path):
+        if file_name.endswith('.jar'):
+            jar_path = os.path.join(extract_path, file_name)
+            xml_contents = extract_xml_from_jar(jar_path, xml_filename)
+            for xml_content in xml_contents:
+                property_map.update(parse_xml_file(xml_content))
+
+    markdown_content = generate_markdown(property_map)
 
-    output_file = "hadoop-hdds/docs/content/tools/Configurations.md"
-    write_markdown(properties, output_file)
+    if output_path:
+        output_path = Path(output_path)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        with output_path.open('w', encoding='utf-8') as file:
+            file.write(markdown_content)
+    else:
+        print(markdown_content)
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     main()