From 19e7af5bd8ea71f748d763bae96ec5c2311ab3f9 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 6 Aug 2024 11:42:46 +0530 Subject: [PATCH 1/9] HDDS-1235. Testing after doc modification --- hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml index 88a36835c290..50c912f43e20 100644 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml @@ -33,3 +33,4 @@ data: LOG4J.PROPERTIES_log4j.appender.stdout: "org.apache.log4j.ConsoleAppender" LOG4J.PROPERTIES_log4j.appender.stdout.layout: "org.apache.log4j.PatternLayout" LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n" + From ed275d7e3e531ea08d8b4a1a194757fce3fb712a Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 6 Aug 2024 12:25:14 +0530 Subject: [PATCH 2/9] added space --- hadoop-ozone/dev-support/checks/docs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/dev-support/checks/docs.sh b/hadoop-ozone/dev-support/checks/docs.sh index ce80c3f3e5c4..6a9328d1f73a 100755 --- a/hadoop-ozone/dev-support/checks/docs.sh +++ b/hadoop-ozone/dev-support/checks/docs.sh @@ -40,3 +40,5 @@ if [[ -s "${REPORT_FILE}" ]]; then fi exit ${rc} + + From cf7805c08a36f68d40786cc0f2567498acf705cf Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 6 Aug 2024 12:30:25 +0530 Subject: [PATCH 3/9] Added Doc generation workflow in post-commit actions --- .github/workflows/post-commit.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index 91d9e1fcd9fe..728952f004ad 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -25,3 +25,6 @@ jobs: if: github.event_name == 'pull_request' || !startsWith(github.ref_name, 'dependabot') uses: ./.github/workflows/ci.yml secrets: inherit + Generate-Docs: + uses: ./.github/workflows/doc.yml + secrets: inherit \ No newline at end of file From e05943a96410ff381820b2a9bd82ee100b30607f Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 6 Aug 2024 12:39:23 +0530 Subject: [PATCH 4/9] Change action branch --- .github/workflows/doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 7ffe4d6cd2d5..579567901837 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -17,7 +17,7 @@ name: xml-to-md on: push: branches: - - HDDS-11072 + - test-branch-2 jobs: xml-to-md: From 626dfca4eebdbcb7f48c758f9d293a291ca6632a Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 6 Aug 2024 12:40:11 +0530 Subject: [PATCH 5/9] Reset post commit workflow --- .github/workflows/post-commit.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index 728952f004ad..45049e87d22e 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -24,7 +24,4 @@ jobs: CI: if: github.event_name == 'pull_request' || !startsWith(github.ref_name, 'dependabot') uses: ./.github/workflows/ci.yml - secrets: inherit - Generate-Docs: - uses: ./.github/workflows/doc.yml secrets: inherit \ No newline at end of file From f278aa5cf89e6ab7e4d6aeb515710a005a6ab267 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 6 Aug 2024 16:17:58 +0530 Subject: [PATCH 6/9] Corrected the py path --- .github/workflows/doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 579567901837..061d4b77729c 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -36,7 +36,7 @@ jobs: run: sudo apt-get update && sudo apt-get install -y python3-pip - name: Run the Python script to convert XML properties into Markdown - run: python3 dev_support/ci/xml_to_md.py + run: python3 dev-support/ci/xml_to_md.py - name: Upload the markdown file uses: actions/upload-artifact@v3 From 7a5e1378031d5de5210581e3e3b224fd8794a9b0 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 6 Aug 2024 16:42:40 +0530 Subject: [PATCH 7/9] Added build steps --- .github/workflows/doc.yml | 71 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 061d4b77729c..5f06c86bc7c4 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -20,7 +20,76 @@ on: - test-branch-2 jobs: + build: + runs-on: ubuntu-20.04 + timeout-minutes: 60 + strategy: + matrix: + java: [ 8 ] + fail-fast: false + steps: + - name: Checkout project + uses: actions/checkout@v4 + + - name: Cache for npm dependencies + uses: actions/cache@v4 + with: + path: | + ~/.pnpm-store + **/node_modules + key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm- + + - name: Cache for maven dependencies + uses: actions/cache/restore@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ozone + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + - name: Download Ratis repo + if: ${{ inputs.ratis_args != '' }} + uses: actions/download-artifact@v4 + with: + name: ratis-jars + path: | + ~/.m2/repository/org/apache/ratis + - name: Setup java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + - name: Run a full build + run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }} + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + - name: Store binaries for tests + uses: actions/upload-artifact@v4 + with: + name: ozone-bin + path: | + hadoop-ozone/dist/target/ozone-*.tar.gz + !hadoop-ozone/dist/target/ozone-*-src.tar.gz + retention-days: 1 + - name: Store source tarball for compilation + uses: actions/upload-artifact@v4 + with: + name: ozone-src + path: hadoop-ozone/dist/target/ozone-*-src.tar.gz + retention-days: 1 + - name: Store Maven repo for tests + uses: actions/upload-artifact@v4 + with: + name: ozone-repo + path: | + ~/.m2/repository/org/apache/ozone + retention-days: 1 xml-to-md: + needs: + - build runs-on: ubuntu-20.04 steps: @@ -47,4 +116,4 @@ jobs: - name: Download the markdown file uses: actions/download-artifact@v3 with: - name: Configurations.md + name: Configurations.md \ No newline at end of file From 780ba412660b8f4d3c73d3dbdd64cf2daa319624 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Wed, 7 Aug 2024 11:52:00 +0530 Subject: [PATCH 8/9] Find files --- .github/workflows/doc.yml | 9 +++ dev-support/ci/xml_to_md.py | 135 +++++++++++++++--------------------- 2 files changed, 66 insertions(+), 78 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 5f06c86bc7c4..445878961084 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -104,6 +104,15 @@ jobs: - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y python3-pip + - name: Download the source tarball + uses: actions/download-artifact@v3 + with: + name: ozone-src + path: . + + - name: Extract the source tarball + run: unzip ozone-src.zip -d ozone-src + - name: Run the Python script to convert XML properties into Markdown run: python3 dev-support/ci/xml_to_md.py diff --git a/dev-support/ci/xml_to_md.py b/dev-support/ci/xml_to_md.py index d8da7cfadb51..2c811f79d258 100644 --- a/dev-support/ci/xml_to_md.py +++ b/dev-support/ci/xml_to_md.py @@ -1,97 +1,76 @@ import os import xml.etree.ElementTree as ET -from collections import namedtuple -from pathlib import Path +import zipfile -# Define the Property namedtuple to hold property details -Property = namedtuple('Property', ['name', 'value', 'tag', 'description']) +class Property: + def __init__(self, name, value, tag, description): + self.name = name + self.value = value + self.tag = tag + self.description = description -def parse_xml_file(file_path): - """ - Parse the given XML file and extract properties. - - :param file_path: Path to the XML file - :return: Dictionary of properties with property names as keys - """ - tree = ET.parse(file_path) - root = tree.getroot() +def parse_xml_file(xml_file): properties = {} + tree = ET.parse(xml_file) + root = tree.getroot() for prop in root.findall('property'): - name = prop.find('name').text if prop.find('name') is not None else '' - value = prop.find('value').text if prop.find('value') is not None else '' - tag = prop.find('tag').text if prop.find('tag') is not None else '' - description = prop.find('description').text if prop.find('description') is not None else '' - description = ' '.join(description.split()).strip() # Clean up whitespace + name = prop.find('name').text + value = prop.find('value').text if prop.find('value') is not None else "" + tag = prop.find('tag').text if prop.find('tag') is not None else "" + description = prop.find('description').text if prop.find('description') is not None else "" properties[name] = Property(name, value, tag, description) return properties -def generate_markdown(properties): - """ - Generate Markdown content from properties. +def write_markdown(properties, output_file): + with open(output_file, 'w') as f: + f.write("\n\n") - :param properties: Dictionary of properties - :return: Markdown string - """ - markdown = [] - markdown.append("\n\n") + for prop in sorted(properties.values(), key=lambda p: p.name): + f.write(f"| **Name** | `{prop.name}` |\n") + f.write(f"|:----------------|:----------------------------|\n") + f.write(f"| **Value** | {prop.value} |\n") + f.write(f"| **Tag** | {prop.tag} |\n") + f.write(f"| **Description** | {prop.description} |\n") + f.write("--------------------------------------------------------------------------------\n") - for prop in sorted(properties.values(), key=lambda p: p.name): - markdown.append(f"| **Name** | `{prop.name}` |\n") - markdown.append("|:----------------|:----------------------------|\n") - markdown.append(f"| **Value** | {prop.value} |\n") - markdown.append(f"| **Tag** | {prop.tag} |\n") - markdown.append(f"| **Description** | {prop.description} |\n") - markdown.append("--------------------------------------------------------------------------------\n") - - return ''.join(markdown) +def find_xml_files(directory): + xml_files = [] + for root, _, files in os.walk(directory): + for file in files: + if file == 'ozone-default-generated.xml': + xml_files.append(os.path.join(root, file)) + return xml_files def main(): - """ - Main function to parse XML files and generate Markdown documentation. - """ - xml_files = [ - "hadoop-hdds/client/target/classes/ozone-default-generated.xml", - "hadoop-hdds/common/target/classes/ozone-default-generated.xml", - "hadoop-hdds/container-service/target/classes/ozone-default-generated.xml", - "hadoop-hdds/framework/target/classes/ozone-default-generated.xml", - "hadoop-hdds/server-scm/target/classes/ozone-default-generated.xml", - "hadoop-ozone/common/target/classes/ozone-default-generated.xml", - "hadoop-ozone/csi/target/classes/ozone-default-generated.xml", - "hadoop-ozone/ozone-manager/target/classes/ozone-default-generated.xml", - "hadoop-ozone/recon-codegen/target/classes/ozone-default-generated.xml", - "hadoop-ozone/recon/target/classes/ozone-default-generated.xml", - ] + zip_file = 'ozone-src.zip' + extract_dir = 'ozone-src' - property_map = {} + # Extract the zip file + with zipfile.ZipFile(zip_file, 'r') as zip_ref: + zip_ref.extractall(extract_dir) - for xml_file in xml_files: - if not os.path.exists(xml_file): - print(f"File not found: {xml_file}") - continue + # Find all XML files + xml_files = find_xml_files(extract_dir) - properties = parse_xml_file(xml_file) - property_map.update(properties) - - markdown_content = generate_markdown(property_map) - - output_path = Path("hadoop-hdds/docs/content/tools/Configurations.md") - output_path.parent.mkdir(parents=True, exist_ok=True) + properties = {} + for xml_file in xml_files: + properties.update(parse_xml_file(xml_file)) - with output_path.open('w', encoding='utf-8') as file: - file.write(markdown_content) + output_file = "hadoop-hdds/docs/content/tools/Configurations.md" + write_markdown(properties, output_file) -if __name__ == '__main__': +if __name__ == "__main__": main() From 7cce87741c8e404668bcd89c5fefa67e8dbbb9c8 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Wed, 7 Aug 2024 12:23:55 +0530 Subject: [PATCH 9/9] Find files - 1 --- .github/workflows/doc.yml | 23 ++++++++++++++++------- dev-support/ci/xml_to_md.py | 13 +++++++------ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 445878961084..35e5104c0ae1 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -42,7 +42,7 @@ jobs: ${{ runner.os }}-pnpm- - name: Cache for maven dependencies - uses: actions/cache/restore@v4 + uses: actions/cache@v4 with: path: | ~/.m2/repository/*/*/* @@ -50,6 +50,7 @@ jobs: key: maven-repo-${{ hashFiles('**/pom.xml') }} restore-keys: | maven-repo- + - name: Download Ratis repo if: ${{ inputs.ratis_args != '' }} uses: actions/download-artifact@v4 @@ -57,15 +58,18 @@ jobs: name: ratis-jars path: | ~/.m2/repository/org/apache/ratis + - name: Setup java uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ matrix.java }} + - name: Run a full build run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }} env: DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + - name: Store binaries for tests uses: actions/upload-artifact@v4 with: @@ -74,12 +78,14 @@ jobs: hadoop-ozone/dist/target/ozone-*.tar.gz !hadoop-ozone/dist/target/ozone-*-src.tar.gz retention-days: 1 + - name: Store source tarball for compilation uses: actions/upload-artifact@v4 with: name: ozone-src path: hadoop-ozone/dist/target/ozone-*-src.tar.gz retention-days: 1 + - name: Store Maven repo for tests uses: actions/upload-artifact@v4 with: @@ -87,6 +93,7 @@ jobs: path: | ~/.m2/repository/org/apache/ozone retention-days: 1 + xml-to-md: needs: - build @@ -102,27 +109,29 @@ jobs: python-version: '3.x' - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y python3-pip + run: sudo apt-get update && sudo apt-get install -y python3-pip unzip - name: Download the source tarball - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ozone-src path: . - name: Extract the source tarball - run: unzip ozone-src.zip -d ozone-src + run: | + mkdir -p ozone-src + tar -xzf hadoop-ozone/dist/target/ozone-*.tar.gz -C ozone-src - name: Run the Python script to convert XML properties into Markdown run: python3 dev-support/ci/xml_to_md.py - name: Upload the markdown file - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Configurations.md path: hadoop-hdds/docs/content/tools/Configurations.md - name: Download the markdown file - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: Configurations.md \ No newline at end of file + name: Configurations.md diff --git a/dev-support/ci/xml_to_md.py b/dev-support/ci/xml_to_md.py index 2c811f79d258..50230eca82c6 100644 --- a/dev-support/ci/xml_to_md.py +++ b/dev-support/ci/xml_to_md.py @@ -1,6 +1,7 @@ import os import xml.etree.ElementTree as ET -import zipfile +import tarfile +import fnmatch class Property: def __init__(self, name, value, tag, description): @@ -50,17 +51,17 @@ def find_xml_files(directory): xml_files = [] for root, _, files in os.walk(directory): for file in files: - if file == 'ozone-default-generated.xml': + if file == 'ozone-default-generated.xml' and 'test-classes' not in root: xml_files.append(os.path.join(root, file)) return xml_files def main(): - zip_file = 'ozone-src.zip' + tar_file = 'hadoop-ozone/dist/target/ozone-*.tar.gz' extract_dir = 'ozone-src' - # Extract the zip file - with zipfile.ZipFile(zip_file, 'r') as zip_ref: - zip_ref.extractall(extract_dir) + # Extract the tar file + with tarfile.open(tar_file, 'r:gz') as tar_ref: + tar_ref.extractall(extract_dir) # Find all XML files xml_files = find_xml_files(extract_dir)