diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..cc365f78f7b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + + # Check for updates to GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/repos-config.json b/.github/repos-config.json index 5849d7f98d9..438c3758ab1 100644 --- a/.github/repos-config.json +++ b/.github/repos-config.json @@ -2,140 +2,120 @@ "repositories": [ { "name": "miopen", - "reponame": "MIOpen", "url": "ROCm/MIOpen", "branch": "develop", "category": "projects" }, { "name": "tensile", - "reponame": "Tensile", "url": "ROCm/Tensile", "branch": "develop", "category": "shared" }, { "name": "composablekernel", - "reponame": "composable_kernel", "url": "ROCm/composable_kernel", "branch": "develop", "category": "projects" }, { "name": "hipblas", - "reponame": "hipBLAS", "url": "ROCm/hipBLAS", "branch": "develop", "category": "projects" }, { "name": "hipblas-common", - "reponame": "hipBLAS-common", "url": "ROCm/hipBLAS-common", "branch": "develop", "category": "projects" }, { "name": "hipblaslt", - "reponame": "hipBLASLt", "url": "ROCm/hipBLASLt", "branch": "develop", "category": "projects" }, { "name": "hipcub", - "reponame": "hipCUB", "url": "ROCm/hipCUB", "branch": "develop", "category": "projects" }, { "name": "hipfft", - "reponame": "hipFFT", "url": "ROCm/hipFFT", "branch": "develop", "category": "projects" }, { "name": "hiprand", - "reponame": "hipRAND", "url": "ROCm/hipRAND", "branch": "develop", "category": "projects" }, { "name": "hipsolver", - "reponame": "hipSOLVER", "url": "ROCm/hipSOLVER", "branch": "develop", "category": "projects" }, { "name": "hipsparse", - "reponame": "hipSPARSE", "url": "ROCm/hipSPARSE", "branch": "develop", "category": "projects" }, { "name": "hipsparselt", - "reponame": "hipSPARSELt", "url": "ROCm/hipSPARSELt", "branch": "develop", "category": "projects" }, { "name": "rocblas", - "reponame": "rocBLAS", "url": "ROCm/rocBLAS", "branch": "develop", "category": "projects" }, { "name": "rocfft", - "reponame": "rocFFT", "url": "ROCm/rocFFT", "branch": "develop", "category": "projects" }, { "name": "rocprim", - "reponame": "rocPRIM", "url": "ROCm/rocPRIM", "branch": "develop", "category": "projects" }, { "name": "rocrand", - "reponame": "rocRAND", "url": "ROCm/rocRAND", "branch": "develop", "category": "projects" }, { "name": "rocroller", - "reponame": "rocRoller", "url": "ROCm/rocRoller", "branch": "main", "category": "shared" }, { "name": "rocsolver", - "reponame": "rocSOLVER", "url": "ROCm/rocSOLVER", "branch": "develop", "category": "projects" }, { "name": "rocsparse", - "reponame": "rocSPARSE", "url": "ROCm/rocSPARSE", "branch": "develop", "category": "projects" }, { "name": "rocthrust", - "reponame": "rocThrust", "url": "ROCm/rocThrust", "branch": "develop", "category": "projects" diff --git a/.github/scripts/config_loader.py b/.github/scripts/config_loader.py new file mode 100644 index 00000000000..5a6e7a40759 --- /dev/null +++ b/.github/scripts/config_loader.py @@ -0,0 +1,18 @@ +import json +import sys +import logging +from typing import List +from repo_config_model import RepoConfig, RepoEntry + +logger = logging.getLogger(__name__) + +def load_repo_config(config_path: str) -> List[RepoEntry]: + """Load and validate repository config from JSON using Pydantic.""" + try: + with open(config_path, "r", encoding="utf-8") as f: + data = json.load(f) + config = RepoConfig(**data) + return config.repositories + except Exception as e: + logger.error(f"Failed to load or validate config file '{config_path}': {e}") + sys.exit(1) diff --git a/.github/scripts/github_cli_client.py b/.github/scripts/github_cli_client.py new file mode 100644 index 00000000000..401e470c94c --- /dev/null +++ b/.github/scripts/github_cli_client.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 + +""" +GitHub CLI Client Utility +------------------------- +This utility provides a GitHubCLIClient class that wraps GitHub CLI (gh) operations +used across automation scripts, such as retrieving pull request file changes and labels. + +When doing manual testing, you can run the same gh commands directly in the terminal. +These commands will be output by the debug logging in debug mode. + +Requirements: + - GitHub CLI (`gh`) must be installed and authenticated. + - NOTE: GH_TOKEN environment variable hands authentication token to the CLI in a runner. + - The repository must be accessible to the authenticated user. +""" + +import subprocess +import json +import logging +from typing import List, Optional + +logger = logging.getLogger(__name__) + +class GitHubCLIClient: + def __init__(self) -> None: + """Initialize the GitHub CLI client.""" + if not self._gh_available(): + raise EnvironmentError("GitHub CLI (`gh`) is not installed or not in PATH.") + + def _gh_available(self) -> bool: + """Check if GitHub CLI is available.""" + try: + subprocess.run(["gh", "--version"], check=True, stdout=subprocess.DEVNULL) + return True + except subprocess.CalledProcessError: + return False + + def _run_gh_command(self, args: List[str], dry_run: Optional[bool] = False) -> subprocess.CompletedProcess: + """Run a `gh` CLI command and return the result.""" + cmd = ["gh"] + args + logger.debug(f"Running command: {' '.join(cmd)}") + # dry_run option only matters for operations that write to GitHub + if dry_run: + result = subprocess.CompletedProcess(cmd, 0, stdout="Dry run enabled. No changes made.", stderr="") + else: + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + logger.error(f"Command failed: {' '.join(cmd)}\n{result.stderr.strip()}") + raise subprocess.CalledProcessError(result.returncode, cmd, result.stdout, result.stderr) + return result + + def get_changed_files(self, repo: str, pr: int) -> List[str]: + """Fetch the changed files in a pull request using `gh` CLI.""" + result = self._run_gh_command( + ["pr", "view", str(pr), "--repo", repo, "--json", "files"], + ) + data = json.loads(result.stdout) + files = [f["path"] for f in data.get("files", [])] + logger.debug(f"Changed files in PR #{pr}: {files}") + return files + + def get_defined_labels(self, repo: str) -> List[str]: + """Get all labels defined in the given repository.""" + result = self._run_gh_command(["label", "list", "--repo", repo, "--json", "name"]) + return [label["name"] for label in json.loads(result.stdout)] + + def get_existing_labels_on_pr(self, repo: str, pr: int) -> List[str]: + """Fetch current labels on a PR.""" + result = self._run_gh_command( + ["pr", "view", str(pr), "--repo", repo, "--json", "labels"] + ) + data = json.loads(result.stdout) + labels = [label["name"] for label in data.get("labels", [])] + logger.debug(f"Existing labels on PR #{pr}: {labels}") + return labels + + def pr_view(self, repo: str, head: str) -> Optional[int]: + """Check if a PR exists for the given repo and branch.""" + try: + result = self._run_gh_command(["pr", "list", "--json", "number", "--repo", repo, "--head", head]) + pr_list = json.loads(result.stdout) + return pr_list[0]["number"] if pr_list else None + except subprocess.CalledProcessError: + logger.warning(f"Failed to retrieve PR from {repo} with head {head}") + return None # PR does not exist + + def get_pr_by_head_branch(self, repo: str, head: str) -> Optional[dict]: + """Get the PR object for a given head branch in a repository, if it exists.""" + try: + result = self._run_gh_command(["pr", "list", "--json", "number,title,state", "--repo", repo, "--head", head]) + pr_list = json.loads(result.stdout) + return pr_list[0] if pr_list else None + except subprocess.CalledProcessError: + logger.warning(f"Failed to retrieve PR from {repo} with head {head}") + return None + + def pr_create(self, repo: str, base: str, head: str, title: str, body: str, dry_run: Optional[bool] = False) -> None: + """Create a new pull request.""" + cmd = [ + "pr", "create", + "--repo", repo, + "--base", base, + "--head", head, + "--title", title, + "--body", body + ] + self._run_gh_command(cmd, dry_run=dry_run) + logger.info(f"Created PR from {head} to {base} in {repo}.") + + def close_pr_and_delete_branch(self, repo: str, pr_number: int, dry_run: Optional[bool] = False) -> None: + """Close a pull request and delete the associated branch using the GitHub CLI.""" + cmd = ["pr", "close", str(pr_number), "--repo", repo, "--delete-branch"] + if dry_run: + logger.info(f"Dry run: The pull request #{pr_number} would be closed and the branch would be deleted in repo '{repo}'") + else: + self._run_gh_command(cmd) + logger.info(f"Closed pull request #{pr_number} and deleted the associated branch in repo '{repo}'") + + def sync_labels(self, target_repo: str, pr_number: int, labels: List[str], dry_run: Optional[bool] = False) -> None: + """Sync labels from the source repo to the target repo (only apply existing labels).""" + logger.debug(f"Syncing labels to {target_repo} PR #{pr_number}.") + result = self._run_gh_command( + ["label", "list", "--repo", target_repo, "--json", "name"] + ) + target_repo_labels = {label["name"] for label in json.loads(result.stdout)} + labels_set = set(labels) + labels_to_apply = labels_set & target_repo_labels + # Apply labels that exist in both source PR and target repos + # Wrap in quotes if label contains spaces + labels_arg = ",".join(f'"{label}"' if " " in label else label for label in labels_to_apply) + cmd = [ + "pr", "edit", + str(pr_number), + "--repo", target_repo, + "--add-label", labels_arg + ] + if not dry_run: + self._run_gh_command(cmd, dry_run=dry_run) + logger.info(f"Applied labels '{labels_arg}' to PR #{pr_number} in {target_repo}.") + else: + logger.info(f"Dry run: Labels '{labels_arg}' would be applied to PR #{pr_number} in {target_repo}.") diff --git a/.github/scripts/pr_category_label.py b/.github/scripts/pr_category_label.py new file mode 100644 index 00000000000..239576e74b3 --- /dev/null +++ b/.github/scripts/pr_category_label.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +""" +PR Category Label Script +-------------------- +This script analyzes the file paths changed in a pull request and determines which +category labels should be added or removed based on the modified files. + +It uses GitHub's cli to fetch the changed files and the existing labels on the pull request. +Then, it computes the desired labels based on file paths, compares them to the existing labels, +and applies the necessary additions and removals unless in dry-run mode. + +Arguments: + --repo : Full repository name (e.g., org/repo) + --pr : Pull request number + --dry-run : If set, will only log actions without making changes. + --debug : If set, enables detailed debug logging. + +Outputs: + Writes 'add' and 'remove' keys to the GitHub Actions $GITHUB_OUTPUT file, which + the workflow reads to apply label changes using the GitHub CLI. + +Example Usage: + To run in debug mode and perform a dry-run (no changes made): + python pr_auto_label.py --repo ROCm/rocm-libraries --pr --dry-run --debug + To run in debug mode and apply label changes: + python pr_auto_label.py --repo ROCm/rocm-libraries --pr --debug +""" + +import argparse +import sys +import os +import logging +from pathlib import Path +from typing import List, Optional +from github_cli_client import GitHubCLIClient + +logger = logging.getLogger(__name__) + +def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Apply labels based on PR's changed files.") + parser.add_argument("--repo", required=True, help="Full repository name (e.g., org/repo)") + parser.add_argument("--pr", required=True, type=int, help="Pull request number") + parser.add_argument("--dry-run", action="store_true", help="Print results without writing to GITHUB_OUTPUT.") + parser.add_argument("--debug", action="store_true", help="Enable debug logging") + return parser.parse_args(argv) + +def compute_desired_labels(file_paths: list) -> set: + """Determine the desired labels based on the changed files.""" + desired_labels = set() + for path in file_paths: + parts = Path(path).parts + if len(parts) >= 2: + if parts[0] == "projects": + desired_labels.add(f"project: {parts[1]}") + elif parts[0] == "shared": + desired_labels.add(f"shared: {parts[1]}") + logger.debug(f"Desired labels based on changes: {desired_labels}") + return desired_labels + +def output_labels(existing_labels: List[str], desired_labels: List[str], dry_run: bool) -> None: + """Output the labels to add/remove to GITHUB_OUTPUT or log them in dry-run mode.""" + existing_auto_labels = { + label for label in existing_labels + if label.startswith("project: ") or label.startswith("shared: ") + } + to_add = sorted(desired_labels - set(existing_labels)) + to_remove = sorted(existing_auto_labels - desired_labels) + logger.debug(f"Labels to add: {to_add}") + logger.debug(f"Labels to remove: {to_remove}") + if dry_run: + logger.info("Dry run enabled. Labels will not be applied.") + else: + output_file = os.environ.get("GITHUB_OUTPUT") + if output_file: + with open(output_file, 'a') as f: + print(f"label_add={','.join(to_add)}", file=f) + print(f"label_remove={','.join(to_remove)}", file=f) + logger.info(f"Wrote to GITHUB_OUTPUT: add={','.join(to_add)}") + logger.info(f"Wrote to GITHUB_OUTPUT: remove={','.join(to_remove)}") + else: + print("GITHUB_OUTPUT environment variable not set. Outputs cannot be written.") + sys.exit(1) + +def main(argv=None) -> None: + """Main function to execute the PR auto label logic.""" + args = parse_arguments(argv) + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO + ) + client = GitHubCLIClient() + changed_files = [file for file in client.get_changed_files(args.repo, int(args.pr))] + existing_labels = client.get_existing_labels_on_pr(args.repo, int(args.pr)) + desired_labels = compute_desired_labels(changed_files) + output_labels(existing_labels, desired_labels, args.dry_run) + +if __name__ == "__main__": + main() diff --git a/.github/scripts/pr_close_fanouts.py b/.github/scripts/pr_close_fanouts.py new file mode 100644 index 00000000000..6ebda8a6f97 --- /dev/null +++ b/.github/scripts/pr_close_fanouts.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +""" +Close Fanned-Out PRs and Delete Branches +---------------------------------------- + +This script is used in the monorepo fanout automation system. It runs when a monorepo pull request +is closed (either merged or manually closed) and performs cleanup actions: + +- Identifies all fanned-out pull requests that were created as part of the monorepo PR. +- Closes each corresponding pull request in the original sub-repositories. +- Deletes the temporary branch associated with each fanned-out PR. + +It uses a consistent naming convention for identifying branches: monorepo-pr-- + +Arguments: + --repo : Full repository name (e.g., org/repo) + --pr : Pull request number + --config : OPTIONAL, path to the repos-config.json file + --dry-run : If set, will only log actions without making changes. + --debug : If set, enables detailed debug logging. + +Example Usage: + To run in debug mode and perform a dry-run (no changes made): + python pr_close_fanouts.py --repo ROCm/rocm-libraries --pr 123 --dry-run --debug +""" + +import argparse +import logging +from typing import Optional, List +from github_cli_client import GitHubCLIClient +from repo_config_model import RepoEntry +from config_loader import load_repo_config +from utils_fanout_naming import FanoutNaming + +logger = logging.getLogger(__name__) + +def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Close fanned-out PRs and delete associated branches.") + parser.add_argument("--repo", required=True, help="Full repository name (e.g., org/repo)") + parser.add_argument("--pr", required=True, type=int, help="Pull request number") + parser.add_argument("--config", required=False, default=".github/repos-config.json", help="Path to the repos-config.json file") + parser.add_argument("--dry-run", action="store_true", help="Print results without writing to GITHUB_OUTPUT.") + parser.add_argument("--debug", action="store_true", help="Enable debug logging") + return parser.parse_args(argv) + +def main(argv: Optional[List[str]] = None) -> None: + """Main function to close fanned-out PRs and delete branches.""" + args = parse_arguments(argv) + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO + ) + client = GitHubCLIClient() + config = load_repo_config(args.config) + for entry in config: + branch = FanoutNaming.compute_branch_name(args.pr, entry.name) + pr = client.get_pr_by_head_branch(entry.url, branch) + if pr: + number = pr["number"] + client.close_pr_and_delete_branch(entry.url, number) + logger.info(f"Closing PR #{number} in {entry.url} for branch {branch}") + else: + logger.info(f"No open PR found in {entry.url} for branch {branch}") + +if __name__ == "__main__": + main() diff --git a/.github/scripts/pr_detect_changed_subtrees.py b/.github/scripts/pr_detect_changed_subtrees.py new file mode 100644 index 00000000000..f342769951a --- /dev/null +++ b/.github/scripts/pr_detect_changed_subtrees.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +""" +PR Detect Changed Subtrees Script +--------------------------------- +This script analyzes a pull request's changed files and determines which subtrees +(defined in .github/repos-config.json by category/name) were affected. + +Steps: + 1. Fetch the changed files in the PR using the GitHub API. + 2. Load the subtree mapping from repos-config.json. + 3. Match changed paths against known category/name prefixes. + 4. Emit a new-line separated list of changed subtrees to GITHUB_OUTPUT as 'subtrees'. + +Arguments: + --repo : Full repository name (e.g., org/repo) + --pr : Pull request number + --config : OPTIONAL, path to the repos-config.json file + --dry-run : If set, will only log actions without making changes. + --debug : If set, enables detailed debug logging. + +Outputs: + Writes 'subtrees' key to the GitHub Actions $GITHUB_OUTPUT file, which + the workflow reads to pass paths to the checkout and fanout stages. + The output is a new-line separated list of subtrees in `category/name` format. + +Example Usage: + To run in debug mode and perform a dry-run (no changes made): + python pr_detect_changed_subtrees.py --repo ROCm/rocm-libraries --pr 123 --dry-run +""" + +import argparse +import sys +import os +import logging +from typing import List, Optional, Set +from github_cli_client import GitHubCLIClient +from repo_config_model import RepoEntry +from config_loader import load_repo_config + +logger = logging.getLogger(__name__) + +def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Detect changed subtrees in a PR.") + parser.add_argument("--repo", required=True, help="Full repository name (e.g., org/repo)") + parser.add_argument("--pr", required=True, type=int, help="Pull request number") + parser.add_argument("--config", required=False, default=".github/repos-config.json", help="Path to the repos-config.json file") + parser.add_argument("--dry-run", action="store_true", help="Print results without writing to GITHUB_OUTPUT.") + parser.add_argument("--debug", action="store_true", help="Enable debug logging") + return parser.parse_args(argv) + +def get_valid_prefixes(config: List[RepoEntry]) -> Set[str]: + """Extract valid subtree prefixes from the configuration.""" + valid_prefixes = {f"{entry.category}/{entry.name}" for entry in config} + logger.debug("Valid subtrees:\n" + "\n".join(sorted(valid_prefixes))) + return valid_prefixes + +def find_matched_subtrees(changed_files: List[str], valid_prefixes: Set[str]) -> List[str]: + """Find subtrees that match the changed files.""" + changed_subtrees = { + "/".join(path.split("/", 2)[:2]) + for path in changed_files + if len(path.split("/")) >= 2 + } + matched = sorted(changed_subtrees & valid_prefixes) + logger.debug(f"Matched subtrees: {matched}") + return matched + +def output_subtrees(matched_subtrees: List[str], dry_run: bool) -> None: + """Output the matched subtrees to GITHUB_OUTPUT or log them in dry-run mode.""" + newline_separated = "\n".join(matched_subtrees) + if dry_run: + logger.info(f"[Dry-run] Would output:\n{newline_separated}") + else: + output_file = os.environ.get('GITHUB_OUTPUT') + if output_file: + with open(output_file, 'a') as f: + print(f"subtrees< None: + """Main function to determine changed subtrees in PR.""" + args = parse_arguments(argv) + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO + ) + client = GitHubCLIClient() + config = load_repo_config(args.config) + changed_files = client.get_changed_files(args.repo, int(args.pr)) + valid_prefixes = get_valid_prefixes(config) + matched_subtrees = find_matched_subtrees(changed_files, valid_prefixes) + output_subtrees(matched_subtrees, args.dry_run) + +if __name__ == "__main__": + main() diff --git a/.github/scripts/pr_fanout.py b/.github/scripts/pr_fanout.py new file mode 100644 index 00000000000..c1f192b654e --- /dev/null +++ b/.github/scripts/pr_fanout.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +""" +PR Fanout Script +------------------ +This script takes a list of changed subtrees in `category/name` format and for each: + - Pushes the corresponding subtree directory from the monorepo to the appropriate branch in the sub-repo using `git subtree push`. + - Creates or updates a pull request in the sub-repo with a standardized branch and label. + +Arguments: + --repo : Full repository name (e.g., org/repo) + --pr : Pull request number + --subtrees : A newline-separated list of subtree paths in category/name format (e.g., projects/rocBLAS) + --config : OPTIONAL, path to the repos-config.json file + --dry-run : If set, will only log actions without making changes. + --debug : If set, enables detailed debug logging. + +Example Usage: + + To run in debug mode and perform a dry-run (no changes made): + python pr-fanout.py --repo ROCm/rocm-libraries --pr 123 --subtrees "$(printf 'projects/rocBLAS\nprojects/hipBLASLt\nshared/rocSPARSE')" --dry-run --debug +""" + +import argparse +import subprocess +import logging +from typing import List, Optional +from github_cli_client import GitHubCLIClient +from repo_config_model import RepoEntry +from config_loader import load_repo_config +from utils_fanout_naming import FanoutNaming + +logger = logging.getLogger(__name__) + +def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Fanout monorepo PR to sub-repos.") + parser.add_argument("--repo", required=True, help="Full repository name (e.g., org/repo)") + parser.add_argument("--pr", required=True, type=int, help="Pull request number") + parser.add_argument("--subtrees", required=True, help="Newline-separated list of changed subtrees (category/name)") + parser.add_argument("--config", required=False, default=".github/repos-config.json", help="Path to the repos-config.json file") + parser.add_argument("--dry-run", action="store_true", help="If set, only logs actions without making changes.") + parser.add_argument("--debug", action="store_true", help="If set, enables detailed debug logging.") + return parser.parse_args(argv) + +def get_subtree_info(config: List[RepoEntry], subtrees: List[str]) -> List[RepoEntry]: + """Return config entries matching the given subtrees in category/name format.""" + requested = set(subtrees) + matched = [ + entry for entry in config + if f"{entry.category}/{entry.name}" in requested + ] + missing = requested - {f"{e.category}/{e.name}" for e in matched} + if missing: + logger.warning(f"Some subtrees not found in config: {', '.join(sorted(missing))}") + return matched + +def subtree_push(entry: RepoEntry, branch: str, prefix: str, subrepo_full_url: str, dry_run: bool) -> None: + """Push the specified subtree to the sub-repo using `git subtree push`.""" + # the output for git subtree push spits out thousands of lines for history preservation, suppress it + push_cmd = ["git", "subtree", "push", "--prefix", prefix, subrepo_full_url, branch, "--quiet"] + logger.debug(f"Running: {' '.join(push_cmd)}") + if not dry_run: + # explicitly set the shell to bash if possible to avoid issue linked, which was hit in testing + # https://stackoverflow.com/questions/69493528/git-subtree-maximum-function-recursion-depth + # we also need to increase python's recursion limit to avoid hitting the recursion limit in the subprocess + try: + result = subprocess.run( + push_cmd, + check=True, + capture_output=True, + text=True, + ) + logging.debug(f"subtree push stdout:\n{result.stdout}") + logging.debug(f"subtree push stderr:\n{result.stderr}") + except subprocess.CalledProcessError as e: + logging.error(f"subtree push failed with exit code {e.returncode}") + logging.error(f"stdout:\n{e.stdout}") + logging.error(f"stderr:\n{e.stderr}") + raise RuntimeError("git subtree push failed — see logs for details.") from e + +def main(argv: Optional[List[str]] = None) -> None: + """Main function to execute the PR fanout logic.""" + args = parse_arguments(argv) + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO + ) + client = GitHubCLIClient() + config = load_repo_config(args.config) + # Key in on intersection between the subtrees input argument (new-line delimited) and the config file contents + subtrees = [line.strip() for line in args.subtrees.splitlines() if line.strip()] + relevant_subtrees = get_subtree_info(config, subtrees) + for entry in relevant_subtrees: + entry_naming = FanoutNaming( + pr_number = args.pr, + monorepo = args.repo, + category = entry.category, + name = entry.name, + subrepo = entry.url + ) + logger.debug(f"\nProcessing subtree: {entry_naming.prefix}") + logger.debug(f"\tBranch: {entry_naming.branch_name}") + logger.debug(f"\tRemote: {entry_naming.subrepo_full_url}") + logger.debug(f"\tPR title: {entry_naming.pr_title}") + subtree_push(entry, entry_naming.branch_name, entry_naming.prefix, entry_naming.subrepo_full_url, args.dry_run) + pr_exists: bool = client.pr_view(entry.url, entry_naming.branch_name) + if not pr_exists: + # check if the branch already exists in the subrepo and error out if it did not + # means git subtree push failed + check_branch_subprocess = subprocess.run( + ["git", "ls-remote", "--heads", entry_naming.subrepo_full_url, entry_naming.branch_name], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + check=True, text=True + ) + if not args.dry_run: + if bool(check_branch_subprocess.stdout.strip()): + # entry.branch is the default branch for the subrepo + # entry_naming.branch_name is the pull request branch name + client.pr_create(entry.url, entry.branch, entry_naming.branch_name, entry_naming.pr_title, entry_naming.pr_body) + logger.info(f"Created PR in {entry.url} for branch {entry_naming.branch_name}") + else: + logger.error(f"Branch {entry_naming.branch_name} does not exist in {entry.url}. Cannot create PR.") + else: + logger.info(f"[Dry-run] Would create PR in {entry.url} for branch {entry_naming.branch_name}") + +if __name__ == "__main__": + main() diff --git a/.github/scripts/pr_fanout_sync_labels.py b/.github/scripts/pr_fanout_sync_labels.py new file mode 100644 index 00000000000..8061a9e6438 --- /dev/null +++ b/.github/scripts/pr_fanout_sync_labels.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +""" +PR Fanout Sync Label Script +--------------------------- +This script reads labels from a monorepo pull request and ensures they exist +on all related fanned-out pull requests, skipping any label that does not +already exist in the sub-repos. This algorithm does not involve removing labels. +Further discussion with component teams required for label removal. + +Arguments: + --repo : Full repository name (e.g., org/repo) + --pr : Pull request number + --config : OPTIONAL, path to the repos-config.json file + --dry-run : If set, will only log actions without making changes. + --debug : If set, enables detailed debug logging. + +Example Usage: + To run in debug mode and perform a dry-run (no changes made): + python pr_fanout_sync_labels.py --repo ROCm/rocm-libraries --pr 123 --debug --dry-run +""" + +import argparse +import logging +from typing import List, Optional +from github_cli_client import GitHubCLIClient +from repo_config_model import RepoEntry +from config_loader import load_repo_config +from utils_fanout_naming import FanoutNaming + +logger = logging.getLogger(__name__) + +def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Sync labels from monorepo PR to fanned-out PRs.") + parser.add_argument("--repo", required=True, help="Full repository name (e.g., org/repo)") + parser.add_argument("--pr", required=True, type=int, help="Pull request number") + parser.add_argument("--config", required=False, default=".github/repos-config.json", help="Path to the repos-config.json file") + parser.add_argument("--dry-run", action="store_true", help="If set, only logs actions without making changes.") + parser.add_argument("--debug", action="store_true", help="If set, enables detailed debug logging.") + return parser.parse_args(argv) + +def sync_labels(client: GitHubCLIClient, monorepo: str, pr_number: str, entries: List[RepoEntry], dry_run: bool) -> None: + """Sync labels from the monorepo PR to the fanned-out PRs.""" + source_labels = client.get_existing_labels_on_pr(monorepo, pr_number) + logger.debug(f"Monorepo PR #{pr_number} labels: {source_labels}") + for entry in entries: + branch = FanoutNaming.compute_branch_name(pr_number, entry.name) + logger.debug(f"Processing labels for {entry.url} PR branch {branch}") + existing_pr = client.pr_view(entry.url, branch) + if not existing_pr: + logger.debug(f"No PR found for branch {branch} in {entry.url}") + continue + defined_labels = client.get_defined_labels(entry.url) + applicable_labels = [label for label in source_labels if label in defined_labels] + logger.debug(f"Applying labels to {entry.url}#{existing_pr}: {applicable_labels}") + client.sync_labels(entry.url, existing_pr, applicable_labels, dry_run) + +def main(argv: Optional[List[str]] = None) -> None: + """Main function to execute the PR fanout label sync logic.""" + args = parse_arguments(argv) + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO + ) + client = GitHubCLIClient() + config = load_repo_config(args.config) + sync_labels(client, args.repo, args.pr, config, args.dry_run) + +if __name__ == "__main__": + main() diff --git a/.github/scripts/repo_config_model.py b/.github/scripts/repo_config_model.py new file mode 100644 index 00000000000..ba2e917dc57 --- /dev/null +++ b/.github/scripts/repo_config_model.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +""" +Repository Config Model +------------------------ + +This module defines Pydantic data models for validating and parsing the +repos-config.json file used by the PR fanout system. + +Structure of the expected JSON: + +{ + "repositories": [ + { + "name": "rocblas", + "url": "ROCm/rocBLAS", + "branch": "develop", + "category": "projects" + }, + ... + ] +} +""" + +from typing import List +from pydantic import BaseModel + +class RepoEntry(BaseModel): + """ + Represents a single repository entry in the repos-config.json file. + + Fields: + name : Name of the project matching packaging file names. Lower-cased and no underscores. (e.g., "rocblas") + url : Individual GitHub org plus repo names in matching case and punctuation. (e.g., "ROCm/rocBLAS") + branch : The base branch of the sub-repo to target (e.g., "develop"). + category : Directory category in the monorepo (e.g., "projects" or "shared"). + """ + name: str + url: str + branch: str + category: str + +class RepoConfig(BaseModel): + """ + Represents the full config file structure. + + Fields: + repositories : List of RepoEntry items. + """ + repositories: List[RepoEntry] diff --git a/.github/scripts/utils_fanout_naming.py b/.github/scripts/utils_fanout_naming.py new file mode 100644 index 00000000000..8953ae3ae43 --- /dev/null +++ b/.github/scripts/utils_fanout_naming.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +""" +Utility functions for naming conventions and templates in monorepo fanout automation. + +Example Usage: + + from monorepo_utils import FanoutNaming + + # Static method when PR title is needed + pr_title = FanoutNaming.compute_pr_title(args.pr, entry.name) + + # Static method when PR body is needed + pr_body = FanoutNaming.compute_pr_body(args.pr, entry.name, args.repo) + + # Static method when only branch name is needed + branch = FanoutNaming.compute_branch_name(args.pr, entry.name) + + # Instance method to create a FanoutNaming object + naming = FanoutNaming( + pr_number=args.pr, + monorepo=args.repo, + category=entry.category, + name=entry.name, + subrepo="ROCm/rocBLAS" + ) + +""" + +from dataclasses import dataclass + +@dataclass +class FanoutNaming: + pr_number: int # pull request number in the monorepo + monorepo: str # monorepo in org/repo format + category: str # category of the subrepo (e.g., projects, shared) + name: str # name of the subrepo in category/name format + subrepo: str # subrepo in org/repo format + + @property + def branch_name(self) -> str: + return self.compute_branch_name(self.pr_number, self.name) + + @staticmethod + def compute_branch_name(pr_number: int, name: str) -> str: + return f"monorepo-pr/{pr_number}/{name}" + + @property + def pr_title(self) -> str: + return f"[MONOREPO AUTO-FANOUT] PR #{self.pr_number} to {self.name}" + + @property + def prefix(self) -> str: + return f"{self.category}/{self.name}" + + @property + def pr_body(self) -> str: + return ( + f"This is an automated PR for subtree `{self.prefix}` " + f"originating from monorepo PR [#{self.pr_number}](https://github.com/{self.monorepo}/pull/{self.pr_number}). " + f"PLEASE DO NOT MERGE OR TOUCH THIS PR, AUTOMATED WORKFLOWS FROM THE MONOREPO ARE USING IT." + ) + + @property + def subrepo_full_url(self) -> str: + return f"https://github.com/{self.subrepo}.git" diff --git a/.github/workflows/pr-auto-label.yml b/.github/workflows/pr-auto-label.yml index 0cc3ad1a8fa..c7e8cf34896 100644 --- a/.github/workflows/pr-auto-label.yml +++ b/.github/workflows/pr-auto-label.yml @@ -1,93 +1,92 @@ +# Auto Label PR +# ------------- +# This GitHub Actions workflow automatically adds or removes labels on a pull request +# based on a custom Python script that analyzes the PR content and paths. +# +# Steps: +# - Run pr_category_label.py to determine which category labels to add/remove +# - Update labels on the PR using GitHub CLI (gh) +# - Run pr_fanout_sync_labels.py to sync custom labels from the monorepo PR to the subrepo PRs + name: Auto Label PR on: + workflow_run: + workflows: ["Fanout Subtree PRs"] + types: + - completed pull_request: types: - - opened - - synchronize - - reopened + - labeled + +# ensure that the workflow is not running for the same PR multiple times at once +concurrency: + group: pr-auto-label-${{ github.event.pull_request.number }} + cancel-in-progress: false jobs: - label-pr: - runs-on: ubuntu-latest + auto-label-pr: + runs-on: ubuntu-24.04 steps: - name: Checkout workflows - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: sparse-checkout: '.github' + token: ${{ secrets.MONOREPO_BOT_TOKEN }} # since we are touching labels on subrepository PRs - - name: Get changed files - id: files - run: | - api="https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files?per_page=100" - files="" - - while [ -n "$api" ]; do - response=$(curl -sS -D headers -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" "$api") - files+=$(echo "$response" | jq -r '.[].filename')$'\n' - - api=$(grep -i '^link:' headers | sed -n 's/.*<\([^>]*\)>; rel="next".*/\1/p') - done - - echo "changed_files=$(echo "$files" | tr '\n' ' ' | xargs)" >> "$GITHUB_OUTPUT" + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' - - name: Get current labels - id: current_labels + - name: Install python dependencies run: | - current_labels=$(gh pr view ${{ github.event.pull_request.number }} --json labels -q '.labels[].name' | tr '\n' ' ') - echo "current_labels=${current_labels}" >> "$GITHUB_OUTPUT" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + python -m pip install --upgrade pip + pip install pydantic - - name: Read repo configurations - id: repos_config + - name: Set up Git user run: | - repo_paths=$(jq -r '.repositories[] | "\(.category)/\(.name)"' .github/repos-config.json | tr '\n' ' ') - echo "repo_paths=$repo_paths" >> "$GITHUB_OUTPUT" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" - - name: Apply and remove labels + - name: Determine PR Number (workflow_run case only) + id: pr_number + env: + GH_TOKEN: ${{ secrets.MONOREPO_BOT_TOKEN }} run: | - changed_files="${{ steps.files.outputs.changed_files }}" - current_labels="${{ steps.current_labels.outputs.current_labels }}" - repo_paths="${{ steps.repos_config.outputs.repo_paths }}" - - labels_to_add=() - labels_to_remove=() - - for entry in $repo_paths; do - category_repo="${entry}" # e.g., "projects/hipsparse" - category="${category_repo%%/*}" # e.g., "projects" - name="${category_repo##*/}" # e.g., "hipsparse" - - # Check if the category is 'projects' and set the label format accordingly - if [ "$category" == "projects" ]; then - label="project: ${name}" # e.g., "project: hipsparse" - else - label="${category}: ${name}" # e.g., "shared: rocroller" - fi + if [ "${{ github.event_name }}" = "workflow_run" ]; then + BRANCH_NAME="${{ github.event.workflow_run.head_branch }}" + PR_NUMBER=$(gh pr list --head "${{ github.event.workflow_run.head_branch }}" --state open --json number -R "${{ github.repository }}" --jq '.[0].number') + else + PR_NUMBER="${{ github.event.pull_request.number }}" + fi + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT - # Use full path with leading slash for exact match - if echo "$changed_files" | grep -qE "^${category_repo}/"; then - labels_to_add+=("$label") - else - if echo "$current_labels" | grep -q "$label"; then - labels_to_remove+=("$label") - fi - fi - done + - name: Compute Category Labels for PR + id: compute_labels + env: + GH_TOKEN: ${{ secrets.MONOREPO_BOT_TOKEN }} + run: | + python .github/scripts/pr_category_label.py \ + --repo ${{ github.repository }} \ + --pr ${{ steps.pr_number.outputs.pr_number }} - if [ ${#labels_to_add[@]} -gt 0 ]; then - echo "Applying labels: ${labels_to_add[*]}" - for label in "${labels_to_add[@]}"; do - gh pr edit "${{ github.event.pull_request.number }}" --add-label "$label" - done + - name: Update labels + env: + GH_TOKEN: ${{ secrets.MONOREPO_BOT_TOKEN }} + run: | + if [ -n "${{ steps.compute_labels.outputs.label_remove }}" ]; then + gh pr edit "${{ steps.pr_number.outputs.pr_number }}" --remove-label "${{ steps.compute_labels.outputs.label_remove }}" fi - - if [ ${#labels_to_remove[@]} -gt 0 ]; then - echo "Removing labels: ${labels_to_remove[*]}" - for label in "${labels_to_remove[@]}"; do - gh pr edit "${{ github.event.pull_request.number }}" --remove-label "$label" - done + if [ -n "${{ steps.compute_labels.outputs.label_add }}" ]; then + gh pr edit "${{ steps.pr_number.outputs.pr_number }}" --add-label "${{ steps.compute_labels.outputs.label_add }}" fi + + - name: Sync Custom Labels from Monorepo PR env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.MONOREPO_BOT_TOKEN }} + run: | + python .github/scripts/pr_fanout_sync_labels.py \ + --repo ${{ github.repository }} \ + --pr "${{ steps.pr_number.outputs.pr_number }}" \ + --config ".github/repos-config.json" diff --git a/.github/workflows/pr-close-fanouts.yml b/.github/workflows/pr-close-fanouts.yml new file mode 100644 index 00000000000..577c8e1fe93 --- /dev/null +++ b/.github/workflows/pr-close-fanouts.yml @@ -0,0 +1,44 @@ +name: Close Fanout PRs + +on: + pull_request: + types: + - closed + +# ensure that the workflow is not running for the same PR multiple times at once +# subsequent runs of the workflow will perform no-op if the subrepo PRs are already closed +concurrency: + group: pr-close-fanout-${{ github.event.pull_request.number }} + cancel-in-progress: false + +jobs: + close-fanouts: + runs-on: ubuntu-24.04 + steps: + - name: Checkout workflows + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + sparse-checkout: '.github' + token: ${{ secrets.MONOREPO_BOT_TOKEN }} # since we are touching subrepository PRs + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Install python requirements + run: pip install pydantic + + - name: Set up Git user + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Close fanned out PRs and delete branches + env: + GH_TOKEN: ${{ secrets.MONOREPO_BOT_TOKEN }} + run: | + python .github/scripts/pr_close_fanouts.py \ + --repo "${{ github.repository }}" \ + --pr "${{ github.event.pull_request.number }}" \ + --config ".github/repos-config.json" diff --git a/.github/workflows/pr-fanout.yml b/.github/workflows/pr-fanout.yml new file mode 100644 index 00000000000..90cd60d890e --- /dev/null +++ b/.github/workflows/pr-fanout.yml @@ -0,0 +1,83 @@ +# Fanout Subtree PRs +# ------------------ +# This GitHub Actions workflow detects which subtrees (from a monorepo structure) +# were changed in a pull request, and automatically creates or updates corresponding +# pull requests in their respective original repositories using `git subtree split`. +# +# Steps: +# 1. Checkout the monorepo. +# 2. Set up Python to run detection and fanout scripts. +# 3. Run a Python script to detect which subtrees were changed (based on repos-config.json). +# 4. For each changed subtree, create or update a PR in the corresponding sub-repo. +# 5. (Placeholder) Track or report the child PR status to the monorepo PR. + +name: Fanout Subtree PRs + +on: + pull_request: + types: + - opened + - synchronize + - reopened + +# ensure that the workflow is not running for the same PR multiple times at once +concurrency: + group: pr-fanout-${{ github.event.pull_request.number }} + cancel-in-progress: false + +jobs: + fanout: + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + sparse-checkout: .github + sparse-checkout-cone-mode: true + token: ${{ secrets.MONOREPO_BOT_TOKEN }} # since we are touching subrepository PRs + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + pip install pydantic requests + + - name: Set up Git user + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Detect changed subtrees + id: detect + env: + GH_TOKEN: ${{ secrets.MONOREPO_BOT_TOKEN }} + run: | + python .github/scripts/pr_detect_changed_subtrees.py \ + --repo "${{ github.repository }}" \ + --pr "${{ github.event.pull_request.number }}" \ + --config ".github/repos-config.json" + + - name: Checkout full repo with changed subtrees + if: steps.detect.outputs.subtrees + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + sparse-checkout: | + .github + ${{ steps.detect.outputs.subtrees }} + fetch-depth: 0 # Needed for subtree splitting + token: ${{ secrets.MONOREPO_BOT_TOKEN }} + + - name: Fan out child PRs + if: steps.detect.outputs.subtrees + env: + GH_TOKEN: ${{ secrets.MONOREPO_BOT_TOKEN }} + run: | + python .github/scripts/pr_fanout.py \ + --repo "${{ github.repository }}" \ + --pr "${{ github.event.pull_request.number }}" \ + --subtrees "${{ steps.detect.outputs.subtrees }}" \ + --debug diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..bee8a64b79a --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__