From 7312937e4ce0d7d4a490c5dbc10f221891189193 Mon Sep 17 00:00:00 2001 From: amd-hsivasun Date: Wed, 22 Oct 2025 09:36:01 +0000 Subject: [PATCH] [Github Actions] Added monorepo_source_of_truth flag --- .github/repos-config.json | 69 ++++++++++++------- .github/scripts/pr_detect_changed_subtrees.py | 17 ++--- .github/scripts/repo_config_model.py | 18 +++-- .github/scripts/therock_configure_ci.py | 4 +- .github/workflows/azure-ci-dispatcher.yml | 1 + docs/migration-process.md | 2 +- 6 files changed, 69 insertions(+), 42 deletions(-) diff --git a/.github/repos-config.json b/.github/repos-config.json index 21f0c8bfe54..cbcaa0ce6ed 100644 --- a/.github/repos-config.json +++ b/.github/repos-config.json @@ -6,7 +6,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "tensile", @@ -14,7 +15,8 @@ "branch": "develop", "category": "shared", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "origami", @@ -22,7 +24,8 @@ "branch": "develop", "category": "shared", "auto_subtree_pull": false, - "auto_subtree_push": false + "auto_subtree_push": false, + "monorepo_source_of_truth": true }, { "name": "mxdatagenerator", @@ -30,7 +33,8 @@ "branch": "develop", "category": "shared", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "composablekernel", @@ -38,7 +42,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": true, - "auto_subtree_push": false + "auto_subtree_push": false, + "monorepo_source_of_truth": false }, { "name": "hipblas", @@ -46,7 +51,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "hipblas-common", @@ -54,7 +60,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "hipblaslt", @@ -62,7 +69,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "hipcub", @@ -70,7 +78,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "hipfft", @@ -78,7 +87,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "hiprand", @@ -86,7 +96,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "hipsolver", @@ -94,7 +105,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "hipsparse", @@ -102,7 +114,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "hipsparselt", @@ -110,7 +123,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "rocblas", @@ -118,7 +132,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "rocfft", @@ -126,7 +141,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "rocprim", @@ -134,7 +150,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "rocrand", @@ -142,7 +159,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "rocroller", @@ -150,7 +168,8 @@ "branch": "develop", "category": "shared", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "rocsolver", @@ -158,7 +177,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "rocsparse", @@ -166,7 +186,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "rocthrust", @@ -174,7 +195,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, { "name": "hipdnn", @@ -182,7 +204,8 @@ "branch": "develop", "category": "projects", "auto_subtree_pull": false, - "auto_subtree_push": true + "auto_subtree_push": true, + "monorepo_source_of_truth": true } ] } diff --git a/.github/scripts/pr_detect_changed_subtrees.py b/.github/scripts/pr_detect_changed_subtrees.py index 8e4c2b19686..c0b6020b048 100644 --- a/.github/scripts/pr_detect_changed_subtrees.py +++ b/.github/scripts/pr_detect_changed_subtrees.py @@ -18,6 +18,7 @@ --config : OPTIONAL, path to the repos-config.json file. --require-auto-pull : If set, only include entries with auto_subtree_pull=true. --require-auto-push : If set, only include entries with auto_subtree_push=true. + --require-monorepo-source : If set, only include entries with monorepo_source_of_truth=true. --dry-run : If set, will only log actions without making changes. --debug : If set, enables detailed debug logging. @@ -51,20 +52,12 @@ def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace: parser.add_argument("--config", required=False, default=".github/repos-config.json", help="Path to the repos-config.json file") parser.add_argument("--require-auto-pull", action="store_true", help="Only include entries with auto_subtree_pull=true") parser.add_argument("--require-auto-push", action="store_true", help="Only include entries with auto_subtree_push=true") + parser.add_argument("--require-monorepo-source", action="store_true", help="Only include entries with monorepo_source_of_truth=true") parser.add_argument("--dry-run", action="store_true", help="Print results without writing to GITHUB_OUTPUT.") parser.add_argument("--debug", action="store_true", help="Enable debug logging") return parser.parse_args(argv) -def get_valid_prefixes(config: List[RepoEntry]) -> Set[str]: - """Extract valid subtree prefixes from the configuration.""" - valid_prefixes = { - f"{entry.category}/{entry.name}" - for entry in config - } - logger.debug("Valid subtrees:\n" + "\n".join(sorted(valid_prefixes))) - return valid_prefixes - -def get_valid_prefixes(config: List[RepoEntry], require_auto_pull: bool = False, require_auto_push: bool = False) -> Set[str]: +def get_valid_prefixes(config: List[RepoEntry], require_auto_pull: bool = False, require_auto_push: bool = False, require_monorepo_source: bool = False) -> Set[str]: """Extract valid subtree prefixes from the configuration based on filters.""" valid_prefixes = set() for entry in config: @@ -72,6 +65,8 @@ def get_valid_prefixes(config: List[RepoEntry], require_auto_pull: bool = False, continue if require_auto_push and not getattr(entry, "auto_subtree_push", False): continue + if require_monorepo_source and not getattr(entry, "monorepo_source_of_truth", False): + continue valid_prefixes.add(f"{entry.category}/{entry.name}") logger.debug("Valid subtrees:\n" + "\n".join(sorted(valid_prefixes))) return valid_prefixes @@ -131,7 +126,7 @@ def main(argv=None) -> None: logger.error(f"SHA-based Git CLI fallback failed: {e}") sys.exit(1) - valid_prefixes = get_valid_prefixes(config, args.require_auto_pull, args.require_auto_push) + valid_prefixes = get_valid_prefixes(config, args.require_auto_pull, args.require_auto_push, args.require_monorepo_source) matched_subtrees = find_matched_subtrees(changed_files, valid_prefixes) output_subtrees(matched_subtrees, args.dry_run) diff --git a/.github/scripts/repo_config_model.py b/.github/scripts/repo_config_model.py index 3816d0a6e79..8a3fdfa4f78 100644 --- a/.github/scripts/repo_config_model.py +++ b/.github/scripts/repo_config_model.py @@ -14,7 +14,10 @@ "name": "rocblas", "url": "ROCm/rocBLAS", "branch": "develop", - "category": "projects" + "category": "projects", + "auto_subtree_pull": false, + "auto_subtree_push": true, + "monorepo_source_of_truth": true }, ... ] @@ -22,6 +25,7 @@ """ from typing import List + from pydantic import BaseModel class RepoEntry(BaseModel): @@ -29,10 +33,13 @@ class RepoEntry(BaseModel): Represents a single repository entry in the repos-config.json file. Fields: - name : Name of the project matching packaging file names. Lower-cased and no underscores. (e.g., "rocblas") - url : Individual GitHub org plus repo names in matching case and punctuation. (e.g., "ROCm/rocBLAS") - branch : The base branch of the sub-repo to target (e.g., "develop"). - category : Directory category in the monorepo (e.g., "projects" or "shared"). + name : Name of the project matching packaging file names. Lower-cased and no underscores. (e.g., "rocblas") + url : Individual GitHub org plus repo names in matching case and punctuation. (e.g., "ROCm/rocBLAS") + branch : The base branch of the sub-repo to target (e.g., "develop"). + category : Directory category in the monorepo (e.g., "projects" or "shared"). + auto_subtree_pull : Whether to automatically pull updates from the individual repo to the monorepo. + auto_subtree_push : Whether to automatically push changes from the monorepo to the individual repo. + monorepo_source_of_truth : Whether the monorepo is the source of truth for this project. """ name: str url: str @@ -40,6 +47,7 @@ class RepoEntry(BaseModel): category: str auto_subtree_pull: bool auto_subtree_push: bool + monorepo_source_of_truth: bool class RepoConfig(BaseModel): """ diff --git a/.github/scripts/therock_configure_ci.py b/.github/scripts/therock_configure_ci.py index 2d1db1dbf1f..23e0d424511 100644 --- a/.github/scripts/therock_configure_ci.py +++ b/.github/scripts/therock_configure_ci.py @@ -99,7 +99,7 @@ def retrieve_projects(args): base_ref = args.get("base_ref") modified_paths = get_modified_paths(base_ref) subtrees = get_changed_path_projects(modified_paths) - + # by default, we select full tests test_type = "full" @@ -115,7 +115,7 @@ def retrieve_projects(args): if related_to_therock_ci: subtrees = list(subtree_to_project_map.keys()) test_type = "smoke" - + # for nightly runs, run everything with full tests if args.get("is_nightly"): subtrees = list(subtree_to_project_map.keys()) diff --git a/.github/workflows/azure-ci-dispatcher.yml b/.github/workflows/azure-ci-dispatcher.yml index 954b3dd80b1..3f31994c0db 100644 --- a/.github/workflows/azure-ci-dispatcher.yml +++ b/.github/workflows/azure-ci-dispatcher.yml @@ -95,6 +95,7 @@ jobs: python .github/scripts/pr_detect_changed_subtrees.py \ --repo "${{ github.repository }}" \ --pr "${{ github.event.pull_request.number }}" \ + --require-monorepo-source \ --config ".github/repos-config.json" - name: Cancel in-progress/not-started runs for current PR diff --git a/docs/migration-process.md b/docs/migration-process.md index 4498d77762e..3a3547c3adb 100644 --- a/docs/migration-process.md +++ b/docs/migration-process.md @@ -118,7 +118,7 @@ git push origin develop 1. **Update repos-config.json:** - Update the true/false values in the [`repos-config.json`](/.github/repos-config.json) file that automated workflows use to determine which way the source gets synchronized.. - - `auto_subtree_pull` should now be false, `auto_subtree_push` should now be true for this migrated project. + - `auto_subtree_pull` should now be false, `auto_subtree_push` should now be true for this migrated project. `monorepo_source_of_truth` should be true to reflect the new source of truth is the monorepo. - Make this change on both the `develop` and `release-staging` branches. - https://github.com/ROCm/rocm-libraries/blob/develop/.github/repos-config.json - https://github.com/ROCm/rocm-libraries/blob/release-staging/rocm-rel-7.0/.github/repos-config.json