diff --git a/.github/repos-config.json b/.github/repos-config.json new file mode 100644 index 00000000000..15d604c4ae7 --- /dev/null +++ b/.github/repos-config.json @@ -0,0 +1,184 @@ +{ + "repositories": [ + { + "name": "AMDMIGraphX", + "url": "ROCm/AMDMIGraphX", + "branch": "develop" + }, + { + "name": "MIOpen", + "url": "ROCm/MIOpen", + "branch": "develop" + }, + { + "name": "MIVisionX", + "url": "ROCm/MIVisionX", + "branch": "develop" + }, + { + "name": "Tensile", + "url": "ROCm/Tensile", + "branch": "develop" + }, + { + "name": "TransferBench", + "url": "ROCm/TransferBench", + "branch": "develop" + }, + { + "name": "composable_kernel", + "url": "ROCm/composable_kernel", + "branch": "develop" + }, + { + "name": "hipBLAS", + "url": "ROCm/hipBLAS", + "branch": "develop" + }, + { + "name": "hipBLAS-common", + "url": "ROCm/hipBLAS-common", + "branch": "develop" + }, + { + "name": "hipBLASLt", + "url": "ROCm/hipBLASLt", + "branch": "develop" + }, + { + "name": "hipCUB", + "url": "ROCm/hipCUB", + "branch": "develop" + }, + { + "name": "hipFFT", + "url": "ROCm/hipFFT", + "branch": "develop" + }, + { + "name": "hipRAND", + "url": "ROCm/hipRAND", + "branch": "develop" + }, + { + "name": "hipSOLVER", + "url": "ROCm/hipSOLVER", + "branch": "develop" + }, + { + "name": "hipSPARSE", + "url": "ROCm/hipSPARSE", + "branch": "develop" + }, + { + "name": "hipSPARSELt", + "url": "ROCm/hipSPARSELt", + "branch": "develop" + }, + { + "name": "hipTensor", + "url": "ROCm/hipTensor", + "branch": "develop" + }, + { + "name": "hipfort", + "url": "ROCm/hipfort", + "branch": "develop" + }, + { + "name": "Oragami", + "url": "ROCm/Oragami", + "branch": "develop" + }, + { + "name": "rccl", + "url": "ROCm/rccl", + "branch": "develop" + }, + { + "name": "rocAL", + "url": "ROCm/rocAL", + "branch": "develop" + }, + { + "name": "rocALUTION", + "url": "ROCm/rocALUTION", + "branch": "develop" + }, + { + "name": "rocBLAS", + "url": "ROCm/rocBLAS", + "branch": "develop" + }, + { + "name": "rocDecode", + "url": "ROCm/rocDecode", + "branch": "develop" + }, + { + "name": "rocFFT", + "url": "ROCm/rocFFT", + "branch": "develop" + }, + { + "name": "rocJPEG", + "url": "ROCm/rocJPEG", + "branch": "develop" + }, + { + "name": "rocJenkins", + "url": "ROCm/rocJenkins", + "branch": "pong" + }, + { + "name": "rocPRIM", + "url": "ROCm/rocPRIM", + "branch": "develop" + }, + { + "name": "rocPyDecode", + "url": "ROCm/rocPyDecode", + "branch": "develop" + }, + { + "name": "rocRAND", + "url": "ROCm/rocRAND", + "branch": "develop" + }, + { + "name": "rocRoller", + "url": "ROCm/rocRoller", + "branch": "main" + }, + { + "name": "rocSHMEM", + "url": "ROCm/rocSHMEM", + "branch": "develop" + }, + { + "name": "rocSOLVER", + "url": "ROCm/rocSOLVER", + "branch": "develop" + }, + { + "name": "rocSPARSE", + "url": "ROCm/rocSPARSE", + "branch": "develop" + }, + { + "name": "rocThrust", + "url": "ROCm/rocThrust", + "branch": "develop" + }, + { + "name": "rocWMMA", + "url": "ROCm/rocWMMA", + "branch": "develop" + }, + { + "name": "rpp", + "url": "ROCm/rpp", + "branch": "develop" + } + ] +} diff --git a/.github/scripts/apply-labels.py b/.github/scripts/apply-labels.py new file mode 100644 index 00000000000..2eac5bd82f9 --- /dev/null +++ b/.github/scripts/apply-labels.py @@ -0,0 +1,64 @@ +import os +import sys +import yaml +import requests + +def get_existing_labels(repo, token): + headers = {"Authorization": f"token {token}"} + labels = {} + page = 1 + while True: + url = f"https://api.github.com/repos/{repo}/labels?page={page}&per_page=100" + resp = requests.get(url, headers=headers) + if resp.status_code != 200: + raise Exception(f"Failed to fetch existing labels: {resp.text}") + data = resp.json() + if not data: + break + for label in data: + labels[label["name"]] = { + "color": label["color"], + "description": label.get("description", "") + } + page += 1 + return labels + +def create_or_update_label(repo, token, label, existing): + headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github+json" + } + + if label["name"] not in existing: + # Create label + print(f"Creating label: {label['name']}") + url = f"https://api.github.com/repos/{repo}/labels" + resp = requests.post(url, json=label, headers=headers) + else: + # Update if different + current = existing[label["name"]] + if (label["color"].lower() != current["color"].lower() or + label.get("description", "") != current.get("description", "")): + print(f"Updating label: {label['name']}") + url = f"https://api.github.com/repos/{repo}/labels/{label['name']}" + resp = requests.patch(url, json=label, headers=headers) + else: + print(f"Label '{label['name']}' already up to date. Skipping.") + return + + if not resp.ok: + print(f"Failed to apply label {label['name']}: {resp.status_code} {resp.text}") + +def main(label_file): + token = os.environ["GH_TOKEN"] + repo = os.environ["GITHUB_REPO"] + existing = get_existing_labels(repo, token) + + with open(label_file, "r") as f: + labels = yaml.safe_load(f) + + for label in labels: + create_or_update_label(repo, token, label, existing) + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/.github/scripts/collect-labels.py b/.github/scripts/collect-labels.py new file mode 100644 index 00000000000..130b8114cf9 --- /dev/null +++ b/.github/scripts/collect-labels.py @@ -0,0 +1,48 @@ +import json +import os +import sys +import requests +import yaml + +def get_labels(repo, token): + headers = {"Authorization": f"token {token}"} + labels = [] + page = 1 + while True: + url = f"https://api.github.com/repos/{repo}/labels?page={page}&per_page=100" + resp = requests.get(url, headers=headers) + if resp.status_code != 200: + raise Exception(f"Failed to fetch labels from {repo}: {resp.text}") + data = resp.json() + if not data: + break + labels.extend(data) + page += 1 + return labels + +def main(file_path): + with open(file_path, "r") as f: + repos_data = json.load(f)["repositories"] + + token = os.environ["GH_TOKEN"] + all_labels = {} + + for repo_entry in repos_data: + repo_url = repo_entry["url"] + print(f"Collecting labels from {repo_url}") + for label in get_labels(repo_url, token): + name = label["name"] + if name not in all_labels: + all_labels[name] = { + "name": name, + "color": label["color"], + "description": label.get("description", "") + } + + sorted_labels = sorted(all_labels.values(), key=lambda l: l["name"].lower()) + os.makedirs(".github", exist_ok=True) # Ensure the .github directory exists + with open(".github/labels.yml", "w") as out: + yaml.dump(sorted_labels, out, sort_keys=False) + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/.github/scripts/merge-codeowners.py b/.github/scripts/merge-codeowners.py new file mode 100644 index 00000000000..4b64ea61c42 --- /dev/null +++ b/.github/scripts/merge-codeowners.py @@ -0,0 +1,54 @@ +import os +from pathlib import Path + +# Determine monorepo root and output CODEOWNERS path +monorepo_root = Path(__file__).resolve().parents[2] +output_path = monorepo_root / ".github" / "CODEOWNERS" + +merged_entries = [] + +# Walk top-level directories (excluding .github/.git/etc.) +for subdir in monorepo_root.iterdir(): + if subdir.name.startswith(".") or not subdir.is_dir(): + continue + + # Look for CODEOWNERS in root or .github directory of the submodule + candidates = [subdir / "CODEOWNERS", subdir / ".github" / "CODEOWNERS"] + + for codeowners_file in candidates: + if codeowners_file.is_file(): + with codeowners_file.open("r") as f: + for line in f: + stripped = line.strip() + + # Skip empty lines or comments + if not stripped or stripped.startswith("#"): + continue + + parts = stripped.split() + if not parts: + continue + + original_path = parts[0] + owners = " ".join(parts[1:]) + + # Ensure prefixed path starts with a single slash + prefixed_path = ( + f"/{subdir.name.rstrip('/')}{original_path}" + if original_path.startswith("/") + else f"/{subdir.name}/{original_path}" + ) + + merged_entries.append(f"{prefixed_path} {owners}") + +# Sort for consistency +merged_entries.sort() + +# Write merged CODEOWNERS file +output_path.parent.mkdir(parents=True, exist_ok=True) + +with output_path.open("w") as out: + out.write("# Auto-generated CODEOWNERS file\n\n") + out.write("\n".join(merged_entries)) + +print(f"✅ Merged CODEOWNERS written to {output_path}") diff --git a/.github/workflows/apply-labels.yml b/.github/workflows/apply-labels.yml new file mode 100644 index 00000000000..8489352c759 --- /dev/null +++ b/.github/workflows/apply-labels.yml @@ -0,0 +1,32 @@ +name: Apply Labels + +on: + workflow_dispatch: + inputs: + labelFile: + description: 'Path to YAML file with labels' + required: true + default: '.github/labels.yml' + +jobs: + apply-labels: + runs-on: ubuntu-latest + + steps: + - name: Checkout monorepo + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.x + + - name: Install dependencies + run: pip install PyYAML requests + + - name: Apply labels to monorepo + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPO: ${{ github.repository }} + run: | + python .github/scripts/apply-labels.py "${{ github.event.inputs.labelFile }}" diff --git a/.github/workflows/collect-labels.yml b/.github/workflows/collect-labels.yml new file mode 100644 index 00000000000..bb2ee7aec4f --- /dev/null +++ b/.github/workflows/collect-labels.yml @@ -0,0 +1,36 @@ +name: Collect Labels + +on: + workflow_dispatch: + inputs: + repoListFile: + description: 'Path to JSON file with repo list' + required: true + default: '.github/repos-config.json' + +jobs: + collect-labels: + runs-on: ubuntu-latest + + steps: + - name: Checkout monorepo + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.x + + - name: Install dependencies + run: pip install PyYAML requests + + - name: Collect labels from source repos + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python .github/scripts/collect-labels.py "${{ github.event.inputs.repoListFile }}" + + - name: Print generated labels.yml + run: | + echo "Contents of labels.yml:" + cat .github/labels.yml diff --git a/.github/workflows/initial-setup.yml b/.github/workflows/initial-setup.yml new file mode 100644 index 00000000000..2dfbcbc2f2b --- /dev/null +++ b/.github/workflows/initial-setup.yml @@ -0,0 +1,32 @@ +name: Setup EMU Monorepo + +on: + workflow_dispatch: + +env: + MONOREPO_URL: github.com/ROCm/rocm-system.git + MONOREPO_BRANCH: develop + +jobs: + setup-monorepo: + runs-on: ubuntu-latest + steps: + - name: Checkout the EMU Monorepo + uses: actions/checkout@v3 + + - name: Set up Git user + run: | + git config --global user.name "${{ github.actor }}" + git config --global user.email "${{ github.actor }}@users.noreply.github.com" + + - name: Add Repositories to the EMU Monorepo + run: | + for repo in $(cat .github/repos-config.json | jq -r '.repositories[].name'); do + url=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .url") + branch=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .branch") + git subtree add --prefix $repo https://github.com/${url}.git $branch + done + + - name: Push changes to EMU Monorepo + run: | + git push https://${{ env.MONOREPO_URL }} ${{ env.MONOREPO_BRANCH }} diff --git a/.github/workflows/merge-codeowners.yml b/.github/workflows/merge-codeowners.yml new file mode 100644 index 00000000000..3ea81c96a7c --- /dev/null +++ b/.github/workflows/merge-codeowners.yml @@ -0,0 +1,40 @@ +name: Merge CODEOWNERS Files + +on: + workflow_dispatch: + +jobs: + merge-codeowners: + runs-on: ubuntu-latest + + permissions: + contents: write # Required to commit and push changes + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install dependencies + run: pip install pyyaml + + - name: Run merge script + run: python .github/scripts/merge-codeowners.py + + - name: Commit and push if changed + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add .github/CODEOWNERS + if git diff --cached --quiet; then + echo "No changes to commit" + else + git commit -m "chore: merge CODEOWNERS files" + git push + fi diff --git a/.github/workflows/update-subtrees.yml b/.github/workflows/update-subtrees.yml new file mode 100644 index 00000000000..0312e0e42df --- /dev/null +++ b/.github/workflows/update-subtrees.yml @@ -0,0 +1,46 @@ +name: Synchronize Subtrees + +on: + workflow_dispatch: + schedule: + - cron: '0 * * * *' + +env: + MONOREPO_URL: github.com/ROCm/rocm-system.git + MONOREPO_BRANCH: develop + +jobs: + synchronize-subtrees: + runs-on: ubuntu-latest + steps: + - name: Checkout the EMU Monorepo + uses: actions/checkout@v3 + with: + fetch-depth: 0 # needed for git subtree pull/push + token: ${{ secrets.MONOREPO_BOT_TOKEN }} + - name: Set up Git user + run: | + git config --global user.name "${{ github.actor }}" + git config --global user.email "${{ github.actor }}@users.noreply.github.com" + + - name: Update Repositories in the EMU Monorepo + run: | + has_errors=false + for repo in $(cat .github/repos-config.json | jq -r '.repositories[].name'); do + url=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .url") + branch=$(cat .github/repos-config.json | jq -r ".repositories[] | select(.name == \"$repo\") | .branch") + git subtree pull --prefix $repo https://github.com/${url}.git $branch || { + has_errors=true + } + # git subtree push --prefix $repo https://github.com/${url}.git $branch || { + # has_errors=true + # } + done + + if [ "$has_errors" = true ]; then + echo "One or more errors occurred during the repository update." + exit 1 # This will mark the job as failed + else + git push https://${{ env.MONOREPO_URL }} ${{ env.MONOREPO_BRANCH }} + echo "All repositories updated successfully!" + fi diff --git a/README.md b/README.md index 618295e0395..8a929f8f3d9 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,76 @@ -# rocm-libraries -monorepo for rocm libraries +# ROCm System Quick Start Guide for Developers + +Welcome to the monorepo for ROCm libraries! + +This quick start guide includes tips on how to work efficiently with this monorepo style. + +## 📂 Sparse Checkout: Working with a Subset of the Monorepo + +If you only need to work on specific projects within the monorepo, you can use **sparse checkout** to limit the files in your working directory. + +### Enable Sparse Checkout: +```sh +git clone --no-checkout +cd +git sparse-checkout init --cone +git sparse-checkout set repo1 repo2 +git checkout develop +``` +This ensures that only `repo1/` and `repo2/` directories are checked out. + +### Reset Sparse Checkout to Full Repo: +```sh +git sparse-checkout disable +``` + +📌 **Source:** [Git Sparse-Checkout Docs](https://git-scm.com/docs/git-sparse-checkout) + +--- + +## 🔍 Viewing Old History with Subtree Split + +Since the monorepo was created using **git subtree**, historical commit paths are preserved at the top-level, but per-file granularity of the commit history requires extra steps to view the history of changes before the individual repos were added to this monorepo. To view the commit history of a specific project before it was merged: + +### Create a Temporary History Branch: +```sh +git subtree split --prefix=repo1 -b repo1-history +git checkout repo1-history +``` + +Now, you can use the typical logging commands to go through the old commit history. + +📌 **Source:** [Git Subtree: A Quick Guide to Mastering Git Subtree](https://gitscripts.com/git-subtree) + +--- + +## ⚠️ GitHub Web UI Limitations + +The GitHub web interface does not correctly track file history when files are moved into a subdirectory via `git subtree`. To get the prior history of files, use the aforementioned subtree split method. + +--- + +## ⚠️ Per-Project Submodule Limitations + +Submodules that were present in the individual projects are not migrated over to the monorepo when `git subtree` is used. For the reverse direction, updates to submodules on the monorepo are also not propagated to the individual repos with `git subtree push`. If an individual project has submodules, please maintain submodules in both the monorepo and the individual repos during the migration period. Also consider why the submodule is in place, and whether a better solution should be implemented for the monorepo (e.g., shared top-level directory of submodules). + +--- + +## 🔄 Keeping the Monorepo Updated + +This monorepo will run hourly jobs to synchronize the monorepo with the individual repos using the below commands. Reminder that submodules are ignored during this process. + +### Pull Updates from Individual Repo: +```sh +git subtree pull --prefix=repo1 +``` + +### Push Changes Back to the Individual Repo: +```sh +git subtree push --prefix=repo1 +``` + +📌 **Source:** [Git Subtree: A Quick Guide to Mastering Git Subtree](https://gitscripts.com/git-subtree) + +--- + +This guide will be updated as needed! 🚀 Happy coding!