diff --git a/.claude/commands/upgrade-llvm.md b/.claude/commands/upgrade-llvm.md new file mode 100644 index 000000000..15c0caeca --- /dev/null +++ b/.claude/commands/upgrade-llvm.md @@ -0,0 +1,136 @@ +Upgrade LLVM to a new version. Accepts the target version as argument (e.g., `22.1.4`). + +This is the complete workflow for upgrading the LLVM prebuilt packages that clice depends on. Follow each step in order. Steps that involve CI should use polling (check every ~5 minutes) to wait for completion. + +## Step 1: Trigger LLVM Build + +Trigger the `build-llvm` workflow on GitHub Actions: + +```bash +gh workflow run build-llvm.yml \ + --field llvm_version="" +``` + +- Poll until all 14 matrix builds complete (~2-3 hours), note the workflow run ID + +## Step 2: Download Local Platform Artifact + +Download the artifact matching the development machine: + +```bash +gh run view +gh run download -n x64-linux-gnu-releasedbg.tar.xz -D .llvm-download +mkdir -p .llvm +tar -xf .llvm-download/x64-linux-gnu-releasedbg.tar.xz -C .llvm +``` + +Configure clice to build against it: + +```bash +pixi run cmake-config RelWithDebInfo ON -- "-DLLVM_INSTALL_PATH=.llvm" +pixi run cmake-build RelWithDebInfo +``` + +Compilation will likely fail — that's what Step 3 addresses. + +## Step 3: Adapt API Changes + +Fix LLVM API breaking changes based on compilation errors. Common categories: + +- **Header path changes**: e.g., `clang/Driver/Options.h` → `clang/Options/Options.h` +- **Namespace migrations**: e.g., `clang::driver::options` → `clang::options` +- **Type system changes**: e.g., ElaboratedType removal, NestedNameSpecifier pointer→value +- **Function signature changes**: e.g., `createDiagnostics` parameter changes +- **Type merges/splits**: e.g., DependentTemplateSpecializationType → TemplateSpecializationType + +Strategy: + +1. Fix header/namespace changes first (mechanical) +2. Fix type system and signature changes (requires understanding semantics) +3. Update test expectations (AST structure changes affect test output) +4. Ensure `pixi run unit-test RelWithDebInfo` passes + +When a fix is not obvious, read the LLVM source code to understand the new API. If `../llvm-project` exists locally, use it. Otherwise, look up the upstream commit/PR on GitHub. + +## Step 4: Create PR + +```bash +git checkout -b chore/upgrade-llvm-XX +git add -A +git commit -m "chore: upgrade LLVM to XX.Y.Z" +git push -u origin chore/upgrade-llvm-XX +gh pr create --title "chore: upgrade LLVM to XX.Y.Z" --body "..." +``` + +CI will fail at this point (manifest hashes are stale) — this is expected. + +## Step 5: Run Release LLVM Workflow + +Trigger `release-llvm` to build pruned packages: + +```bash +gh workflow run release-llvm.yml \ + --ref \ + --field source_run_id="" \ + --field llvm_version="" +``` + +This will: discover unused libs → create clice-llvm release → repackage with pruning. Poll until complete. + +## Step 6: Update Version + +Update the version string in `cmake/package.cmake`: + +``` +setup_llvm("") +``` + +Commit and push: + +```bash +git add cmake/package.cmake +git commit -m "chore: update LLVM to " +git push +``` + +Poll CI until all platforms pass. CMake downloads the correct artifact automatically based on the version and platform — no manifest file needed. + +## Step 7: Write LLVM Changelog (REQUIRED) + +**Every LLVM upgrade MUST append to `docs/en/changelog/llvm-changelog.md`.** + +Add a new H2 section (e.g., `## LLVM 22 → 23`) documenting all breaking changes encountered. For each API change, record: + +- Change description +- Upstream commit hash +- PR number (link to `https://github.com/llvm/llvm-project/pull/`) +- Impact on clice + +To find upstream commits, search the LLVM git history between version tags: + +```bash +# If ../llvm-project exists locally: +cd ../llvm-project +git log --oneline llvmorg-..llvmorg- -- clang/include/clang/AST/ +``` + +If the LLVM source is not available locally, look up changes on GitHub by searching the LLVM repository commit history. + +Group changes by category (Type System, NNS, Driver/Frontend, Other) with a table per category. See the existing `LLVM 21 → 22` section as a template. + +## Step 8: Report to User + +Present a summary to the user and **wait for confirmation** before considering the upgrade complete. The summary should include: + +- All API changes that were adapted and how they were resolved +- All test expectation changes (snapshot updates, assertion value changes) and why +- Any unavoidable behavior changes from upstream LLVM (e.g., TypePrinter output differences, type sugar changes) that affect user-visible features like hover +- The LLVM changelog that was written + +The user decides whether all changes are acceptable or if adjustments are needed. Do NOT push final changes or mark the work as done until the user confirms. + +## Notes + +- **Artifact size limit**: GitHub Release max 2GB per file. macOS LTO artifacts are largest, currently ~1.7GB with xz -9e. +- **Pruning safety**: discover phase validates by deleting .a files one by one and rebuilding clice. clang-tidy modules can't be deleted due to force-link. +- **Private headers**: clice depends on private Clang Sema headers (TreeTransform.h etc.), copied from source during `build-llvm.py`. Users must use our packaged LLVM. diff --git a/.github/workflows/build-llvm.yml b/.github/workflows/build-llvm.yml index ff810adb7..a86f5eff9 100644 --- a/.github/workflows/build-llvm.yml +++ b/.github/workflows/build-llvm.yml @@ -7,20 +7,10 @@ on: description: "LLVM version to build (e.g., 21.1.8)" required: true type: string - skip_upload: - description: "Skip upload and PR creation (build-only mode)" - required: false - type: boolean - default: false - skip_pr: - description: "Skip PR creation (upload only, no PR)" - required: false - type: boolean - default: false - pull_request: - # if you want to run this workflow, change the branch name to main, - # if you want to turn off it, change it to non existent branch. - branches: [main-turn-off] + +concurrency: + group: build-llvm-${{ github.ref }} + cancel-in-progress: true jobs: build: @@ -123,6 +113,41 @@ jobs: with: environments: ${{ matrix.pixi_env || 'package' }} + - name: Free Disk Space (macOS) + if: runner.os == 'macOS' + run: | + echo "=== Before cleanup ===" + df -h / + + # Remove iOS/tvOS/watchOS/visionOS simulators + if [ -d "/Library/Developer/CoreSimulator" ]; then + sudo rm -rf /Library/Developer/CoreSimulator + echo "Removed CoreSimulator" + fi + + # Remove unnecessary Xcode platforms (keep macOS only) + XCODE_PATH="$(xcode-select -p)" + PLATFORMS_DIR="${XCODE_PATH}/Platforms" + for platform in iPhoneOS.platform iPhoneSimulator.platform AppleTVOS.platform AppleTVSimulator.platform WatchOS.platform WatchSimulator.platform XROS.platform XRSimulator.platform; do + if [ -d "${PLATFORMS_DIR}/${platform}" ]; then + sudo rm -rf "${PLATFORMS_DIR}/${platform}" + echo "Removed ${platform}" + fi + done + + # Remove Android SDK + sudo rm -rf ~/Library/Android + + # Remove .NET/PowerShell + sudo rm -rf /usr/local/share/dotnet + sudo rm -rf /usr/local/share/powershell + + # Remove Haskell + sudo rm -rf ~/.ghcup + + echo "=== After cleanup ===" + df -h / + - name: Clone llvm-project shell: bash run: | @@ -130,13 +155,6 @@ jobs: echo "Cloning LLVM ${VERSION}..." git clone --branch "llvmorg-${VERSION}" --depth 1 https://github.com/llvm/llvm-project.git .llvm - - name: Validate distribution components - shell: bash - run: | - python3 scripts/validate-llvm-components.py \ - --llvm-src=.llvm \ - --components-file=scripts/llvm-components.json - - name: Build LLVM (install-distribution) shell: bash run: | @@ -145,168 +163,45 @@ jobs: if [[ -n "${{ matrix.target_triple }}" ]]; then EXTRA_ARGS="--target-triple=${{ matrix.target_triple }}" fi - pixi run -e "$ENV" build-llvm \ + pixi run -e "$ENV" python3 scripts/build-llvm.py \ --llvm-src=.llvm \ --mode="${{ matrix.llvm_mode }}" \ --lto="${{ matrix.lto }}" \ --build-dir=build \ ${EXTRA_ARGS} - - name: Build clice using installed LLVM - if: ${{ !matrix.target_triple }} - shell: bash - run: | - pixi run cmake-config ${{ matrix.llvm_mode }} ON -- \ - "-DCLICE_ENABLE_LTO=${{ matrix.lto }}" \ - "-DLLVM_INSTALL_PATH=.llvm/build-install" - pixi run cmake-build ${{ matrix.llvm_mode }} - - - name: Build clice using installed LLVM (cross-compile) - if: ${{ matrix.target_triple }} - shell: bash - run: | - ENV="${{ matrix.pixi_env || 'package' }}" - pixi run -e "$ENV" cmake-config ${{ matrix.llvm_mode }} ON -- \ - "-DCLICE_ENABLE_LTO=${{ matrix.lto }}" \ - "-DCLICE_TARGET_TRIPLE=${{ matrix.target_triple }}" \ - "-DLLVM_INSTALL_PATH=.llvm/build-install" - pixi run -e "$ENV" cmake-build ${{ matrix.llvm_mode }} - - - name: Verify cross-compiled binary architecture - if: ${{ matrix.target_triple && runner.os != 'Windows' }} - shell: bash - run: | - BINARY="build/${{ matrix.llvm_mode }}/bin/clice" - echo "Binary info:" - file "$BINARY" - case "${{ matrix.target_triple }}" in - aarch64-linux-gnu) file "$BINARY" | grep -q "aarch64" ;; - x86_64-apple-darwin) file "$BINARY" | grep -q "x86_64" ;; - esac - - - name: Upload cross-compiled clice for functional test - if: ${{ matrix.target_triple && matrix.lto == 'OFF' }} - uses: actions/upload-artifact@v7 - with: - name: cross-clice-${{ matrix.target_triple }}-${{ matrix.llvm_mode }} - path: | - build/${{ matrix.llvm_mode }}/bin/ - build/${{ matrix.llvm_mode }}/lib/ - if-no-files-found: error - retention-days: 1 - - - name: Run tests - if: ${{ !matrix.target_triple }} - shell: bash - run: pixi run test ${{ matrix.llvm_mode }} - - # Prune is only supported for native builds (requires linking clice to test). - # Cross-compiled targets reuse the native prune manifest of the same OS. - - name: Prune LLVM static libraries (Debug/RelWithDebInfo no LTO) - if: (!matrix.target_triple) && (matrix.llvm_mode == 'Debug' || (matrix.llvm_mode == 'RelWithDebInfo' && matrix.lto == 'OFF')) - shell: bash - run: | - MANIFEST="pruned-libs-${{ matrix.os }}.json" - echo "LLVM_PRUNED_MANIFEST=${MANIFEST}" >> "${GITHUB_ENV}" - python3 scripts/prune-llvm-bin.py \ - --action discover \ - --install-dir ".llvm/build-install/lib" \ - --build-dir "build/${{ matrix.llvm_mode }}" \ - --max-attempts 60 \ - --sleep-seconds 60 \ - --manifest "${MANIFEST}" - - - name: Upload pruned-libs manifest - if: (!matrix.target_triple) && matrix.llvm_mode == 'RelWithDebInfo' && matrix.lto == 'OFF' - uses: actions/upload-artifact@v7 - with: - name: llvm-pruned-libs-${{ matrix.os }} - path: ${{ env.LLVM_PRUNED_MANIFEST }} - if-no-files-found: error - compression-level: 0 - - - name: Apply pruned-libs manifest (RelWithDebInfo + LTO, native only) - if: (!matrix.target_triple) && matrix.llvm_mode == 'RelWithDebInfo' && matrix.lto == 'ON' - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - MANIFEST="pruned-libs-${{ matrix.os }}.json" - python3 scripts/prune-llvm-bin.py \ - --action apply \ - --manifest "${MANIFEST}" \ - --install-dir ".llvm/build-install/lib" \ - --build-dir "build/${{ matrix.llvm_mode }}" \ - --gh-run-id "${{ github.run_id }}" \ - --gh-artifact "llvm-pruned-libs-${{ matrix.os }}" \ - --gh-download-dir "artifacts" \ - --max-attempts 60 \ - --sleep-seconds 60 - - # For cross-compiled LTO builds, apply the native prune manifest. - # The unused library set is arch-independent (same API surface). - - name: Apply pruned-libs manifest (cross-compile + LTO) - if: matrix.target_triple && matrix.lto == 'ON' - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - MANIFEST="pruned-libs-${{ matrix.os }}.json" - python3 scripts/prune-llvm-bin.py \ - --action apply \ - --manifest "${MANIFEST}" \ - --install-dir ".llvm/build-install/lib" \ - --build-dir "build/${{ matrix.llvm_mode }}" \ - --gh-run-id "${{ github.run_id }}" \ - --gh-artifact "llvm-pruned-libs-${{ matrix.os }}" \ - --gh-download-dir "artifacts" \ - --max-attempts 60 \ - --sleep-seconds 60 - - name: Package LLVM install directory shell: bash run: | - MODE_TAG="releasedbg" - if [[ "${{ matrix.llvm_mode }}" == "Debug" ]]; then - MODE_TAG="debug" - fi - - # Determine arch/platform/toolchain from target triple or runner OS + # Determine platform/arch from target triple or runner OS if [[ -n "${{ matrix.target_triple }}" ]]; then case "${{ matrix.target_triple }}" in - x86_64-apple-darwin) - ARCH="x64"; PLATFORM="macos"; TOOLCHAIN="clang" ;; - aarch64-linux-gnu) - ARCH="aarch64"; PLATFORM="linux"; TOOLCHAIN="gnu" ;; - aarch64-pc-windows-msvc) - ARCH="aarch64"; PLATFORM="windows"; TOOLCHAIN="msvc" ;; + x86_64-apple-darwin) PLATFORM="macos"; ARCH="x64" ;; + aarch64-linux-gnu) PLATFORM="linux"; ARCH="arm64" ;; + aarch64-pc-windows-msvc) PLATFORM="windows"; ARCH="arm64" ;; esac else - ARCH="x64" - PLATFORM="linux" - TOOLCHAIN="gnu" + PLATFORM="linux"; ARCH="x64" if [[ "${{ matrix.os }}" == windows-* ]]; then PLATFORM="windows" - TOOLCHAIN="msvc" elif [[ "${{ matrix.os }}" == macos-* ]]; then - ARCH="arm64" - PLATFORM="macos" - TOOLCHAIN="clang" + PLATFORM="macos"; ARCH="arm64" fi fi + case "$PLATFORM" in + linux) TOOLCHAIN="gnu" ;; + macos) TOOLCHAIN="clang" ;; + windows) TOOLCHAIN="msvc" ;; + esac + if [[ "${{ matrix.llvm_mode }}" == "Debug" ]]; then MODE_TAG="debug"; else MODE_TAG="releasedbg"; fi SUFFIX="" - if [[ "${{ matrix.lto }}" == "ON" ]]; then - SUFFIX="-lto" - fi - if [[ "${{ matrix.llvm_mode }}" == "Debug" && "${{ matrix.os }}" != windows-* ]]; then - SUFFIX="${SUFFIX}-asan" - fi - + if [[ "${{ matrix.lto }}" == "ON" ]]; then SUFFIX+="-lto"; fi + if [[ "${{ matrix.llvm_mode }}" == "Debug" && "${{ matrix.os }}" != windows-* ]]; then SUFFIX+="-asan"; fi ARCHIVE="${ARCH}-${PLATFORM}-${TOOLCHAIN}-${MODE_TAG}${SUFFIX}.tar.xz" set -eo pipefail - tar -C .llvm -cf - build-install | xz -T0 -9 -c > "${ARCHIVE}" + tar -C .llvm/build-install -cf - . | xz -T0 -9 -c > "${ARCHIVE}" echo "LLVM_INSTALL_ARCHIVE=${ARCHIVE}" >> "${GITHUB_ENV}" - name: Upload LLVM install artifact @@ -315,140 +210,3 @@ jobs: name: ${{ env.LLVM_INSTALL_ARCHIVE }} path: ${{ env.LLVM_INSTALL_ARCHIVE }} if-no-files-found: error - - test-cross: - needs: build - strategy: - fail-fast: false - matrix: - include: - - os: macos-15-intel - llvm_mode: RelWithDebInfo - target_triple: x86_64-apple-darwin - - os: ubuntu-24.04-arm - llvm_mode: RelWithDebInfo - target_triple: aarch64-linux-gnu - - os: windows-11-arm - llvm_mode: RelWithDebInfo - target_triple: aarch64-pc-windows-msvc - runs-on: ${{ matrix.os }} - steps: - - name: Checkout repository - uses: actions/checkout@v7 - with: - persist-credentials: false - - - uses: ./.github/actions/setup-pixi - with: - environments: test-run - - - name: Download cross-compiled clice - uses: actions/download-artifact@v8 - with: - name: cross-clice-${{ matrix.target_triple }}-${{ matrix.llvm_mode }} - path: build/${{ matrix.llvm_mode }}/ - - - name: Make binaries executable - if: runner.os != 'Windows' - run: chmod +x build/${{ matrix.llvm_mode }}/bin/* - - - name: Run tests - run: pixi run -e test-run test ${{ matrix.llvm_mode }} - - upload: - needs: build - if: ${{ !cancelled() && inputs.llvm_version && !inputs.skip_upload }} - runs-on: ubuntu-24.04 - permissions: - contents: read - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Download all build artifacts - env: - GH_TOKEN: ${{ github.token }} - run: scripts/download-llvm.sh "${{ github.run_id }}" - - - name: Upload to clice-llvm - env: - GH_TOKEN: ${{ secrets.UPLOAD_LLVM }} - TARGET_REPO: clice-io/clice-llvm - run: python3 scripts/upload-llvm.py "${{ inputs.llvm_version }}" "${TARGET_REPO}" "${{ github.run_id }}" - - - name: Save manifest for update-clice job - uses: actions/upload-artifact@v7 - with: - name: llvm-manifest-final - path: artifacts/llvm-manifest.json - if-no-files-found: error - compression-level: 0 - - update-clice: - needs: upload - if: ${{ !inputs.skip_pr }} - runs-on: ubuntu-24.04 - permissions: - contents: write - pull-requests: write - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Download manifest - uses: actions/download-artifact@v8 - with: - name: llvm-manifest-final - path: . - - - name: Update manifest and version - run: | - python3 scripts/update-llvm-version.py \ - --version "${{ inputs.llvm_version }}" \ - --manifest-src llvm-manifest.json \ - --manifest-dest config/llvm-manifest.json \ - --package-cmake cmake/package.cmake - - - name: Create or update PR - env: - GH_TOKEN: ${{ github.token }} - run: | - VERSION="${{ inputs.llvm_version }}" - BRANCH="chore/update-llvm-${VERSION}" - RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - RELEASE_URL="https://github.com/clice-io/clice-llvm/releases/tag/${VERSION}" - - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git checkout -b "${BRANCH}" - git add config/llvm-manifest.json cmake/package.cmake - git commit -m "chore: update LLVM to ${VERSION}" - git push --force-with-lease origin "${BRANCH}" - - # Check if PR already exists for this branch - EXISTING_PR=$(gh pr list --head "${BRANCH}" --json number --jq '.[0].number // empty') - - BODY="$(cat < Auto-generated by build-llvm workflow - EOF - )" - - if [[ -n "${EXISTING_PR}" ]]; then - echo "Updating existing PR #${EXISTING_PR}" - gh pr edit "${EXISTING_PR}" --body "${BODY}" - else - gh pr create \ - --title "chore: update LLVM to ${VERSION}" \ - --body "${BODY}" \ - --base main - fi diff --git a/.github/workflows/check-format.yml b/.github/workflows/check-format.yml index 6ad22dc91..59b8fb5ad 100644 --- a/.github/workflows/check-format.yml +++ b/.github/workflows/check-format.yml @@ -16,12 +16,6 @@ jobs: with: environments: format - - name: Validate update-llvm-version.py can still patch package.cmake - run: | - python3 scripts/update-llvm-version.py --check \ - --manifest-dest config/llvm-manifest.json \ - --package-cmake cmake/package.cmake - - name: Run formatter run: pixi run format continue-on-error: true diff --git a/.github/workflows/release-llvm.yml b/.github/workflows/release-llvm.yml new file mode 100644 index 000000000..e229f8431 --- /dev/null +++ b/.github/workflows/release-llvm.yml @@ -0,0 +1,176 @@ +name: release llvm + +on: + workflow_dispatch: + inputs: + source_run_id: + description: "Run ID from build-llvm containing LLVM artifacts" + required: true + type: string + llvm_version: + description: "LLVM version (e.g. 22.1.4)" + required: true + type: string + +concurrency: + group: release-llvm + cancel-in-progress: true + +env: + SOURCE_RUN_ID: ${{ inputs.source_run_id }} + LLVM_VERSION: ${{ inputs.llvm_version }} + +jobs: + discover: + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + artifact: x64-linux-gnu-releasedbg.tar.xz + - os: macos-15 + artifact: arm64-macos-clang-releasedbg.tar.xz + - os: windows-2025 + artifact: x64-windows-msvc-releasedbg.tar.xz + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + + - uses: ./.github/actions/setup-pixi + with: + environments: package + + - name: Free Disk Space (macOS) + if: runner.os == 'macOS' + run: | + if [ -d "/Library/Developer/CoreSimulator" ]; then + sudo rm -rf /Library/Developer/CoreSimulator + fi + XCODE_PATH="$(xcode-select -p)" + PLATFORMS_DIR="${XCODE_PATH}/Platforms" + for p in iPhoneOS.platform iPhoneSimulator.platform AppleTVOS.platform AppleTVSimulator.platform WatchOS.platform WatchSimulator.platform XROS.platform XRSimulator.platform; do + [ -d "${PLATFORMS_DIR}/${p}" ] && sudo rm -rf "${PLATFORMS_DIR}/${p}" + done + sudo rm -rf ~/Library/Android /usr/local/share/dotnet /usr/local/share/powershell ~/.ghcup + + - name: Download LLVM artifact + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + gh run download "${{ env.SOURCE_RUN_ID }}" \ + -n "${{ matrix.artifact }}" -D .llvm-download + mkdir -p .llvm + tar -xf ".llvm-download/${{ matrix.artifact }}" -C .llvm + rm -rf .llvm-download + + - name: Build clice + shell: bash + run: | + pixi run cmake-config RelWithDebInfo ON -- \ + "-DLLVM_INSTALL_PATH=.llvm" + pixi run cmake-build RelWithDebInfo + + - name: Discover unused libraries + shell: bash + run: | + python3 scripts/release-llvm.py discover \ + --install-dir .llvm/lib \ + --build-dir build/RelWithDebInfo \ + --manifest pruned-libs.json \ + --skip-pattern "*Tidy*Module*" + + - name: Print prune results + shell: bash + run: | + python3 -c " + import json + data = json.load(open('pruned-libs.json')) + removed = data['removed'] + total = data.get('total_saved_bytes', 0) + print(f'Libraries prunable: {len(removed)}') + print(f'Space saved: {total / 1048576:.1f} MB') + for entry in sorted(removed, key=lambda e: e['size'], reverse=True)[:20]: + print(f' {entry[\"size\"] / 1048576:>7.1f} MB {entry[\"name\"]}') + " + + - name: Upload manifest + uses: actions/upload-artifact@v7 + with: + name: prune-manifest-${{ matrix.os }} + path: pruned-libs.json + + create-release: + needs: discover + runs-on: ubuntu-24.04 + steps: + - name: Create or reuse release + env: + GH_TOKEN: ${{ secrets.UPLOAD_LLVM }} + run: | + TAG="${{ env.LLVM_VERSION }}" + REPO="clice-io/clice-llvm" + NOTES="LLVM ${TAG} — build run https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + if gh release view "$TAG" --repo "$REPO" &>/dev/null; then + gh release edit "$TAG" --repo "$REPO" --notes "$NOTES" + else + gh release create "$TAG" --repo "$REPO" --title "$TAG" --notes "$NOTES" + fi + + repackage: + needs: [discover, create-release] + runs-on: ubuntu-24.04 + permissions: + contents: read + actions: read + strategy: + fail-fast: false + matrix: + artifact: + - arm64-linux-gnu-releasedbg-lto.tar.xz + - arm64-linux-gnu-releasedbg.tar.xz + - arm64-windows-msvc-releasedbg-lto.tar.xz + - arm64-windows-msvc-releasedbg.tar.xz + - arm64-macos-clang-debug-asan.tar.xz + - arm64-macos-clang-releasedbg-lto.tar.xz + - arm64-macos-clang-releasedbg.tar.xz + - x64-linux-gnu-debug-asan.tar.xz + - x64-linux-gnu-releasedbg-lto.tar.xz + - x64-linux-gnu-releasedbg.tar.xz + - x64-macos-clang-releasedbg-lto.tar.xz + - x64-macos-clang-releasedbg.tar.xz + - x64-windows-msvc-releasedbg-lto.tar.xz + - x64-windows-msvc-releasedbg.tar.xz + steps: + - uses: actions/checkout@v7 + + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + + - name: Download prune manifests + uses: actions/download-artifact@v8 + with: + pattern: prune-manifest-* + path: manifests + + - name: Repackage + env: + GH_TOKEN: ${{ github.token }} + run: | + python3 scripts/release-llvm.py repackage \ + --source-run-id "${{ env.SOURCE_RUN_ID }}" \ + --manifests-dir manifests \ + --output-dir artifacts \ + --max-parallel 1 \ + --artifacts "${{ matrix.artifact }}" + + - name: Upload to release + env: + GH_TOKEN: ${{ secrets.UPLOAD_LLVM }} + run: | + gh release upload "${{ env.LLVM_VERSION }}" \ + "artifacts/${{ matrix.artifact }}" \ + --repo clice-io/clice-llvm \ + --clobber diff --git a/.github/workflows/upload-llvm.yml b/.github/workflows/upload-llvm.yml deleted file mode 100644 index 4d1900c6b..000000000 --- a/.github/workflows/upload-llvm.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: upload-llvm - -permissions: - contents: write - -on: - pull_request: - # if you want to run this workflow, change the branch name to main, - # if you want to turn off it, change it to non existent branch. - branches: [main-turn-off] - - workflow_dispatch: - inputs: - workflow_id: - description: "Workflow run ID to pull artifacts from" - required: true - type: string - version: - description: "Release version/tag to publish (e.g., v1.2.3)" - required: true - type: string - -jobs: - upload: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Download artifacts from workflow - env: - GH_TOKEN: ${{ github.token }} - run: scripts/download-llvm.sh "${{ inputs.workflow_id }}" - - - name: Recreate release with artifacts - env: - GH_TOKEN: ${{ secrets.UPLOAD_LLVM }} - TARGET_REPO: clice-io/clice-llvm - run: python3 scripts/upload-llvm.py "${{ inputs.version }}" "${TARGET_REPO}" "${{ inputs.workflow_id }}" diff --git a/cmake/llvm.cmake b/cmake/llvm.cmake index 694857ef5..2f5b1e2cb 100644 --- a/cmake/llvm.cmake +++ b/cmake/llvm.cmake @@ -1,131 +1,120 @@ include_guard() +include(FetchContent) -function(setup_llvm LLVM_VERSION) - find_package(Python3 COMPONENTS Interpreter REQUIRED) - - set(LLVM_SETUP_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/.llvm/setup-llvm.json") - set(LLVM_SETUP_SCRIPT "${PROJECT_SOURCE_DIR}/scripts/setup-llvm.py") - set(LLVM_SETUP_ARGS - "--version" "${LLVM_VERSION}" - "--build-type" "${CMAKE_BUILD_TYPE}" - "--binary-dir" "${CMAKE_CURRENT_BINARY_DIR}" - "--manifest" "${PROJECT_SOURCE_DIR}/config/llvm-manifest.json" - "--output" "${LLVM_SETUP_OUTPUT}" - ) - - if(CLICE_ENABLE_LTO) - list(APPEND LLVM_SETUP_ARGS "--enable-lto") - endif() - - if(DEFINED LLVM_INSTALL_PATH AND NOT LLVM_INSTALL_PATH STREQUAL "") - list(APPEND LLVM_SETUP_ARGS "--install-path" "${LLVM_INSTALL_PATH}") - endif() - - if(DEFINED CLICE_OFFLINE_BUILD AND CLICE_OFFLINE_BUILD) - list(APPEND LLVM_SETUP_ARGS "--offline") - endif() - +function(_download_llvm LLVM_VERSION) if(DEFINED CLICE_TARGET_TRIPLE) if(CLICE_TARGET_TRIPLE MATCHES "linux") - list(APPEND LLVM_SETUP_ARGS "--target-platform" "Linux") + set(_PLATFORM "linux") + set(_TOOLCHAIN "gnu") elseif(CLICE_TARGET_TRIPLE MATCHES "darwin") - list(APPEND LLVM_SETUP_ARGS "--target-platform" "macosx") + set(_PLATFORM "macos") + set(_TOOLCHAIN "clang") elseif(CLICE_TARGET_TRIPLE MATCHES "windows") - list(APPEND LLVM_SETUP_ARGS "--target-platform" "Windows") + set(_PLATFORM "windows") + set(_TOOLCHAIN "msvc") + else() + message(FATAL_ERROR "Unsupported platform: ${CLICE_TARGET_TRIPLE}") endif() if(CLICE_TARGET_TRIPLE MATCHES "^aarch64") - list(APPEND LLVM_SETUP_ARGS "--target-arch" "arm64") + set(_ARCH "arm64") elseif(CLICE_TARGET_TRIPLE MATCHES "^x86_64") - list(APPEND LLVM_SETUP_ARGS "--target-arch" "x64") + set(_ARCH "x64") + else() + message(FATAL_ERROR "Unsupported arch: ${CLICE_TARGET_TRIPLE}") + endif() + else() + if(WIN32) + set(_PLATFORM "windows") + set(_TOOLCHAIN "msvc") + elseif(APPLE) + set(_PLATFORM "macos") + set(_TOOLCHAIN "clang") + else() + set(_PLATFORM "linux") + set(_TOOLCHAIN "gnu") + endif() + + if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|ARM64") + set(_ARCH "arm64") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64|x64") + set(_ARCH "x64") + else() + message(FATAL_ERROR "Unsupported processor: ${CMAKE_SYSTEM_PROCESSOR}") endif() endif() - execute_process( - COMMAND "${Python3_EXECUTABLE}" "${LLVM_SETUP_SCRIPT}" ${LLVM_SETUP_ARGS} - RESULT_VARIABLE LLVM_SETUP_RESULT - OUTPUT_VARIABLE LLVM_SETUP_STDOUT - ERROR_VARIABLE LLVM_SETUP_STDERR - ECHO_OUTPUT_VARIABLE - ECHO_ERROR_VARIABLE - COMMAND_ERROR_IS_FATAL ANY - ) + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(_MODE "debug") + else() + set(_MODE "releasedbg") + endif() - file(READ "${LLVM_SETUP_OUTPUT}" LLVM_SETUP_JSON) - string(JSON LLVM_INSTALL_PATH GET "${LLVM_SETUP_JSON}" install_path) - string(JSON LLVM_CMAKE_DIR GET "${LLVM_SETUP_JSON}" cmake_dir) - set(LLVM_INSTALL_PATH "${LLVM_INSTALL_PATH}" CACHE PATH "Path to LLVM installation" FORCE) - set(LLVM_CMAKE_DIR "${LLVM_CMAKE_DIR}" CACHE PATH "Path to LLVM CMake files" FORCE) + set(_SUFFIX "") + if(CLICE_ENABLE_LTO) + string(APPEND _SUFFIX "-lto") + endif() + if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT WIN32) + string(APPEND _SUFFIX "-asan") + endif() - get_filename_component(LLVM_INSTALL_PATH "${LLVM_INSTALL_PATH}" ABSOLUTE) + set(_FILENAME "${_ARCH}-${_PLATFORM}-${_TOOLCHAIN}-${_MODE}${_SUFFIX}.tar.xz") + string(REPLACE "+" "%2B" _URL_VERSION "${LLVM_VERSION}") - if(NOT EXISTS "${LLVM_INSTALL_PATH}") - message(FATAL_ERROR "Error: The specified LLVM_INSTALL_PATH does not exist: ${LLVM_INSTALL_PATH}") + FetchContent_Declare(llvm_prebuilt + URL "https://github.com/clice-io/clice-llvm/releases/download/${_URL_VERSION}/${_FILENAME}" + SOURCE_SUBDIR _none + ) + FetchContent_MakeAvailable(llvm_prebuilt) + + set(LLVM_INSTALL_PATH "${llvm_prebuilt_SOURCE_DIR}" PARENT_SCOPE) +endfunction() + +function(setup_llvm LLVM_VERSION) + if(DEFINED LLVM_INSTALL_PATH AND NOT LLVM_INSTALL_PATH STREQUAL "") + get_filename_component(LLVM_INSTALL_PATH "${LLVM_INSTALL_PATH}" ABSOLUTE) + elseif(DEFINED CLICE_OFFLINE_BUILD AND CLICE_OFFLINE_BUILD) + message(FATAL_ERROR "LLVM_INSTALL_PATH must be set in offline mode") + else() + _download_llvm("${LLVM_VERSION}") endif() - # set llvm include and lib path + set(LLVM_INSTALL_PATH "${LLVM_INSTALL_PATH}" CACHE PATH "LLVM install" FORCE) + + find_package(LLVM REQUIRED CONFIG + PATHS "${LLVM_INSTALL_PATH}/lib/cmake/llvm" NO_DEFAULT_PATH) + find_package(Clang REQUIRED CONFIG + PATHS "${LLVM_INSTALL_PATH}/lib/cmake/clang" NO_DEFAULT_PATH) + + llvm_map_components_to_libnames(LLVM_RESOLVED + support frontendopenmp option targetparser) + add_library(llvm-libs INTERFACE IMPORTED) + target_link_libraries(llvm-libs INTERFACE + ${LLVM_RESOLVED} + clangAST clangASTMatchers clangBasic clangDriver + clangFormat clangFrontend clangLex clangSema clangSerialization + clangTidy clangTidyUtils + clangTidyAbseilModule clangTidyAlteraModule clangTidyAndroidModule + clangTidyBoostModule clangTidyBugproneModule clangTidyCERTModule + clangTidyConcurrencyModule clangTidyCppCoreGuidelinesModule + clangTidyDarwinModule clangTidyFuchsiaModule + clangTidyGoogleModule clangTidyHICPPModule clangTidyLinuxKernelModule + clangTidyLLVMModule clangTidyLLVMLibcModule clangTidyMiscModule + clangTidyModernizeModule clangTidyMPIModule clangTidyObjCModule + clangTidyOpenMPModule clangTidyPerformanceModule + clangTidyPortabilityModule clangTidyReadabilityModule + clangTidyZirconModule + clangTooling clangToolingCore + clangToolingInclusions clangToolingInclusionsStdlib clangToolingSyntax + ) - # add to include directories - target_include_directories(llvm-libs INTERFACE "${LLVM_INSTALL_PATH}/include") + target_include_directories(llvm-libs SYSTEM INTERFACE + "${LLVM_INSTALL_PATH}/include") - if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT WIN32) - target_link_directories(llvm-libs INTERFACE "${LLVM_INSTALL_PATH}/lib") - target_link_libraries(llvm-libs INTERFACE - LLVMSupport - LLVMFrontendOpenMP - LLVMOption - LLVMTargetParser - clangAST - clangASTMatchers - clangBasic - clangDriver - clangFormat - clangFrontend - clangLex - clangSema - clangSerialization - clangTidy - clangTidyUtils - clangTidyAndroidModule - clangTidyAbseilModule - clangTidyAlteraModule - clangTidyBoostModule - clangTidyBugproneModule - clangTidyCERTModule - clangTidyConcurrencyModule - clangTidyCppCoreGuidelinesModule - clangTidyDarwinModule - clangTidyFuchsiaModule - clangTidyGoogleModule - clangTidyHICPPModule - clangTidyLinuxKernelModule - clangTidyLLVMModule - clangTidyLLVMLibcModule - clangTidyMiscModule - clangTidyModernizeModule - clangTidyObjCModule - clangTidyOpenMPModule - clangTidyPerformanceModule - clangTidyPortabilityModule - clangTidyReadabilityModule - clangTidyZirconModule - clangTooling - clangToolingCore - clangToolingInclusions - clangToolingInclusionsStdlib - clangToolingSyntax - ) - else() - file(GLOB LLVM_LIBRARIES CONFIGURE_DEPENDS "${LLVM_INSTALL_PATH}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}LLVM[a-zA-Z]*${CMAKE_STATIC_LIBRARY_SUFFIX}") - file(GLOB CLANG_LIBRARIES CONFIGURE_DEPENDS "${LLVM_INSTALL_PATH}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clang[a-zA-Z]*${CMAKE_STATIC_LIBRARY_SUFFIX}") - # TODO: find a better way to find out whether zlib and zstd are needed - # Currently link if present in the LLVM lib directory - file(GLOB OTHER_REQUIRED_LIBS CONFIGURE_DEPENDS - "${LLVM_INSTALL_PATH}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}z${CMAKE_STATIC_LIBRARY_SUFFIX}" - "${LLVM_INSTALL_PATH}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}zstd${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - target_link_libraries(llvm-libs INTERFACE ${LLVM_LIBRARIES} ${CLANG_LIBRARIES} ${OTHER_REQUIRED_LIBS}) + if(NOT BUILD_SHARED_LIBS) target_compile_definitions(llvm-libs INTERFACE CLANG_BUILD_STATIC=1) endif() + + message(STATUS "LLVM ${LLVM_VERSION} at ${LLVM_INSTALL_PATH}") endfunction() diff --git a/cmake/package.cmake b/cmake/package.cmake index e125adb10..a797b7919 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -1,7 +1,7 @@ include_guard() include(${CMAKE_CURRENT_LIST_DIR}/llvm.cmake) -setup_llvm("21.1.8") +setup_llvm("21.1.8+r1") # install dependencies include(FetchContent) diff --git a/config/llvm-manifest.json b/config/llvm-manifest.json deleted file mode 100644 index fd9eefeb9..000000000 --- a/config/llvm-manifest.json +++ /dev/null @@ -1,142 +0,0 @@ -[ - { - "version": "21.1.8", - "filename": "aarch64-linux-gnu-releasedbg-lto.tar.xz", - "sha256": "f3444ee840b50933c23656cbee7c4d010e752ac55ca66095b97f7c0e997b13b5", - "lto": true, - "asan": false, - "platform": "linux", - "arch": "arm64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "aarch64-linux-gnu-releasedbg.tar.xz", - "sha256": "b9012bf059e4d8673fb564b5780e5fc78c6a2e47f5cc6a39f444d1879b42dd2a", - "lto": false, - "asan": false, - "platform": "linux", - "arch": "arm64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "aarch64-windows-msvc-releasedbg-lto.tar.xz", - "sha256": "8870d16141ba7f9ea12f5147b8d91329abbbaa4376cd4576667dd323d896dd08", - "lto": true, - "asan": false, - "platform": "windows", - "arch": "arm64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "aarch64-windows-msvc-releasedbg.tar.xz", - "sha256": "ad394e79ec85dd40f942671bb0342ffe54a103eb2baabacb773999d57d80134b", - "lto": false, - "asan": false, - "platform": "windows", - "arch": "arm64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "arm64-macos-clang-debug-asan.tar.xz", - "sha256": "b02d20e4f7294ee33f49a09dfdd765b3b44135e003ef50e3a760aeee39e3f993", - "lto": false, - "asan": true, - "platform": "macosx", - "arch": "arm64", - "build_type": "Debug" - }, - { - "version": "21.1.8", - "filename": "arm64-macos-clang-releasedbg-lto.tar.xz", - "sha256": "e40c21eb0d0b91d9d4ab31212a5cb01ea46707f5c29839414567857e4147604d", - "lto": true, - "asan": false, - "platform": "macosx", - "arch": "arm64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "arm64-macos-clang-releasedbg.tar.xz", - "sha256": "e1b01de34f0edfd41c118e4981a93afb35556ae369597e864f4a393db623b926", - "lto": false, - "asan": false, - "platform": "macosx", - "arch": "arm64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "x64-linux-gnu-debug-asan.tar.xz", - "sha256": "76bb82d822b5377fb5e0fac8abcfba125142e6a0acc02bb36d1fa1532a268646", - "lto": false, - "asan": true, - "platform": "linux", - "arch": "x64", - "build_type": "Debug" - }, - { - "version": "21.1.8", - "filename": "x64-linux-gnu-releasedbg-lto.tar.xz", - "sha256": "32f5edddec1e689124f045b586fb402ae30febc05203af7391b088bc8494cd53", - "lto": true, - "asan": false, - "platform": "linux", - "arch": "x64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "x64-linux-gnu-releasedbg.tar.xz", - "sha256": "8ba3c84f23a2a81a86c54780754a61adf99048aa2ac0dc9b9708d0f842d553de", - "lto": false, - "asan": false, - "platform": "linux", - "arch": "x64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "x64-macos-clang-releasedbg-lto.tar.xz", - "sha256": "97e81d6296896d7237f118f728d05291707b9e4e5791e07ce4be8aee0517505d", - "lto": true, - "asan": false, - "platform": "macosx", - "arch": "x64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "x64-macos-clang-releasedbg.tar.xz", - "sha256": "53c13f8e1082fa2fe2f9c05303de48cb3133bf5f24271f4b3062f1dec578159c", - "lto": false, - "asan": false, - "platform": "macosx", - "arch": "x64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "x64-windows-msvc-releasedbg-lto.tar.xz", - "sha256": "16bcf0e4cbc3d2b1204edd619a3837004dacea28eeff0a101c8d0212f936427d", - "lto": true, - "asan": false, - "platform": "windows", - "arch": "x64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "x64-windows-msvc-releasedbg.tar.xz", - "sha256": "81d31fad05e200726c8178314b0b2045c947483dddd8cb974f4c376ae5f441fa", - "lto": false, - "asan": false, - "platform": "windows", - "arch": "x64", - "build_type": "RelWithDebInfo" - } -] diff --git a/docs/en/changelog/feature-changelog.md b/docs/en/changelog/feature-changelog.md new file mode 100644 index 000000000..53e23bf46 --- /dev/null +++ b/docs/en/changelog/feature-changelog.md @@ -0,0 +1,20 @@ +# Feature Changelog + +This file tracks user-facing feature changes across clice releases. Each release section documents new features, breaking changes, and deprecations. + + + diff --git a/docs/en/changelog/llvm-changelog.md b/docs/en/changelog/llvm-changelog.md new file mode 100644 index 000000000..efcfd93c3 --- /dev/null +++ b/docs/en/changelog/llvm-changelog.md @@ -0,0 +1,3 @@ +# LLVM Changelog + +Breaking changes encountered during each LLVM upgrade, with upstream commit references. diff --git a/pixi.toml b/pixi.toml index da0203ef8..999c8ffc3 100644 --- a/pixi.toml +++ b/pixi.toml @@ -241,25 +241,6 @@ depends-on = [ { task = "vscode-e2e", args = ["{{ type }}"] }, ] -# ============================================================================== # -# HELPER TASKS # -# ============================================================================== # -[tasks.build-llvm] -cmd = ["python3", "scripts/build-llvm.py"] - -[tasks.upload-llvm] -args = ["workflow_id", "llvm_version", { "arg" = "branch", default = "main" }] -cmd = """ -gh workflow run upload-llvm.yml \ - --ref {{ branch }} \ - --field workflow_id={{workflow_id}} \ - --field version={{llvm_version}} -""" - -[tasks.delete-artifacts] -args = ["file_name"] -cmd = ["scripts/delete-artifacts.bash", "{{ file_name }}"] - [dependencies] # ============================================================================== # diff --git a/scripts/build-llvm.py b/scripts/build-llvm.py index f769d9068..180c1e871 100644 --- a/scripts/build-llvm.py +++ b/scripts/build-llvm.py @@ -4,23 +4,15 @@ import shutil import argparse import os -import json from pathlib import Path -def normalize_mode(value: str) -> str: - mapping = { - "debug": "Debug", - "release": "Release", - "relwithdebinfo": "RelWithDebInfo", - "releasedbg": "RelWithDebInfo", - } - key = value.strip().lower() - if key in mapping: - return mapping[key] - raise argparse.ArgumentTypeError( - f"Invalid mode '{value}'. Choose from Debug, Release, RelWithDebInfo." - ) +MODE_MAP = { + "debug": "Debug", + "release": "Release", + "relwithdebinfo": "RelWithDebInfo", + "releasedbg": "RelWithDebInfo", +} def build_native_tools(project_root: Path, build_dir: Path) -> Path: @@ -94,8 +86,6 @@ def main(): parser.add_argument( "--mode", default="Release", - type=normalize_mode, - choices=["Debug", "Release", "RelWithDebInfo"], help="Build mode (default: Release)", ) parser.add_argument( @@ -116,6 +106,13 @@ def main(): args = parser.parse_args() + mode_key = args.mode.strip().lower() + if mode_key not in MODE_MAP: + parser.error( + f"Invalid mode '{args.mode}'. Choose from Debug, Release, RelWithDebInfo." + ) + args.mode = MODE_MAP[mode_key] + repo_root = Path(__file__).resolve().parent.parent toolchain_file = repo_root / "cmake" / "toolchain.cmake" if not toolchain_file.exists(): @@ -147,19 +144,24 @@ def main(): build_dir = project_root / build_dir install_prefix = build_dir.parent / f"{build_dir.name}-install" - print("--- Configuration ---") - print(f"Mode: {args.mode}") - print(f"LTO: {args.lto}") - print(f"Target Triple: {args.target_triple or '(native)'}") - print(f"Root: {project_root}") - print(f"Build Dir: {build_dir}") - print(f"Install Prefix: {install_prefix}") - print(f"Toolchain: {toolchain_file}") - print("---------------------") - - components_path = Path(__file__).resolve().parent / "llvm-components.json" - with components_path.open() as f: - llvm_distribution_components = json.load(f)["components"] + print(f"mode={args.mode}") + print(f"lto={args.lto}") + print(f"target_triple={args.target_triple or '(native)'}") + print(f"root={project_root}") + print(f"build_dir={build_dir}") + print(f"install_prefix={install_prefix}") + print(f"toolchain={toolchain_file}") + + llvm_distribution_components = [ + "llvm-libraries", + "clang-libraries", + "llvm-headers", + "clang-headers", + "clang-tidy-headers", + "clang-resource-headers", + "cmake-exports", + "clang-cmake-exports", + ] components_joined = ";".join(llvm_distribution_components) cmake_args = [ @@ -321,13 +323,6 @@ def main(): else: print(f" Warning: {header} not found in source.") - def human_readable(num: int) -> str: - for unit in ["B", "KB", "MB", "GB"]: - if num < 1024.0: - return f"{num:,.1f}{unit}" - num /= 1024.0 - return f"{num:.1f}TB" - lib_dir = install_prefix / "lib" sizes = [] if lib_dir.exists(): @@ -338,10 +333,10 @@ def human_readable(num: int) -> str: total_size = sum(sz for _, sz in sizes) print(f"\nLibrary size summary under {lib_dir}:") - print(f" Total: {human_readable(total_size)} across {len(sizes)} files") + print(f" Total: {total_size / 1048576:.1f} MB across {len(sizes)} files") for path, sz in sizes: rel = path.relative_to(install_prefix) - print(f" {human_readable(sz):>8} {rel}") + print(f" {sz / 1048576:>8.1f} MB {rel}") if not sizes: print(" (no files found)") diff --git a/scripts/delete-artifacts.bash b/scripts/delete-artifacts.bash deleted file mode 100755 index d34a9b5b5..000000000 --- a/scripts/delete-artifacts.bash +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Configuration: workflow file name can be provided as the first argument, defaults to llvm.yml -WORKFLOW_FILE_NAME="${1:-llvm.yml}" -RUN_LIMIT=1000 - -echo "Searching for all runs of workflow ${WORKFLOW_FILE_NAME}..." - -# Retrieve all workflow run IDs -RUN_IDS=$(gh run list --workflow "${WORKFLOW_FILE_NAME}" --json databaseId --jq '.[].databaseId' --limit $RUN_LIMIT) - -if [ -z "$RUN_IDS" ]; then - echo "No runs found for workflow ${WORKFLOW_FILE_NAME}." - exit 0 -fi - -echo "Found the following Run IDs:" -echo "$RUN_IDS" -echo "---" - -# Loop and delete each workflow run (and its Artifacts) -echo "Starting deletion of runs and their Artifacts..." - -for RUN_ID in $RUN_IDS; do - echo "Deleting Run ID: ${RUN_ID}..." - # Older gh releases may not support --confirm; pipe "y" to avoid interactive prompt. - if yes | gh run delete "${RUN_ID}"; then - echo "Successfully deleted Run ${RUN_ID}." - else - echo "Failed to delete Run ${RUN_ID}!" - fi - sleep 0.5 -done - -echo "" -echo "---" -echo "All workflow runs and associated Artifacts have been processed." diff --git a/scripts/download-llvm.sh b/scripts/download-llvm.sh deleted file mode 100755 index 3a640e031..000000000 --- a/scripts/download-llvm.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -if [[ $# -ne 1 ]]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -WORKFLOW_ID="$1" - -mkdir -p artifacts -gh run download "${WORKFLOW_ID}" --dir artifacts - -echo "Downloaded artifacts:" -find artifacts -maxdepth 4 -type f -print diff --git a/scripts/llvm-components.json b/scripts/llvm-components.json deleted file mode 100644 index 6578f04f6..000000000 --- a/scripts/llvm-components.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "components": [ - "LLVMDemangle", - "LLVMSupport", - "LLVMCore", - "LLVMOption", - "LLVMBinaryFormat", - "LLVMMC", - "LLVMMCParser", - "LLVMObject", - "LLVMProfileData", - "LLVMBitReader", - "LLVMBitstreamReader", - "LLVMRemarks", - "LLVMObjectYAML", - "LLVMAggressiveInstCombine", - "LLVMInstCombine", - "LLVMIRReader", - "LLVMTextAPI", - "LLVMSymbolize", - "LLVMDebugInfoDWARF", - "LLVMDebugInfoDWARFLowLevel", - "LLVMDebugInfoCodeView", - "LLVMDebugInfoGSYM", - "LLVMDebugInfoPDB", - "LLVMDebugInfoBTF", - "LLVMDebugInfoMSF", - "LLVMAsmParser", - "LLVMTargetParser", - "LLVMTransformUtils", - "LLVMAnalysis", - "LLVMScalarOpts", - "LLVMFrontendHLSL", - "LLVMFrontendOpenMP", - "LLVMFrontendOffloading", - "LLVMFrontendAtomic", - "LLVMFrontendDirective", - "LLVMWindowsDriver", - "clangIndex", - "clangAPINotes", - "clangAST", - "clangASTMatchers", - "clangBasic", - "clangDriver", - "clangFormat", - "clangFrontend", - "clangLex", - "clangParse", - "clangSema", - "clangSerialization", - "clangRewrite", - "clangAnalysis", - "clangEdit", - "clangSupport", - "clangStaticAnalyzerCore", - "clangStaticAnalyzerFrontend", - "clangTidy", - "clangTidyUtils", - "clangTidyAndroidModule", - "clangTidyAbseilModule", - "clangTidyAlteraModule", - "clangTidyBoostModule", - "clangTidyBugproneModule", - "clangTidyCERTModule", - "clangTidyConcurrencyModule", - "clangTidyCppCoreGuidelinesModule", - "clangTidyDarwinModule", - "clangTidyFuchsiaModule", - "clangTidyGoogleModule", - "clangTidyHICPPModule", - "clangTidyLinuxKernelModule", - "clangTidyLLVMModule", - "clangTidyLLVMLibcModule", - "clangTidyMiscModule", - "clangTidyModernizeModule", - "clangTidyObjCModule", - "clangTidyOpenMPModule", - "clangTidyPerformanceModule", - "clangTidyPortabilityModule", - "clangTidyReadabilityModule", - "clangTidyZirconModule", - "clangTooling", - "clangToolingCore", - "clangToolingInclusions", - "clangToolingInclusionsStdlib", - "clangToolingSyntax", - "clangToolingRefactoring", - "clangTransformer", - "clangCrossTU", - "clangAnalysisFlowSensitive", - "clangAnalysisFlowSensitiveModels", - "clangStaticAnalyzerCheckers", - "clangIncludeCleaner", - "llvm-headers", - "clang-headers", - "clang-tidy-headers", - "clang-resource-headers" - ] -} diff --git a/scripts/monitor-resources.ps1 b/scripts/monitor-resources.ps1 deleted file mode 100644 index f044c2377..000000000 --- a/scripts/monitor-resources.ps1 +++ /dev/null @@ -1,39 +0,0 @@ -param( - [string]$LogFile = "ci-resource-usage.log", - [int]$IntervalSeconds = 30 -) - -function Write-Log { - param( - [Parameter(ValueFromPipeline = $true)] - [string]$Message - ) - process { - Add-Content -Path $LogFile -Value $Message - } -} - -"Resource monitor started for Windows, interval ${IntervalSeconds}s, writing to ${LogFile}" | Write-Log - -while ($true) { - $timestamp = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") - - "[$timestamp] ===== Disk =====" | Write-Log - try { - $disk = Get-Volume | Select-Object DriveLetter, FileSystem, @{Name="FreeGB";Expression={[math]::Round($_.SizeRemaining/1GB,2)}}, @{Name="SizeGB";Expression={[math]::Round($_.Size/1GB,2)}} | Format-Table -AutoSize | Out-String - $disk | Write-Log - } catch { - "Disk query failed: $($_.Exception.Message)" | Write-Log - } - - "[$timestamp] ===== Memory =====" | Write-Log - try { - $mem = Get-CimInstance Win32_OperatingSystem | Select-Object @{Name="TotalGB";Expression={[math]::Round($_.TotalVisibleMemorySize/1MB,2)}}, @{Name="FreeGB";Expression={[math]::Round($_.FreePhysicalMemory/1MB,2)}} | Format-Table -AutoSize | Out-String - $mem | Write-Log - } catch { - "Memory query failed: $($_.Exception.Message)" | Write-Log - } - - "" | Write-Log - Start-Sleep -Seconds $IntervalSeconds -} diff --git a/scripts/monitor-resources.sh b/scripts/monitor-resources.sh deleted file mode 100644 index 895687a62..000000000 --- a/scripts/monitor-resources.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash - -# Periodically logs disk and memory usage to a file. -# Usage: monitor-resources.sh [log-file] - -set -uo pipefail - -LOG_FILE="${1:-ci-resource-usage.log}" -INTERVAL="${CLICE_MONITOR_INTERVAL:-30}" -OS="$(uname -s)" - -echo "Resource monitor started for ${OS}, interval ${INTERVAL}s, writing to ${LOG_FILE}" >> "${LOG_FILE}" - -log_disk() { - echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] ===== Disk =====" >> "${LOG_FILE}" - df -h >> "${LOG_FILE}" 2>&1 -} - -log_mem() { - echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] ===== Memory =====" >> "${LOG_FILE}" - case "${OS}" in - Linux) - if command -v free >/dev/null 2>&1; then - free -h >> "${LOG_FILE}" 2>&1 - else - cat /proc/meminfo >> "${LOG_FILE}" 2>&1 - fi - ;; - Darwin) - if command -v vm_stat >/dev/null 2>&1; then - PAGESIZE="$(sysctl -n hw.pagesize 2>/dev/null || echo 4096)" - TOTAL_MEM="$(sysctl -n hw.memsize 2>/dev/null || echo 0)" - echo "Total Memory: ${TOTAL_MEM} bytes" >> "${LOG_FILE}" - vm_stat >> "${LOG_FILE}" 2>&1 - # Derive an approximate free/used breakdown to make the log easier to read. - FREE_PAGES="$(vm_stat | awk '/(Pages free|Pages speculative)/ {gsub(/\\./,\"\",$3); total+=$3} END {print total+0}')" - if [[ -n "${FREE_PAGES}" ]]; then - FREE_BYTES=$((FREE_PAGES * PAGESIZE)) - echo "Approx Free: ${FREE_BYTES} bytes" >> "${LOG_FILE}" - fi - else - echo "vm_stat unavailable" >> "${LOG_FILE}" - fi - ;; - *) - echo "Unsupported OS for memory logging: ${OS}" >> "${LOG_FILE}" - ;; - esac -} - -while true; do - log_disk - log_mem - echo "" >> "${LOG_FILE}" - sleep "${INTERVAL}" -done diff --git a/scripts/prune-llvm-bin.py b/scripts/prune-llvm-bin.py deleted file mode 100644 index 9a392156f..000000000 --- a/scripts/prune-llvm-bin.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python3 -""" -Prune unused LLVM binaries/libraries from the install tree. - -Two modes: - - discover: iteratively delete candidates and rebuild to confirm they are unused, - then write the deletion list to a manifest JSON file. - - apply: read the manifest and delete the listed files without rebuilding - (useful for LTO builds that trust the non-LTO pruning result). -""" - -import argparse -import json -import shutil -import subprocess -import time -from datetime import datetime, timezone -from pathlib import Path -from typing import Iterable, List, Optional - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Prune unused LLVM artifacts.") - parser.add_argument( - "--action", - choices=["discover", "apply"], - default="discover", - help="discover: probe deletable files and record them; apply: delete using manifest", - ) - parser.add_argument( - "--install-dir", - type=Path, - default=Path(".llvm/build-install/lib"), - help="Path to the LLVM install lib directory (default: .llvm/build-install/lib)", - ) - parser.add_argument( - "--build-dir", - type=Path, - default=Path("build"), - help="CMake build directory used for validation builds (default: build)", - ) - parser.add_argument( - "--manifest", - type=Path, - default=Path("pruned-libs.json"), - help="Path to manifest JSON for recording or applying deletions", - ) - parser.add_argument( - "--gh-run-id", - type=str, - help="GitHub run ID to download manifest from when applying", - ) - parser.add_argument( - "--gh-artifact", - type=str, - default="llvm-pruned-libs", - help="Artifact name/pattern that contains the manifest", - ) - parser.add_argument( - "--gh-download-dir", - type=Path, - default=Path("artifacts"), - help="Directory to place downloaded artifacts", - ) - parser.add_argument( - "--max-attempts", - type=int, - default=30, - help="Maximum attempts when waiting for manifest download", - ) - parser.add_argument( - "--sleep-seconds", - type=int, - default=60, - help="Seconds to sleep between manifest download attempts", - ) - return parser.parse_args() - - -def _print_failure_output( - result: subprocess.CalledProcessError, max_lines: int = 50 -) -> None: - stdout = getattr(result, "stdout", "") or "" - stderr = getattr(result, "stderr", "") or "" - combined = stdout + stderr - if not combined: - return - print("Build output (last lines):") - lines = combined.splitlines() - for line in lines[-max_lines:]: - print(line) - - -def run_build(build_dir: Path) -> bool: - cmd = ["cmake", "--build", str(build_dir)] - print(f"Running: {' '.join(cmd)}") - try: - subprocess.run(cmd, check=True, capture_output=True, text=True) - return True - except subprocess.CalledProcessError as exc: - _print_failure_output(exc) - return False - - -def candidate_files(install_dir: Path) -> Iterable[Path]: - if not install_dir.is_dir(): - raise FileNotFoundError(f"lib dir not found: {install_dir}") - for path in sorted(install_dir.iterdir()): - if not path.is_file(): - continue - if path.suffix.lower() in {".a", ".lib"}: - yield path - else: - print(f"Skipping non-static file: {path.name}") - - -def try_delete(path: Path, build_dir: Path) -> bool: - backup = path.with_suffix(path.suffix + ".bak") - print(f"Testing deletion: {path}") - shutil.move(path, backup) - success = run_build(build_dir) - if success: - backup.unlink(missing_ok=True) - print(f"Safe to delete: {path.name}") - return True - shutil.move(backup, path) - print(f"Required; restored: {path.name}") - return False - - -def discover(install_dir: Path, build_dir: Path) -> List[str]: - deletable: List[str] = [] - for path in candidate_files(install_dir): - if try_delete(path, build_dir): - deletable.append(path.name) - return deletable - - -def write_manifest( - manifest: Path, removed: List[str], install_dir: Path, build_dir: Path -) -> None: - data = { - "generated_at": datetime.now(timezone.utc).isoformat(), - "install_dir": str(install_dir), - "build_dir": str(build_dir), - "removed": removed, - } - manifest.write_text(json.dumps(data, indent=2)) - print(f"Wrote manifest with {len(removed)} entries to {manifest}") - - -def apply_manifest(manifest: Path, install_dir: Path) -> None: - if not manifest.is_file(): - raise FileNotFoundError(f"Manifest not found: {manifest}") - data = json.loads(manifest.read_text()) - removed = data.get("removed", []) - if not isinstance(removed, list): - raise ValueError("Manifest missing 'removed' list") - for name in removed: - target = install_dir / name - if target.exists(): - print(f"Deleting {target}") - target.unlink() - else: - print(f"Already absent: {target}") - - -def find_manifest(download_dir: Path, manifest_name: str) -> Optional[Path]: - for path in download_dir.rglob(manifest_name): - if path.is_file(): - return path - return None - - -def wait_and_download_manifest( - run_id: str, - artifact: str, - download_dir: Path, - manifest_name: str, - max_attempts: int, - sleep_seconds: int, -) -> Path: - download_dir.mkdir(parents=True, exist_ok=True) - for attempt in range(1, max_attempts + 1): - print( - f"[{attempt}/{max_attempts}] Downloading manifest via gh run download " - f"(run={run_id}, artifact={artifact})" - ) - cmd = [ - "gh", - "run", - "download", - str(run_id), - "--pattern", - artifact, - "--dir", - str(download_dir), - ] - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode == 0: - found = find_manifest(download_dir, manifest_name) - if found: - print(f"Found manifest at {found}") - return found - print("Download succeeded but manifest not found; retrying...") - else: - print("gh run download failed; stderr follows:") - print(result.stderr) - if attempt < max_attempts: - time.sleep(sleep_seconds) - raise RuntimeError("Manifest could not be downloaded within the allotted attempts") - - -def ensure_manifest( - manifest: Path, - run_id: Optional[str], - artifact: str, - download_dir: Path, - max_attempts: int, - sleep_seconds: int, -) -> Path: - if manifest.exists(): - return manifest - if not run_id: - raise FileNotFoundError( - f"Manifest {manifest} missing and no gh run ID provided for download" - ) - downloaded = wait_and_download_manifest( - run_id=run_id, - artifact=artifact, - download_dir=download_dir, - manifest_name=manifest.name, - max_attempts=max_attempts, - sleep_seconds=sleep_seconds, - ) - if downloaded != manifest: - shutil.copy(downloaded, manifest) - return manifest - - -def main() -> None: - args = parse_args() - install_dir = args.install_dir - build_dir = args.build_dir - if args.action == "discover": - deletable = discover(install_dir, build_dir) - write_manifest(args.manifest, deletable, install_dir, build_dir) - else: - manifest = ensure_manifest( - manifest=args.manifest, - run_id=args.gh_run_id, - artifact=args.gh_artifact, - download_dir=args.gh_download_dir, - max_attempts=args.max_attempts, - sleep_seconds=args.sleep_seconds, - ) - apply_manifest(manifest, install_dir) - - -if __name__ == "__main__": - main() diff --git a/scripts/release-llvm.py b/scripts/release-llvm.py new file mode 100644 index 000000000..052dd81c3 --- /dev/null +++ b/scripts/release-llvm.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +""" +LLVM release pipeline utilities. + +Actions: + discover: Iteratively probe which static libs can be removed by deleting + and rebuilding, then write the result to a manifest JSON. + apply: Read a manifest and replace listed libs with empty archives. + repackage: Download all LLVM build artifacts, apply pruning, and repackage + them in parallel. +""" + +import argparse +import concurrent.futures +import fnmatch +import json +import shutil +import subprocess +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterable, List, Optional + +ARTIFACTS = [ + "arm64-linux-gnu-releasedbg-lto.tar.xz", + "arm64-linux-gnu-releasedbg.tar.xz", + "arm64-windows-msvc-releasedbg-lto.tar.xz", + "arm64-windows-msvc-releasedbg.tar.xz", + "arm64-macos-clang-debug-asan.tar.xz", + "arm64-macos-clang-releasedbg-lto.tar.xz", + "arm64-macos-clang-releasedbg.tar.xz", + "x64-linux-gnu-debug-asan.tar.xz", + "x64-linux-gnu-releasedbg-lto.tar.xz", + "x64-linux-gnu-releasedbg.tar.xz", + "x64-macos-clang-releasedbg-lto.tar.xz", + "x64-macos-clang-releasedbg.tar.xz", + "x64-windows-msvc-releasedbg-lto.tar.xz", + "x64-windows-msvc-releasedbg.tar.xz", +] + +ARCHIVE_MAGIC = b"!\n" + + +def _is_shared_lib(path: Path) -> bool: + return ".so" in path.suffixes or ".dylib" in path.suffixes + + +def _replace_with_empty_archive(path: Path) -> None: + if _is_shared_lib(path): + path.write_bytes(b"") + else: + path.write_bytes(ARCHIVE_MAGIC) + + +def _remove_binaries(build_dir: Path) -> None: + bin_dir = build_dir / "bin" + if not bin_dir.is_dir(): + return + for f in bin_dir.iterdir(): + if f.is_file() and f.suffix in {"", ".exe"}: + f.unlink() + + +def _run_build(build_dir: Path) -> bool: + try: + subprocess.run( + ["cmake", "--build", str(build_dir)], + check=True, + capture_output=True, + text=True, + ) + return True + except subprocess.CalledProcessError as exc: + combined = (exc.stdout or "") + (exc.stderr or "") + if combined: + print("Build output (last lines):") + for line in combined.splitlines()[-50:]: + print(line) + return False + + +def _manifest_for(artifact: str, manifests_dir: Path) -> Optional[Path]: + manifest_dirs = { + "linux": "prune-manifest-ubuntu-24.04", + "macos": "prune-manifest-macos-15", + "windows": "prune-manifest-windows-2025", + } + for platform, dirname in manifest_dirs.items(): + if platform in artifact: + return manifests_dir / dirname / "pruned-libs.json" + return None + + +def _candidate_files( + install_dir: Path, skip_patterns: Optional[List[str]] = None +) -> Iterable[Path]: + if not install_dir.is_dir(): + raise FileNotFoundError(f"lib dir not found: {install_dir}") + for path in sorted(install_dir.iterdir()): + if not path.is_file(): + continue + if path.suffix.lower() not in {".a", ".lib"}: + print(f"Skipping non-static file: {path.name}") + continue + if skip_patterns and any(fnmatch.fnmatch(path.name, p) for p in skip_patterns): + print(f"Skipping (never-prune): {path.name}") + continue + yield path + + +def _nullify_shared_libs(install_dir: Path) -> List[dict]: + nullified: List[dict] = [] + for path in sorted(install_dir.iterdir()): + if not path.is_file() or not _is_shared_lib(path): + continue + size = path.stat().st_size + if size > 0: + print(f"Nullifying shared lib: {path.name} ({size} bytes)") + path.write_bytes(b"") + nullified.append({"name": path.name, "size": size}) + return nullified + + +def _try_delete(path: Path, build_dir: Path) -> Optional[int]: + size = path.stat().st_size + backup = path.with_suffix(path.suffix + ".bak") + print(f"Testing deletion: {path}") + shutil.move(path, backup) + _remove_binaries(build_dir) + if _run_build(build_dir): + backup.unlink(missing_ok=True) + print(f"Safe to delete: {path.name} ({size} bytes)") + return size + shutil.move(backup, path) + print(f"Required; restored: {path.name}") + return None + + +def discover( + install_dir: Path, + build_dir: Path, + skip_patterns: Optional[List[str]] = None, +) -> List[dict]: + nullified = _nullify_shared_libs(install_dir) + deletable: List[dict] = [] + for path in _candidate_files(install_dir, skip_patterns): + size = _try_delete(path, build_dir) + if size is not None: + deletable.append({"name": path.name, "size": size}) + return nullified + deletable + + +def apply_manifest(manifest: Path, install_dir: Path) -> None: + if not manifest.is_file(): + raise FileNotFoundError(f"Manifest not found: {manifest}") + data = json.loads(manifest.read_text()) + removed = data.get("removed", []) + if not isinstance(removed, list): + raise ValueError("Manifest missing 'removed' list") + for entry in removed: + name = entry["name"] if isinstance(entry, dict) else entry + target = install_dir / name + if target.exists(): + _replace_with_empty_archive(target) + else: + print(f"Already absent: {target}") + + +def _compress_tar_xz( + source_dir: Path, + output_path: Path, + xz_level: str, + label: str, +) -> int: + print(f"[{label}] Compressing (xz {xz_level})...", flush=True) + with output_path.open("wb") as out: + tar = subprocess.Popen( + ["tar", "-C", str(source_dir), "-cf", "-", "."], + stdout=subprocess.PIPE, + ) + xz = subprocess.Popen( + ["xz", "-T0", xz_level, "-c"], + stdin=tar.stdout, + stdout=out, + ) + tar.stdout.close() + xz.communicate() + tar.wait() + if tar.returncode != 0 or xz.returncode != 0: + raise RuntimeError( + f"tar/xz failed (tar={tar.returncode}, xz={xz.returncode})" + ) + return output_path.stat().st_size + + +def _process_artifact( + artifact: str, + source_run_id: str, + manifests_dir: Path, + output_dir: Path, +) -> None: + workdir = Path(tempfile.mkdtemp(prefix="repackage-")) + try: + dl_dir = workdir / "dl" + content_dir = workdir / "content" + + print(f"[{artifact}] Downloading...", flush=True) + subprocess.run( + ["gh", "run", "download", source_run_id, "-n", artifact, "-D", str(dl_dir)], + check=True, + ) + + print(f"[{artifact}] Extracting...", flush=True) + content_dir.mkdir() + subprocess.run( + [ + "tar", + "-xf", + str(dl_dir / artifact), + "-C", + str(content_dir), + ], + check=True, + ) + shutil.rmtree(dl_dir) + + if "debug" in artifact or "asan" in artifact: + print(f"[{artifact}] Debug/ASAN — skipping prune", flush=True) + else: + manifest = _manifest_for(artifact, manifests_dir) + if not manifest: + raise RuntimeError(f"No manifest mapping for artifact: {artifact}") + if not manifest.is_file(): + raise FileNotFoundError(f"Prune manifest missing: {manifest}") + print(f"[{artifact}] Pruning...", flush=True) + apply_manifest(manifest, content_dir / "lib") + + output_path = output_dir / artifact + file_size = _compress_tar_xz(content_dir, output_path, "-9e", artifact) + + print(f"[{artifact}] Done ({file_size / 1048576:.1f} MB)", flush=True) + finally: + shutil.rmtree(workdir, ignore_errors=True) + + +def repackage( + source_run_id: str, + manifests_dir: Path, + output_dir: Path, + max_parallel: int = 3, + artifacts: Optional[List[str]] = None, +) -> None: + targets = artifacts if artifacts else ARTIFACTS + output_dir.mkdir(parents=True, exist_ok=True) + + failed: List[str] = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=max_parallel) as pool: + futures = { + pool.submit( + _process_artifact, a, source_run_id, manifests_dir, output_dir + ): a + for a in targets + } + for future in concurrent.futures.as_completed(futures): + artifact = futures[future] + try: + future.result() + except Exception as exc: + print(f"[{artifact}] FAILED: {exc}", flush=True) + failed.append(artifact) + + if failed: + print(f"\nFailed artifacts ({len(failed)}):") + for name in failed: + print(f" {name}") + sys.exit(1) + + print(f"\nAll {len(targets)} artifacts repackaged:") + for path in sorted(output_dir.iterdir()): + if path.is_file() and path.suffix != ".json": + print(f" {path.stat().st_size / 1048576:>8.1f} MB {path.name}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="LLVM release pipeline utilities.") + sub = parser.add_subparsers(dest="action", required=True) + + dp = sub.add_parser("discover", help="Probe which static libs can be removed") + dp.add_argument("--install-dir", type=Path, required=True) + dp.add_argument("--build-dir", type=Path, required=True) + dp.add_argument("--manifest", type=Path, required=True) + dp.add_argument("--skip-pattern", action="append", default=[]) + + rp = sub.add_parser("repackage", help="Download, prune, and repackage artifacts") + rp.add_argument("--source-run-id", type=str, required=True) + rp.add_argument("--manifests-dir", type=Path, required=True) + rp.add_argument("--output-dir", type=Path, required=True) + rp.add_argument("--max-parallel", type=int, default=3) + rp.add_argument("--artifacts", nargs="*") + + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + if args.action == "discover": + removed = discover(args.install_dir, args.build_dir, args.skip_pattern) + total_size = sum(e["size"] for e in removed) + data = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "install_dir": str(args.install_dir), + "build_dir": str(args.build_dir), + "total_saved_bytes": total_size, + "removed": removed, + } + args.manifest.write_text(json.dumps(data, indent=2)) + print(f"Wrote manifest with {len(removed)} entries to {args.manifest}") + print(f"Total space saved: {total_size / 1048576:.1f} MB") + elif args.action == "repackage": + repackage( + source_run_id=args.source_run_id, + manifests_dir=args.manifests_dir, + output_dir=args.output_dir, + max_parallel=args.max_parallel, + artifacts=args.artifacts, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/setup-llvm.py b/scripts/setup-llvm.py deleted file mode 100644 index 8c530c422..000000000 --- a/scripts/setup-llvm.py +++ /dev/null @@ -1,405 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import hashlib -import json -import os -import shutil -import subprocess -import sys -import tarfile -from pathlib import Path -from urllib.error import HTTPError, URLError -from urllib.request import Request, urlopen - - -PRIVATE_CLANG_FILES = [ - "Sema/CoroutineStmtBuilder.h", - "Sema/TypeLocBuilder.h", - "Sema/TreeTransform.h", -] - - -def log(message: str) -> None: - print(f"[setup-llvm] {message}", flush=True) - - -def read_manifest(path: Path) -> list[dict]: - with path.open("r", encoding="utf-8") as handle: - return json.load(handle) - - -def detect_platform() -> str: - plat = sys.platform - if plat.startswith("win"): - return "Windows" - if plat == "darwin": - return "macosx" - if plat.startswith("linux"): - return "Linux" - raise RuntimeError(f"Unsupported platform: {plat}") - - -def detect_arch() -> str: - import platform - - machine = platform.machine().lower() - if machine in ("x86_64", "amd64"): - return "x64" - if machine in ("aarch64", "arm64"): - return "arm64" - raise RuntimeError(f"Unsupported architecture: {machine}") - - -def pick_artifact( - manifest: list[dict], - version: str, - build_type: str, - is_lto: bool, - platform: str, - arch: str, -) -> dict: - base_version = version.split("+", 1)[0] - saw_missing_arch = False - for entry in manifest: - if entry.get("version") != version: - continue - if entry.get("platform") != platform.lower(): - continue - entry_arch = entry.get("arch") - if entry_arch is None: - saw_missing_arch = True - continue - if entry_arch != arch: - continue - if entry.get("build_type") != build_type: - continue - if bool(entry.get("lto")) != is_lto: - continue - return entry - if saw_missing_arch: - raise RuntimeError( - f"Manifest contains entries without an 'arch' field for version={base_version}, " - f"platform={platform}. The manifest format changed to require explicit " - f"architectures; regenerate it via scripts/update-llvm-version.py." - ) - raise RuntimeError( - f"No matching LLVM artifact in manifest for version={base_version}, platform={platform}, " - f"arch={arch}, build_type={build_type}, lto={is_lto}" - ) - - -def sha256sum(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def download(url: str, dest: Path, token: str | None) -> None: - log(f"Start download: {url} -> {dest}") - dest.parent.mkdir(parents=True, exist_ok=True) - headers = {"User-Agent": "clice-setup-llvm"} - if token: - headers["Authorization"] = f"Bearer {token}" - request = Request(url, headers=headers) - try: - with urlopen(request) as response, dest.open("wb") as handle: - total_bytes = response.length - if total_bytes is None: - header_len = response.getheader("Content-Length") - if header_len and header_len.isdigit(): - total_bytes = int(header_len) - downloaded = 0 - next_percent = 10 - next_unknown_mark = 10 * 1024 * 1024 # 10MB steps when size is unknown - while True: - chunk = response.read(1024 * 512) - if not chunk: - break - handle.write(chunk) - downloaded += len(chunk) - if total_bytes: - percent = int(downloaded * 100 / total_bytes) - while percent >= next_percent and next_percent <= 100: - log( - f"Download progress: {next_percent}% " - f"({downloaded / 1024 / 1024:.1f}MB/" - f"{total_bytes / 1024 / 1024:.1f}MB)" - ) - next_percent += 10 - else: - if downloaded >= next_unknown_mark: - log( - f"Downloaded {downloaded / 1024 / 1024:.1f}MB (size unknown)" - ) - next_unknown_mark += 10 * 1024 * 1024 - if total_bytes and next_percent <= 100: - log("Download progress: 100% (size verified by server)") - log(f"Finished download: {dest} ({downloaded / 1024 / 1024:.1f}MB)") - except HTTPError as err: - raise RuntimeError(f"HTTP error {err.code} while downloading {url}") from err - except URLError as err: - raise RuntimeError(f"Failed to download {url}: {err.reason}") from err - - -def ensure_download( - url: str, dest: Path, expected_sha256: str, token: str | None -) -> None: - if dest.exists(): - current = sha256sum(dest) - if current == expected_sha256: - return - dest.unlink() - download(url, dest, token) - current = sha256sum(dest) - if current != expected_sha256: - dest.unlink(missing_ok=True) - raise RuntimeError( - f"SHA256 mismatch for {dest.name}: expected {expected_sha256}, got {current}" - ) - - -def extract_archive(archive: Path, dest_dir: Path) -> None: - log(f"Extracting {archive.name} to {dest_dir}") - dest_dir.mkdir(parents=True, exist_ok=True) - name = archive.name.lower() - if name.endswith(".tar.xz") or name.endswith(".tar.gz") or name.endswith(".tar"): - with tarfile.open(archive, "r:*") as tar: - tar.extractall(path=dest_dir) - log("Extraction complete") - return - raise RuntimeError(f"Unsupported archive format: {archive}") - - -def flatten_install_dir(dest_dir: Path) -> None: - # Some archives add an extra root directory (llvm-install, build-install, etc.). - for name in ("llvm-install", "build-install"): - nested = dest_dir / name - if not nested.is_dir(): - continue - log(f"Flattening nested install directory: {nested}") - for entry in nested.iterdir(): - target = dest_dir / entry.name - if target.exists(): - raise RuntimeError( - f"Cannot flatten {nested}: target already exists: {target}" - ) - shutil.move(str(entry), str(target)) - nested.rmdir() - break - - -def parse_version_tuple(text: str) -> tuple[int, ...]: - digits = [] - current = "" - for ch in text: - if ch.isdigit(): - current += ch - else: - if current: - digits.append(int(current)) - current = "" - if ch in {".", "-"}: - continue - if current: - digits.append(int(current)) - return tuple(digits) - - -def system_llvm_ok(required_version: str, build_type: str) -> Path | None: - if build_type.lower().startswith("debug"): - return None - llvm_config = shutil.which("llvm-config") - if not llvm_config: - return None - try: - version = subprocess.check_output([llvm_config, "--version"], text=True).strip() - prefix = subprocess.check_output([llvm_config, "--prefix"], text=True).strip() - except (subprocess.CalledProcessError, OSError): - return None - required = parse_version_tuple(required_version.split("+", 1)[0]) - found = parse_version_tuple(version) - if not found or found < required: - return None - return Path(prefix) - - -def github_api(url: str, token: str | None) -> dict: - headers = { - "Accept": "application/vnd.github+json", - "User-Agent": "clice-setup-llvm", - } - if token: - headers["Authorization"] = f"Bearer {token}" - request = Request(url, headers=headers) - with urlopen(request) as response: - return json.load(response) - - -def lookup_llvm_commit(version: str, token: str | None) -> str | None: - tag_version = version.split("+", 1)[0] - tag = f"llvmorg-{tag_version}" - ref_url = f"https://api.github.com/repos/llvm/llvm-project/git/ref/tags/{tag}" - try: - ref = github_api(ref_url, token) - except Exception: - return None - obj = ref.get("object") or {} - obj_type = obj.get("type") - obj_sha = obj.get("sha") - if obj_type == "commit": - return obj_sha - if obj_type == "tag" and obj_sha: - tag_url = f"https://api.github.com/repos/llvm/llvm-project/git/tags/{obj_sha}" - try: - tag_info = github_api(tag_url, token) - except Exception: - return None - return tag_info.get("object", {}).get("sha") or tag_info.get("sha") - return None - - -def ensure_private_headers( - install_path: Path, work_dir: Path, version: str, token: str | None, offline: bool -) -> None: - missing = [] - for rel in PRIVATE_CLANG_FILES: - if (install_path / "include" / "clang" / rel).exists(): - continue - if (work_dir / "include" / "clang" / rel).exists(): - continue - missing.append(rel) - if not missing or offline: - return - commit = lookup_llvm_commit(version, token) - if not commit: - return - for rel in missing: - dest = work_dir / "include" / "clang" / rel - dest.parent.mkdir(parents=True, exist_ok=True) - url = f"https://raw.githubusercontent.com/llvm/llvm-project/{commit}/clang/lib/{rel}" - log(f"Fetching private header: {url}") - download(url, dest, token) - - -def main() -> None: - parser = argparse.ArgumentParser(description="Setup LLVM dependencies for CMake") - parser.add_argument("--version", required=True) - parser.add_argument("--build-type", required=True) - parser.add_argument("--binary-dir", required=True) - parser.add_argument("--manifest", required=True) - parser.add_argument("--install-path") - parser.add_argument("--enable-lto", action="store_true") - parser.add_argument("--offline", action="store_true") - parser.add_argument( - "--target-platform", - help="Override platform for cross-compilation (e.g. macosx, linux, windows)", - ) - parser.add_argument( - "--target-arch", - help="Override architecture for cross-compilation (e.g. x64, arm64)", - ) - parser.add_argument("--output", required=True) - args = parser.parse_args() - - log( - "Args: " - f"version={args.version}, build_type={args.build_type}, " - f"binary_dir={args.binary_dir}, install_path={args.install_path or '(auto)'}, " - f"enable_lto={args.enable_lto}, offline={args.offline}" - ) - token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") - build_type = args.build_type - platform_name = args.target_platform if args.target_platform else detect_platform() - arch_name = args.target_arch if args.target_arch else detect_arch() - log( - f"Platform: {platform_name}, arch: {arch_name}, normalized build type: {build_type}" - ) - manifest = read_manifest(Path(args.manifest)) - - binary_dir = Path(args.binary_dir).resolve() - install_root = binary_dir / ".llvm" - - install_path: Path | None = None - needs_install = False - if args.install_path: - candidate = Path(args.install_path) - if candidate.exists(): - log(f"Using provided LLVM install at {candidate}") - else: - log( - f"Provided LLVM install path does not exist; will install to {candidate}" - ) - needs_install = True - install_path = candidate - else: - detected = system_llvm_ok(args.version, build_type) - if detected: - log(f"Found suitable system LLVM at {detected}") - install_path = detected - - artifact = None - if install_path is None: - needs_install = True - artifact = pick_artifact( - manifest, - args.version, - build_type, - args.enable_lto, - platform_name, - arch_name, - ) - log(f"Selected artifact: {artifact.get('filename')} for download") - filename = artifact["filename"] - url_version = args.version.replace("+", "%2B") - url = f"https://github.com/clice-io/clice-llvm/releases/download/{url_version}/{filename}" - download_path = binary_dir / filename - ensure_download(url, download_path, artifact["sha256"], token) - extract_archive(download_path, install_root) - flatten_install_dir(install_root) - install_path = install_root - elif needs_install: - artifact = pick_artifact( - manifest, - args.version, - build_type, - args.enable_lto, - platform_name, - arch_name, - ) - log(f"Selected artifact: {artifact.get('filename')} for download") - filename = artifact["filename"] - url_version = args.version.replace("+", "%2B") - url = f"https://github.com/clice-io/clice-llvm/releases/download/{url_version}/{filename}" - download_path = binary_dir / filename - ensure_download(url, download_path, artifact["sha256"], token) - target_dir = install_path.resolve() - extract_archive(download_path, target_dir) - flatten_install_dir(target_dir) - install_path = target_dir - else: - install_path = install_path.resolve() - log(f"Using existing LLVM install at {install_path}") - - cmake_dir = install_path / "lib" / "cmake" / "llvm" - ensure_private_headers(install_path, binary_dir, args.version, token, args.offline) - - output = Path(args.output) - output.parent.mkdir(parents=True, exist_ok=True) - with output.open("w", encoding="utf-8") as handle: - json.dump( - { - "install_path": str(install_path), - "cmake_dir": str(cmake_dir), - "artifact": artifact or {}, - }, - handle, - indent=2, - ) - handle.write("\n") - - -if __name__ == "__main__": - main() diff --git a/scripts/update-llvm-version.py b/scripts/update-llvm-version.py deleted file mode 100755 index c18f84355..000000000 --- a/scripts/update-llvm-version.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import json -import re -import sys -from pathlib import Path - - -def copy_manifest(src: Path, dest: Path) -> None: - text = src.read_text(encoding="utf-8") - - try: - data = json.loads(text) - except json.JSONDecodeError as err: - print(f"Error: {src} is not valid JSON: {err}", file=sys.stderr) - sys.exit(1) - - if not isinstance(data, list) or len(data) == 0: - print(f"Error: {src} must be a non-empty JSON array", file=sys.stderr) - sys.exit(1) - - dest.parent.mkdir(parents=True, exist_ok=True) - with dest.open("w", encoding="utf-8") as handle: - json.dump(data, handle, indent=2) - handle.write("\n") - - print(f"Copied manifest: {src} -> {dest} ({len(data)} entries)") - - -def update_package_cmake(path: Path, version: str) -> None: - text = path.read_text(encoding="utf-8") - - pattern = r'setup_llvm\("[^"]*"\)' - matches = re.findall(pattern, text) - - if len(matches) == 0: - print(f"Error: no setup_llvm(...) call found in {path}", file=sys.stderr) - sys.exit(1) - - if len(matches) > 1: - print( - f"Error: expected exactly 1 setup_llvm(...) call in {path}, " - f"found {len(matches)}", - file=sys.stderr, - ) - sys.exit(1) - - old_call = matches[0] - new_call = f'setup_llvm("{version}")' - - if old_call == new_call: - print(f"Version in {path} is already {version}, no change needed") - return - - updated = text.replace(old_call, new_call) - path.write_text(updated, encoding="utf-8") - print(f"Updated {path}: {old_call} -> {new_call}") - - -def check_package_cmake(path: Path) -> None: - """Verify package.cmake has exactly one setup_llvm(...) call that the - update script can rewrite. Used by CI to catch drift before the next bump.""" - text = path.read_text(encoding="utf-8") - matches = re.findall(r'setup_llvm\("[^"]*"\)', text) - if len(matches) == 0: - print(f"Error: no setup_llvm(...) call found in {path}", file=sys.stderr) - sys.exit(1) - if len(matches) > 1: - print( - f"Error: expected exactly 1 setup_llvm(...) call in {path}, " - f"found {len(matches)}: {matches}", - file=sys.stderr, - ) - sys.exit(1) - print(f"OK: {path} has a single setup_llvm(...) call: {matches[0]}") - - -def check_manifest(path: Path) -> None: - """Verify the manifest is a well-formed non-empty array with required fields.""" - try: - data = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as err: - print(f"Error: {path} is not valid JSON: {err}", file=sys.stderr) - sys.exit(1) - if not isinstance(data, list) or len(data) == 0: - print(f"Error: {path} must be a non-empty JSON array", file=sys.stderr) - sys.exit(1) - required = ("version", "platform", "arch", "build_type", "filename", "sha256") - for idx, entry in enumerate(data): - missing = [k for k in required if k not in entry] - if missing: - print( - f"Error: {path} entry {idx} is missing fields: {missing}", - file=sys.stderr, - ) - sys.exit(1) - print(f"OK: {path} has {len(data)} well-formed entries") - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Update LLVM version references in the clice project." - ) - parser.add_argument( - "--check", - action="store_true", - help="Validate existing state without modifying files (for CI drift checks)", - ) - parser.add_argument( - "--version", - help="New LLVM version string (e.g. 21.2.0); required unless --check", - ) - parser.add_argument( - "--manifest-src", - help="Path to the source llvm-manifest.json; required unless --check", - ) - parser.add_argument( - "--manifest-dest", - required=True, - help="Path to destination manifest (e.g. config/llvm-manifest.json)", - ) - parser.add_argument( - "--package-cmake", - required=True, - help="Path to cmake/package.cmake", - ) - args = parser.parse_args() - - manifest_dest = Path(args.manifest_dest) - package_cmake = Path(args.package_cmake) - - if not package_cmake.is_file(): - print(f"Error: package.cmake not found: {package_cmake}", file=sys.stderr) - sys.exit(1) - - if args.check: - check_package_cmake(package_cmake) - check_manifest(manifest_dest) - print("Done (check mode).") - return - - if not args.version or not args.manifest_src: - print( - "Error: --version and --manifest-src are required unless --check is set", - file=sys.stderr, - ) - sys.exit(1) - - manifest_src = Path(args.manifest_src) - if not manifest_src.is_file(): - print(f"Error: manifest source not found: {manifest_src}", file=sys.stderr) - sys.exit(1) - - copy_manifest(manifest_src, manifest_dest) - update_package_cmake(package_cmake, args.version) - - print("Done.") - - -if __name__ == "__main__": - main() diff --git a/scripts/upload-llvm.py b/scripts/upload-llvm.py deleted file mode 100644 index 5a0db3e5d..000000000 --- a/scripts/upload-llvm.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 - -import hashlib -import json -import os -import subprocess -import sys -from pathlib import Path - - -def sha256sum(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def parse_platform(name: str) -> str: - lowered = name.lower() - if "windows" in lowered: - return "windows" - if "linux" in lowered: - return "linux" - if "macos" in lowered: - return "macosx" - raise ValueError(f"Unable to determine platform from filename: {name}") - - -def parse_arch(name: str) -> str: - lowered = name.lower() - if lowered.startswith("aarch64-") or lowered.startswith("arm64-"): - return "arm64" - if lowered.startswith("x64-") or lowered.startswith("x86_64-"): - return "x64" - raise ValueError(f"Unable to determine arch from filename: {name}") - - -def parse_build_type(name: str) -> str: - lowered = name.lower() - if "debug" in lowered: - return "Debug" - return "RelWithDebInfo" - - -def build_metadata_entry(path: Path, version: str) -> dict: - filename = path.name - return { - "version": version, - "filename": filename, - "sha256": sha256sum(path), - "lto": "-lto" in filename.lower(), - "asan": "-asan" in filename.lower(), - "platform": parse_platform(filename), - "arch": parse_arch(filename), - "build_type": parse_build_type(filename), - } - - -def main() -> None: - if len(sys.argv) != 4: - print( - "Usage: upload-llvm.py ", file=sys.stderr - ) - sys.exit(1) - - tag, target_repo, workflow_id = sys.argv[1:] - artifacts_dir = Path("artifacts") - - if not artifacts_dir.is_dir(): - print(f"Artifacts directory not found: {artifacts_dir}", file=sys.stderr) - sys.exit(1) - - artifact_files = sorted( - p - for p in artifacts_dir.rglob("*") - if p.is_file() and p.suffix.lower() != ".json" - ) - - if not artifact_files: - print("No artifacts found to upload.", file=sys.stderr) - sys.exit(1) - - version_without_prefix = tag.lstrip("vV") - metadata = [ - build_metadata_entry(path, version_without_prefix) for path in artifact_files - ] - - json_path = artifacts_dir / "llvm-manifest.json" - with json_path.open("w", encoding="utf-8") as handle: - json.dump(metadata, handle, indent=2) - handle.write("\n") - - assets = [str(path) for path in artifact_files] - assets.append(str(json_path)) - - env = os.environ.copy() - - print(f"Checking for existing release {tag} in {target_repo}...") - view_result = subprocess.run( - ["gh", "release", "view", tag, "--repo", target_repo], - env=env, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - if view_result.returncode == 0: - print(f"Deleting existing release {tag} in {target_repo}...") - subprocess.run( - ["gh", "release", "delete", tag, "--repo", target_repo, "-y"], - env=env, - check=True, - ) - - print(f"Creating release {tag} in {target_repo} with {len(assets)} assets...") - # fmt: off - args = [ - "gh", "release", "create", tag, *assets, - "--repo", target_repo, - "--title", tag, - "--notes", f"Artifacts build from workflow run https://github.com/clice-io/clice/actions/runs/{workflow_id}", - "--latest", - ] - # fmt: on - subprocess.run(args, env=env, check=True) - - -if __name__ == "__main__": - main() diff --git a/scripts/validate-llvm-components.py b/scripts/validate-llvm-components.py deleted file mode 100755 index 26c804ffe..000000000 --- a/scripts/validate-llvm-components.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python3 -""" -Validate the LLVM distribution component list against the actual LLVM source tree. - -Scans the LLVM source for CMake library targets and compares them against -a components JSON file to detect stale or misspelled entries. -""" - -import argparse -import difflib -import json -import re -import sys -from pathlib import Path - - -# CMake function calls that define library targets. -# The captured group uses [^\s)]+ to grab the target name without -# trailing parentheses or whitespace. -LLVM_LIB_PATTERNS = [ - re.compile(r"add_llvm_component_library\(\s*([^\s)]+)"), - re.compile(r"add_llvm_library\(\s*([^\s)]+)"), -] - -CLANG_LIB_PATTERNS = [ - re.compile(r"add_clang_library\(\s*([^\s)]+)"), -] - -# Header-only / custom install targets. -HEADER_PATTERNS = [ - re.compile(r"add_llvm_install_targets\(\s*([^\s)]+)"), - re.compile(r"add_custom_target\(\s*([^\s)]+)"), - re.compile(r"add_library\(\s*([^\s)]+)"), -] - -# Targets we recognise as header-only distribution components. -KNOWN_HEADER_TARGETS = { - "llvm-headers", - "clang-headers", - "clang-tidy-headers", - "clang-resource-headers", -} - - -def scan_targets(directory: Path, patterns: list[re.Pattern]) -> set[str]: - """Recursively scan *directory* for CMakeLists.txt files and extract target names.""" - targets: set[str] = set() - if not directory.is_dir(): - return targets - for cmake_file in directory.rglob("CMakeLists.txt"): - text = cmake_file.read_text(errors="replace") - for pattern in patterns: - for match in pattern.finditer(text): - targets.add(match.group(1)) - return targets - - -def scan_header_targets(llvm_src: Path) -> set[str]: - """Scan for well-known header / custom-install targets across the tree.""" - found: set[str] = set() - for cmake_file in llvm_src.rglob("CMakeLists.txt"): - text = cmake_file.read_text(errors="replace") - for pattern in HEADER_PATTERNS: - for match in pattern.finditer(text): - name = match.group(1) - if name in KNOWN_HEADER_TARGETS: - found.add(name) - return found - - -def collect_source_targets(llvm_src: Path) -> set[str]: - """Return the full set of library / header targets found in the LLVM source tree.""" - targets: set[str] = set() - targets |= scan_targets(llvm_src / "llvm" / "lib", LLVM_LIB_PATTERNS) - targets |= scan_targets(llvm_src / "clang" / "lib", CLANG_LIB_PATTERNS) - targets |= scan_targets(llvm_src / "clang-tools-extra", CLANG_LIB_PATTERNS) - targets |= scan_header_targets(llvm_src) - return targets - - -def load_components(path: Path) -> list[str]: - with path.open("r", encoding="utf-8") as handle: - data = json.load(handle) - if isinstance(data, dict): - data = data.get("components", []) - if not isinstance(data, list) or not data: - print(f"Error: no component list found in {path}", file=sys.stderr) - sys.exit(1) - return data - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Validate LLVM distribution components against the source tree." - ) - parser.add_argument( - "--llvm-src", - required=True, - help="Path to the llvm-project source root", - ) - parser.add_argument( - "--components-file", - required=True, - help="Path to llvm-components.json", - ) - args = parser.parse_args() - - llvm_src = Path(args.llvm_src).expanduser().resolve() - components_file = Path(args.components_file).expanduser().resolve() - - if not llvm_src.is_dir(): - print(f"Error: LLVM source directory not found: {llvm_src}") - sys.exit(1) - - if not (llvm_src / "llvm" / "CMakeLists.txt").exists(): - print(f"Error: {llvm_src} does not look like an llvm-project root.") - sys.exit(1) - - if not components_file.is_file(): - print(f"Error: components file not found: {components_file}") - sys.exit(1) - - components = load_components(components_file) - source_targets = collect_source_targets(llvm_src) - - print(f"Found {len(source_targets)} targets in LLVM source tree") - print(f"Components file lists {len(components)} entries") - - # Check for components that are missing from the source tree. - missing: list[tuple[str, list[str]]] = [] - for name in components: - if name not in source_targets: - suggestions = difflib.get_close_matches( - name, source_targets, n=3, cutoff=0.6 - ) - missing.append((name, suggestions)) - - if missing: - print(f"\nError: {len(missing)} component(s) not found in the source tree:\n") - for name, suggestions in missing: - print(f" - {name}") - if suggestions: - print(f" Did you mean: {', '.join(suggestions)}?") - sys.exit(1) - - # Warn about source targets not present in the component list. - component_set = set(components) - new_targets = sorted(source_targets - component_set - KNOWN_HEADER_TARGETS) - # Filter to targets that follow LLVM/Clang naming conventions to reduce noise. - noteworthy = [t for t in new_targets if t.startswith(("LLVM", "clang", "Clang"))] - if noteworthy: - print( - f"\nWarning: {len(noteworthy)} target(s) in source not listed in components:" - ) - for name in noteworthy: - print(f" + {name}") - - print("\nAll components validated successfully.") - sys.exit(0) - - -if __name__ == "__main__": - main()