From f324686f083c890770d81e523e858cc2ddcc4849 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 17:37:19 -0400 Subject: [PATCH 01/47] Temporarily remove other workflows --- .github/workflows/clp-core-build.yaml | 376 ------------------ .github/workflows/clp-docs.yaml | 57 --- .../workflows/clp-execution-image-build.yaml | 70 ---- .github/workflows/clp-lint.yaml | 42 -- .github/workflows/clp-pr-title-checks.yaml | 31 -- .../clp-s-generated-code-checks.yaml | 42 -- 6 files changed, 618 deletions(-) delete mode 100644 .github/workflows/clp-core-build.yaml delete mode 100644 .github/workflows/clp-docs.yaml delete mode 100644 .github/workflows/clp-execution-image-build.yaml delete mode 100644 .github/workflows/clp-lint.yaml delete mode 100644 .github/workflows/clp-pr-title-checks.yaml delete mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml deleted file mode 100644 index ad0ea8fbf2..0000000000 --- a/.github/workflows/clp-core-build.yaml +++ /dev/null @@ -1,376 +0,0 @@ -name: "clp-core-build" - -on: - pull_request: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - push: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -env: - BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted - # publishing of container images. - cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - clp_changed: "${{steps.filter.outputs.clp}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - # Consider changes between the current commit and `main` - # NOTE: If a pull request changes one of the images, then we need to (1) build the image - # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull - # request doesn't change an image, then we don't need to rebuild the image; instead we can - # use the published image which is based on `main`. So when determining what files have - # changed, we need to consider the delta between the current commit and `main` (rather - # than the current and previous commits) in order to detect if we need to rebuild the - # image (since it would be different from the published image). - base: "main" - filters: | - centos_stream_9_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - - "components/core/tools/scripts/lib_install/centos-stream-9/**" - ubuntu_jammy_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" - clp: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/cmake/**" - - "components/core/CMakeLists.txt" - - "components/core/src/**" - - "components/core/tests/**" - - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - centos-stream-9-deps-image: - name: "centos-stream-9-deps-image" - if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - ubuntu-jammy-deps-image: - name: "ubuntu-jammy-deps-image" - if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - centos-stream-9-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "centos-stream-9-deps-image" - - "filter-relevant-changes" - strategy: - matrix: - use_shared_libs: [true, false] - name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - ubuntu-jammy-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - strategy: - matrix: - include: - - use_shared_libs: true - upload_binaries: false - - use_shared_libs: false - upload_binaries: true - env: - OS_NAME: "ubuntu-jammy" - name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - - if: "matrix.upload_binaries == true" - id: "copy_binaries" - run: |- - output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - - mkdir -p "${output_dir}" - cd "$GITHUB_WORKSPACE/components/core/build" - tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable - shell: "bash" - - - if: "matrix.upload_binaries == true" - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{steps.copy_binaries.outputs.output_dir}}" - retention-days: 1 - - ubuntu-jammy-binaries-image: - name: "ubuntu-jammy-binaries-image" - # Run if the ancestor jobs were successful/skipped and building clp was successful. - if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" - needs: "ubuntu-jammy-binaries" - runs-on: "ubuntu-latest" - env: - OS_NAME: "ubuntu-jammy" - TMP_OUTPUT_DIR: "/tmp" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - - - name: "Untar binaries" - working-directory: >- - ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} - run: |- - tar xf clp.tar - rm clp.tar - - - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" - with: - registry: "ghcr.io" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" - - - name: "Sanitize the repo's name" - id: "sanitize_repo_name" - run: |- - # Docker doesn't support repository names with uppercase characters, so we convert to - # lowercase here. - lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') - echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" - shell: "bash" - - - id: "core_image_meta" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: >- - ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} - tags: "type=raw,value=${{github.ref_name}}" - - # Only publish the image if this workflow was triggered by a push to `main`. - # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. - - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" - push: true - tags: "${{steps.core_image_meta.outputs.tags}}" - labels: "${{steps.core_image_meta.outputs.labels}}" - - ubuntu-jammy-lint: - name: "ubuntu-jammy-lint" - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. - fetch-depth: 0 - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full - # on all files. - - if: "'schedule' != github.event_name" - name: "Restore lint:check-cpp-static-full cache" - id: "cache-restore-lint-check-cpp-static-full" - uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - - # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy - # violations. - key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. - # run_command: >- - # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) - task lint:check-cpp-full - - # Cache the source file checksums and the generated files (logs) for the - # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. - # NOTE: If we don't cache the generated files, the task will re-run to generate them. - - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" - name: "Update lint:check-cpp-static-full cache" - uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml deleted file mode 100644 index 82fc4a0e90..0000000000 --- a/.github/workflows/clp-docs.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: "clp-docs" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - build: - name: "build" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - lfs: "true" - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.10" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Build docs" - shell: "bash" - run: "task docs:site" - - # Upload the built docs so we can download and deploy them from y-scope/yscope-docs - - if: >- - contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) - && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) - && 'ubuntu-latest' == matrix.os - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "docs-html" - path: "build/docs/html" - if-no-files-found: "error" - - # Retain the artifact for a week in case there was a deployment issue - retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml deleted file mode 100644 index 17789906e8..0000000000 --- a/.github/workflows/clp-execution-image-build.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: "clp-execution-image-build" - -on: - pull_request: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - push: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - base: "main" - filters: | - ubuntu_jammy_image: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" - - ubuntu-jammy-execution-image: - name: "ubuntu-jammy-execution-image" - if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "./.github/actions/clp-execution-image-build" - with: - image_registry: "ghcr.io" - image_registry_username: "${{github.actor}}" - image_registry_password: "${{secrets.GITHUB_TOKEN}}" - platform_id: "ubuntu" - platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml deleted file mode 100644 index 5f800d8c23..0000000000 --- a/.github/workflows/clp-lint.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-lint" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - lint-check: - name: "lint-check" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.11" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Run lint task" - shell: "bash" - run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml deleted file mode 100644 index 0aa1533a98..0000000000 --- a/.github/workflows/clp-pr-title-checks.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: "clp-pr-title-checks" - -on: - pull_request_target: - # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the - # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: - # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the - # pull request triggered by this event. - # - Each job has `permissions` set to only those necessary. - types: ["edited", "opened", "reopened"] - branches: ["main"] - -permissions: {} - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - conventional-commits: - name: "conventional-commits" - permissions: - # For amannn/action-semantic-pull-request - pull-requests: "read" - runs-on: "ubuntu-latest" - steps: - - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" - env: - GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml deleted file mode 100644 index e605db9005..0000000000 --- a/.github/workflows/clp-s-generated-code-checks.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-s-generated-code-checks" - -on: - pull_request: - push: - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - antlr-code-committed: - name: "antlr-code-committed" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Check if the generated parsers are the latest" - shell: "bash" - run: - | - git status --porcelain \ - components/core/src/clp_s/search/kql/generated \ - components/core/src/clp_s/search/sql/generated \ - | grep . > /dev/null \ - && exit 1 \ - || exit 0 From a51b899e3c0b0f4d253c967959610acbf4d4bd7b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 17:37:37 -0400 Subject: [PATCH 02/47] Add setting of environment variables --- .github/workflows/clp-core-build-macos.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 51eb6985d0..79ba380a0c 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -71,6 +71,18 @@ jobs: - name: "Install dependencies" run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" + - name: "Set environment to use Homebrew's llvm" + run: | + LLVM_PREFIX=$(brew --prefix llvm) + echo "$LLVM_PREFIX/bin" >> "$GITHUB_PATH" + { + echo "LLVM_PREFIX=$LLVM_PREFIX" + echo "Cc=clang" + echo "CXX=clang++" + echo "LDFLAGS=-L$LLVM_PREFIX/lib" + echo "CPPFLAGS=-I$LLVM_PREFIX/include" + } >> "$GITHUB_ENV" + - run: "./tools/scripts/deps-download/init.sh" shell: "bash" From 04cc82ac9bacde7501ce0f555d62b3d1477e7b95 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 17:52:03 -0400 Subject: [PATCH 03/47] Revert "Temporarily remove other workflows" This reverts commit f324686f083c890770d81e523e858cc2ddcc4849. --- .github/workflows/clp-core-build.yaml | 376 ++++++++++++++++++ .github/workflows/clp-docs.yaml | 57 +++ .../workflows/clp-execution-image-build.yaml | 70 ++++ .github/workflows/clp-lint.yaml | 42 ++ .github/workflows/clp-pr-title-checks.yaml | 31 ++ .../clp-s-generated-code-checks.yaml | 42 ++ 6 files changed, 618 insertions(+) create mode 100644 .github/workflows/clp-core-build.yaml create mode 100644 .github/workflows/clp-docs.yaml create mode 100644 .github/workflows/clp-execution-image-build.yaml create mode 100644 .github/workflows/clp-lint.yaml create mode 100644 .github/workflows/clp-pr-title-checks.yaml create mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml new file mode 100644 index 0000000000..ad0ea8fbf2 --- /dev/null +++ b/.github/workflows/clp-core-build.yaml @@ -0,0 +1,376 @@ +name: "clp-core-build" + +on: + pull_request: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + push: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +env: + BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" + DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted + # publishing of container images. + cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + clp_changed: "${{steps.filter.outputs.clp}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + # Consider changes between the current commit and `main` + # NOTE: If a pull request changes one of the images, then we need to (1) build the image + # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull + # request doesn't change an image, then we don't need to rebuild the image; instead we can + # use the published image which is based on `main`. So when determining what files have + # changed, we need to consider the delta between the current commit and `main` (rather + # than the current and previous commits) in order to detect if we need to rebuild the + # image (since it would be different from the published image). + base: "main" + filters: | + centos_stream_9_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" + - "components/core/tools/scripts/lib_install/centos-stream-9/**" + ubuntu_jammy_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" + - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" + clp: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/cmake/**" + - "components/core/CMakeLists.txt" + - "components/core/src/**" + - "components/core/tests/**" + - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + + centos-stream-9-deps-image: + name: "centos-stream-9-deps-image" + if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + ubuntu-jammy-deps-image: + name: "ubuntu-jammy-deps-image" + if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + centos-stream-9-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "centos-stream-9-deps-image" + - "filter-relevant-changes" + strategy: + matrix: + use_shared_libs: [true, false] + name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + ubuntu-jammy-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + strategy: + matrix: + include: + - use_shared_libs: true + upload_binaries: false + - use_shared_libs: false + upload_binaries: true + env: + OS_NAME: "ubuntu-jammy" + name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + - if: "matrix.upload_binaries == true" + id: "copy_binaries" + run: |- + output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + mkdir -p "${output_dir}" + cd "$GITHUB_WORKSPACE/components/core/build" + tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable + shell: "bash" + + - if: "matrix.upload_binaries == true" + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{steps.copy_binaries.outputs.output_dir}}" + retention-days: 1 + + ubuntu-jammy-binaries-image: + name: "ubuntu-jammy-binaries-image" + # Run if the ancestor jobs were successful/skipped and building clp was successful. + if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" + needs: "ubuntu-jammy-binaries" + runs-on: "ubuntu-latest" + env: + OS_NAME: "ubuntu-jammy" + TMP_OUTPUT_DIR: "/tmp" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + + - name: "Untar binaries" + working-directory: >- + ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} + run: |- + tar xf clp.tar + rm clp.tar + + - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Sanitize the repo's name" + id: "sanitize_repo_name" + run: |- + # Docker doesn't support repository names with uppercase characters, so we convert to + # lowercase here. + lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') + echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" + shell: "bash" + + - id: "core_image_meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} + tags: "type=raw,value=${{github.ref_name}}" + + # Only publish the image if this workflow was triggered by a push to `main`. + # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. + - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" + push: true + tags: "${{steps.core_image_meta.outputs.tags}}" + labels: "${{steps.core_image_meta.outputs.labels}}" + + ubuntu-jammy-lint: + name: "ubuntu-jammy-lint" + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. + fetch-depth: 0 + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full + # on all files. + - if: "'schedule' != github.event_name" + name: "Restore lint:check-cpp-static-full cache" + id: "cache-restore-lint-check-cpp-static-full" + uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + + # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy + # violations. + key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. + # run_command: >- + # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + task lint:check-cpp-full + + # Cache the source file checksums and the generated files (logs) for the + # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. + # NOTE: If we don't cache the generated files, the task will re-run to generate them. + - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" + name: "Update lint:check-cpp-static-full cache" + uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml new file mode 100644 index 0000000000..82fc4a0e90 --- /dev/null +++ b/.github/workflows/clp-docs.yaml @@ -0,0 +1,57 @@ +name: "clp-docs" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + build: + name: "build" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + lfs: "true" + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.10" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Build docs" + shell: "bash" + run: "task docs:site" + + # Upload the built docs so we can download and deploy them from y-scope/yscope-docs + - if: >- + contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) + && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) + && 'ubuntu-latest' == matrix.os + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "docs-html" + path: "build/docs/html" + if-no-files-found: "error" + + # Retain the artifact for a week in case there was a deployment issue + retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml new file mode 100644 index 0000000000..17789906e8 --- /dev/null +++ b/.github/workflows/clp-execution-image-build.yaml @@ -0,0 +1,70 @@ +name: "clp-execution-image-build" + +on: + pull_request: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + push: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + base: "main" + filters: | + ubuntu_jammy_image: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" + + ubuntu-jammy-execution-image: + name: "ubuntu-jammy-execution-image" + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - uses: "./.github/actions/clp-execution-image-build" + with: + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml new file mode 100644 index 0000000000..5f800d8c23 --- /dev/null +++ b/.github/workflows/clp-lint.yaml @@ -0,0 +1,42 @@ +name: "clp-lint" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + lint-check: + name: "lint-check" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.11" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Run lint task" + shell: "bash" + run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml new file mode 100644 index 0000000000..0aa1533a98 --- /dev/null +++ b/.github/workflows/clp-pr-title-checks.yaml @@ -0,0 +1,31 @@ +name: "clp-pr-title-checks" + +on: + pull_request_target: + # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the + # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: + # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the + # pull request triggered by this event. + # - Each job has `permissions` set to only those necessary. + types: ["edited", "opened", "reopened"] + branches: ["main"] + +permissions: {} + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + conventional-commits: + name: "conventional-commits" + permissions: + # For amannn/action-semantic-pull-request + pull-requests: "read" + runs-on: "ubuntu-latest" + steps: + - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" + env: + GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml new file mode 100644 index 0000000000..e605db9005 --- /dev/null +++ b/.github/workflows/clp-s-generated-code-checks.yaml @@ -0,0 +1,42 @@ +name: "clp-s-generated-code-checks" + +on: + pull_request: + push: + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + antlr-code-committed: + name: "antlr-code-committed" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Check if the generated parsers are the latest" + shell: "bash" + run: + | + git status --porcelain \ + components/core/src/clp_s/search/kql/generated \ + components/core/src/clp_s/search/sql/generated \ + | grep . > /dev/null \ + && exit 1 \ + || exit 0 From 0e2949958788a9f8a515f7f5c9a40bbef0ea1636 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 18:15:23 -0400 Subject: [PATCH 04/47] Bug fix --- .github/workflows/clp-core-build-macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 79ba380a0c..4b644b6144 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -77,7 +77,7 @@ jobs: echo "$LLVM_PREFIX/bin" >> "$GITHUB_PATH" { echo "LLVM_PREFIX=$LLVM_PREFIX" - echo "Cc=clang" + echo "CC=clang" echo "CXX=clang++" echo "LDFLAGS=-L$LLVM_PREFIX/lib" echo "CPPFLAGS=-I$LLVM_PREFIX/include" From cfad209be24e966a2883bc34eb32d2df6e80389e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 18:41:29 -0400 Subject: [PATCH 05/47] Temporarily remove other GH workflows --- .github/workflows/clp-core-build.yaml | 376 ------------------ .github/workflows/clp-docs.yaml | 57 --- .../workflows/clp-execution-image-build.yaml | 70 ---- .github/workflows/clp-lint.yaml | 42 -- .github/workflows/clp-pr-title-checks.yaml | 31 -- .../clp-s-generated-code-checks.yaml | 42 -- 6 files changed, 618 deletions(-) delete mode 100644 .github/workflows/clp-core-build.yaml delete mode 100644 .github/workflows/clp-docs.yaml delete mode 100644 .github/workflows/clp-execution-image-build.yaml delete mode 100644 .github/workflows/clp-lint.yaml delete mode 100644 .github/workflows/clp-pr-title-checks.yaml delete mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml deleted file mode 100644 index ad0ea8fbf2..0000000000 --- a/.github/workflows/clp-core-build.yaml +++ /dev/null @@ -1,376 +0,0 @@ -name: "clp-core-build" - -on: - pull_request: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - push: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -env: - BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted - # publishing of container images. - cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - clp_changed: "${{steps.filter.outputs.clp}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - # Consider changes between the current commit and `main` - # NOTE: If a pull request changes one of the images, then we need to (1) build the image - # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull - # request doesn't change an image, then we don't need to rebuild the image; instead we can - # use the published image which is based on `main`. So when determining what files have - # changed, we need to consider the delta between the current commit and `main` (rather - # than the current and previous commits) in order to detect if we need to rebuild the - # image (since it would be different from the published image). - base: "main" - filters: | - centos_stream_9_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - - "components/core/tools/scripts/lib_install/centos-stream-9/**" - ubuntu_jammy_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" - clp: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/cmake/**" - - "components/core/CMakeLists.txt" - - "components/core/src/**" - - "components/core/tests/**" - - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - centos-stream-9-deps-image: - name: "centos-stream-9-deps-image" - if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - ubuntu-jammy-deps-image: - name: "ubuntu-jammy-deps-image" - if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - centos-stream-9-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "centos-stream-9-deps-image" - - "filter-relevant-changes" - strategy: - matrix: - use_shared_libs: [true, false] - name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - ubuntu-jammy-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - strategy: - matrix: - include: - - use_shared_libs: true - upload_binaries: false - - use_shared_libs: false - upload_binaries: true - env: - OS_NAME: "ubuntu-jammy" - name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - - if: "matrix.upload_binaries == true" - id: "copy_binaries" - run: |- - output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - - mkdir -p "${output_dir}" - cd "$GITHUB_WORKSPACE/components/core/build" - tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable - shell: "bash" - - - if: "matrix.upload_binaries == true" - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{steps.copy_binaries.outputs.output_dir}}" - retention-days: 1 - - ubuntu-jammy-binaries-image: - name: "ubuntu-jammy-binaries-image" - # Run if the ancestor jobs were successful/skipped and building clp was successful. - if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" - needs: "ubuntu-jammy-binaries" - runs-on: "ubuntu-latest" - env: - OS_NAME: "ubuntu-jammy" - TMP_OUTPUT_DIR: "/tmp" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - - - name: "Untar binaries" - working-directory: >- - ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} - run: |- - tar xf clp.tar - rm clp.tar - - - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" - with: - registry: "ghcr.io" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" - - - name: "Sanitize the repo's name" - id: "sanitize_repo_name" - run: |- - # Docker doesn't support repository names with uppercase characters, so we convert to - # lowercase here. - lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') - echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" - shell: "bash" - - - id: "core_image_meta" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: >- - ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} - tags: "type=raw,value=${{github.ref_name}}" - - # Only publish the image if this workflow was triggered by a push to `main`. - # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. - - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" - push: true - tags: "${{steps.core_image_meta.outputs.tags}}" - labels: "${{steps.core_image_meta.outputs.labels}}" - - ubuntu-jammy-lint: - name: "ubuntu-jammy-lint" - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. - fetch-depth: 0 - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full - # on all files. - - if: "'schedule' != github.event_name" - name: "Restore lint:check-cpp-static-full cache" - id: "cache-restore-lint-check-cpp-static-full" - uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - - # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy - # violations. - key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. - # run_command: >- - # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) - task lint:check-cpp-full - - # Cache the source file checksums and the generated files (logs) for the - # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. - # NOTE: If we don't cache the generated files, the task will re-run to generate them. - - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" - name: "Update lint:check-cpp-static-full cache" - uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml deleted file mode 100644 index 82fc4a0e90..0000000000 --- a/.github/workflows/clp-docs.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: "clp-docs" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - build: - name: "build" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - lfs: "true" - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.10" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Build docs" - shell: "bash" - run: "task docs:site" - - # Upload the built docs so we can download and deploy them from y-scope/yscope-docs - - if: >- - contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) - && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) - && 'ubuntu-latest' == matrix.os - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "docs-html" - path: "build/docs/html" - if-no-files-found: "error" - - # Retain the artifact for a week in case there was a deployment issue - retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml deleted file mode 100644 index 17789906e8..0000000000 --- a/.github/workflows/clp-execution-image-build.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: "clp-execution-image-build" - -on: - pull_request: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - push: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - base: "main" - filters: | - ubuntu_jammy_image: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" - - ubuntu-jammy-execution-image: - name: "ubuntu-jammy-execution-image" - if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "./.github/actions/clp-execution-image-build" - with: - image_registry: "ghcr.io" - image_registry_username: "${{github.actor}}" - image_registry_password: "${{secrets.GITHUB_TOKEN}}" - platform_id: "ubuntu" - platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml deleted file mode 100644 index 5f800d8c23..0000000000 --- a/.github/workflows/clp-lint.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-lint" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - lint-check: - name: "lint-check" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.11" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Run lint task" - shell: "bash" - run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml deleted file mode 100644 index 0aa1533a98..0000000000 --- a/.github/workflows/clp-pr-title-checks.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: "clp-pr-title-checks" - -on: - pull_request_target: - # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the - # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: - # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the - # pull request triggered by this event. - # - Each job has `permissions` set to only those necessary. - types: ["edited", "opened", "reopened"] - branches: ["main"] - -permissions: {} - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - conventional-commits: - name: "conventional-commits" - permissions: - # For amannn/action-semantic-pull-request - pull-requests: "read" - runs-on: "ubuntu-latest" - steps: - - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" - env: - GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml deleted file mode 100644 index e605db9005..0000000000 --- a/.github/workflows/clp-s-generated-code-checks.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-s-generated-code-checks" - -on: - pull_request: - push: - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - antlr-code-committed: - name: "antlr-code-committed" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Check if the generated parsers are the latest" - shell: "bash" - run: - | - git status --porcelain \ - components/core/src/clp_s/search/kql/generated \ - components/core/src/clp_s/search/sql/generated \ - | grep . > /dev/null \ - && exit 1 \ - || exit 0 From 753e55588db3b687f5672543b949e713d249cc03 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 18:43:22 -0400 Subject: [PATCH 06/47] Add check on PATH variable --- taskfiles/deps/main.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 16f999df94..d0169e2680 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -74,6 +74,8 @@ tasks: - task: "utfcpp" - task: "yaml-cpp" - task: "ystdlib" + cmds: + - echo "PATH:$PATH" absl: internal: true From 15f3097324c444e9f6f98e61e3fd54c5e81ccffd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 20:12:50 -0400 Subject: [PATCH 07/47] Revert "Add check on PATH variable" This reverts commit 753e55588db3b687f5672543b949e713d249cc03. --- taskfiles/deps/main.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index d0169e2680..16f999df94 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -74,8 +74,6 @@ tasks: - task: "utfcpp" - task: "yaml-cpp" - task: "ystdlib" - cmds: - - echo "PATH:$PATH" absl: internal: true From 55516f561460a7598f9142c22a265c21a9c61e26 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 20:20:22 -0400 Subject: [PATCH 08/47] Use llvm --- .github/workflows/clp-core-build-macos.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 4b644b6144..ab8cdf10ee 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -77,8 +77,8 @@ jobs: echo "$LLVM_PREFIX/bin" >> "$GITHUB_PATH" { echo "LLVM_PREFIX=$LLVM_PREFIX" - echo "CC=clang" - echo "CXX=clang++" + echo "CC=$LLVM_PREFIX/bin/clang" + echo "CXX=$LLVM_PREFIX/bin/clang++" echo "LDFLAGS=-L$LLVM_PREFIX/lib" echo "CPPFLAGS=-I$LLVM_PREFIX/include" } >> "$GITHUB_ENV" From 2f7963d32d472f98c955e8956b10dc471513135a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 20:24:21 -0400 Subject: [PATCH 09/47] Bug fix --- .github/workflows/clp-core-build-macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index ab8cdf10ee..1a4310bf9b 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -73,7 +73,7 @@ jobs: - name: "Set environment to use Homebrew's llvm" run: | - LLVM_PREFIX=$(brew --prefix llvm) + LLVM_PREFIX=$(brew --prefix llvm@16) echo "$LLVM_PREFIX/bin" >> "$GITHUB_PATH" { echo "LLVM_PREFIX=$LLVM_PREFIX" From 067930318596089a3c39602535ee21ec26093abd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 20:52:33 -0400 Subject: [PATCH 10/47] Add llvm library --- .github/workflows/clp-core-build-macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 1a4310bf9b..c3ab824a8e 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -79,7 +79,7 @@ jobs: echo "LLVM_PREFIX=$LLVM_PREFIX" echo "CC=$LLVM_PREFIX/bin/clang" echo "CXX=$LLVM_PREFIX/bin/clang++" - echo "LDFLAGS=-L$LLVM_PREFIX/lib" + echo "LDFLAGS=-L$LLVM_PREFIX/lib -Wl,-rpath,$LLVM_PREFIX/lib" echo "CPPFLAGS=-I$LLVM_PREFIX/include" } >> "$GITHUB_ENV" From 71c0a05e9a6fd734761d076d96f88ffd4b3b3973 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 21:18:48 -0400 Subject: [PATCH 11/47] Remove ld flags --- .github/workflows/clp-core-build-macos.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index c3ab824a8e..654fe0b930 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -79,8 +79,6 @@ jobs: echo "LLVM_PREFIX=$LLVM_PREFIX" echo "CC=$LLVM_PREFIX/bin/clang" echo "CXX=$LLVM_PREFIX/bin/clang++" - echo "LDFLAGS=-L$LLVM_PREFIX/lib -Wl,-rpath,$LLVM_PREFIX/lib" - echo "CPPFLAGS=-I$LLVM_PREFIX/include" } >> "$GITHUB_ENV" - run: "./tools/scripts/deps-download/init.sh" From 7d534bf12cbd5fabc0cb8dabc94112973bc5c9da Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 21:46:53 -0400 Subject: [PATCH 12/47] Revert "Temporarily remove other GH workflows" This reverts commit cfad209be24e966a2883bc34eb32d2df6e80389e. --- .github/workflows/clp-core-build.yaml | 376 ++++++++++++++++++ .github/workflows/clp-docs.yaml | 57 +++ .../workflows/clp-execution-image-build.yaml | 70 ++++ .github/workflows/clp-lint.yaml | 42 ++ .github/workflows/clp-pr-title-checks.yaml | 31 ++ .../clp-s-generated-code-checks.yaml | 42 ++ 6 files changed, 618 insertions(+) create mode 100644 .github/workflows/clp-core-build.yaml create mode 100644 .github/workflows/clp-docs.yaml create mode 100644 .github/workflows/clp-execution-image-build.yaml create mode 100644 .github/workflows/clp-lint.yaml create mode 100644 .github/workflows/clp-pr-title-checks.yaml create mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml new file mode 100644 index 0000000000..ad0ea8fbf2 --- /dev/null +++ b/.github/workflows/clp-core-build.yaml @@ -0,0 +1,376 @@ +name: "clp-core-build" + +on: + pull_request: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + push: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +env: + BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" + DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted + # publishing of container images. + cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + clp_changed: "${{steps.filter.outputs.clp}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + # Consider changes between the current commit and `main` + # NOTE: If a pull request changes one of the images, then we need to (1) build the image + # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull + # request doesn't change an image, then we don't need to rebuild the image; instead we can + # use the published image which is based on `main`. So when determining what files have + # changed, we need to consider the delta between the current commit and `main` (rather + # than the current and previous commits) in order to detect if we need to rebuild the + # image (since it would be different from the published image). + base: "main" + filters: | + centos_stream_9_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" + - "components/core/tools/scripts/lib_install/centos-stream-9/**" + ubuntu_jammy_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" + - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" + clp: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/cmake/**" + - "components/core/CMakeLists.txt" + - "components/core/src/**" + - "components/core/tests/**" + - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + + centos-stream-9-deps-image: + name: "centos-stream-9-deps-image" + if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + ubuntu-jammy-deps-image: + name: "ubuntu-jammy-deps-image" + if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + centos-stream-9-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "centos-stream-9-deps-image" + - "filter-relevant-changes" + strategy: + matrix: + use_shared_libs: [true, false] + name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + ubuntu-jammy-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + strategy: + matrix: + include: + - use_shared_libs: true + upload_binaries: false + - use_shared_libs: false + upload_binaries: true + env: + OS_NAME: "ubuntu-jammy" + name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + - if: "matrix.upload_binaries == true" + id: "copy_binaries" + run: |- + output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + mkdir -p "${output_dir}" + cd "$GITHUB_WORKSPACE/components/core/build" + tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable + shell: "bash" + + - if: "matrix.upload_binaries == true" + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{steps.copy_binaries.outputs.output_dir}}" + retention-days: 1 + + ubuntu-jammy-binaries-image: + name: "ubuntu-jammy-binaries-image" + # Run if the ancestor jobs were successful/skipped and building clp was successful. + if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" + needs: "ubuntu-jammy-binaries" + runs-on: "ubuntu-latest" + env: + OS_NAME: "ubuntu-jammy" + TMP_OUTPUT_DIR: "/tmp" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + + - name: "Untar binaries" + working-directory: >- + ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} + run: |- + tar xf clp.tar + rm clp.tar + + - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Sanitize the repo's name" + id: "sanitize_repo_name" + run: |- + # Docker doesn't support repository names with uppercase characters, so we convert to + # lowercase here. + lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') + echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" + shell: "bash" + + - id: "core_image_meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} + tags: "type=raw,value=${{github.ref_name}}" + + # Only publish the image if this workflow was triggered by a push to `main`. + # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. + - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" + push: true + tags: "${{steps.core_image_meta.outputs.tags}}" + labels: "${{steps.core_image_meta.outputs.labels}}" + + ubuntu-jammy-lint: + name: "ubuntu-jammy-lint" + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. + fetch-depth: 0 + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full + # on all files. + - if: "'schedule' != github.event_name" + name: "Restore lint:check-cpp-static-full cache" + id: "cache-restore-lint-check-cpp-static-full" + uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + + # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy + # violations. + key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. + # run_command: >- + # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + task lint:check-cpp-full + + # Cache the source file checksums and the generated files (logs) for the + # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. + # NOTE: If we don't cache the generated files, the task will re-run to generate them. + - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" + name: "Update lint:check-cpp-static-full cache" + uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml new file mode 100644 index 0000000000..82fc4a0e90 --- /dev/null +++ b/.github/workflows/clp-docs.yaml @@ -0,0 +1,57 @@ +name: "clp-docs" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + build: + name: "build" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + lfs: "true" + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.10" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Build docs" + shell: "bash" + run: "task docs:site" + + # Upload the built docs so we can download and deploy them from y-scope/yscope-docs + - if: >- + contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) + && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) + && 'ubuntu-latest' == matrix.os + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "docs-html" + path: "build/docs/html" + if-no-files-found: "error" + + # Retain the artifact for a week in case there was a deployment issue + retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml new file mode 100644 index 0000000000..17789906e8 --- /dev/null +++ b/.github/workflows/clp-execution-image-build.yaml @@ -0,0 +1,70 @@ +name: "clp-execution-image-build" + +on: + pull_request: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + push: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + base: "main" + filters: | + ubuntu_jammy_image: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" + + ubuntu-jammy-execution-image: + name: "ubuntu-jammy-execution-image" + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - uses: "./.github/actions/clp-execution-image-build" + with: + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml new file mode 100644 index 0000000000..5f800d8c23 --- /dev/null +++ b/.github/workflows/clp-lint.yaml @@ -0,0 +1,42 @@ +name: "clp-lint" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + lint-check: + name: "lint-check" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.11" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Run lint task" + shell: "bash" + run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml new file mode 100644 index 0000000000..0aa1533a98 --- /dev/null +++ b/.github/workflows/clp-pr-title-checks.yaml @@ -0,0 +1,31 @@ +name: "clp-pr-title-checks" + +on: + pull_request_target: + # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the + # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: + # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the + # pull request triggered by this event. + # - Each job has `permissions` set to only those necessary. + types: ["edited", "opened", "reopened"] + branches: ["main"] + +permissions: {} + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + conventional-commits: + name: "conventional-commits" + permissions: + # For amannn/action-semantic-pull-request + pull-requests: "read" + runs-on: "ubuntu-latest" + steps: + - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" + env: + GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml new file mode 100644 index 0000000000..e605db9005 --- /dev/null +++ b/.github/workflows/clp-s-generated-code-checks.yaml @@ -0,0 +1,42 @@ +name: "clp-s-generated-code-checks" + +on: + pull_request: + push: + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + antlr-code-committed: + name: "antlr-code-committed" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Check if the generated parsers are the latest" + shell: "bash" + run: + | + git status --porcelain \ + components/core/src/clp_s/search/kql/generated \ + components/core/src/clp_s/search/sql/generated \ + | grep . > /dev/null \ + && exit 1 \ + || exit 0 From cf2c37b97314433e4167b94b45b619002ed31ee4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 10:55:51 -0400 Subject: [PATCH 13/47] Experiment with extra clang-tidy flags --- taskfiles/lint.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index b07c4cad31..aef2da5d32 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -177,6 +177,8 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" + - "-Drsize_t=size_t" + - "-Derrno_t=int" OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" @@ -764,6 +766,8 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" + - "-Drsize_t=size_t" + - "-Derrno_t=int" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From fa5dfc2d2f6d4d99106b0c3f2cda968f33e57a12 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 15:19:57 -0400 Subject: [PATCH 14/47] Temporarily remove other GH workflows --- .github/workflows/clp-core-build.yaml | 376 ------------------ .github/workflows/clp-docs.yaml | 57 --- .../workflows/clp-execution-image-build.yaml | 70 ---- .github/workflows/clp-lint.yaml | 42 -- .github/workflows/clp-pr-title-checks.yaml | 31 -- .../clp-s-generated-code-checks.yaml | 42 -- 6 files changed, 618 deletions(-) delete mode 100644 .github/workflows/clp-core-build.yaml delete mode 100644 .github/workflows/clp-docs.yaml delete mode 100644 .github/workflows/clp-execution-image-build.yaml delete mode 100644 .github/workflows/clp-lint.yaml delete mode 100644 .github/workflows/clp-pr-title-checks.yaml delete mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml deleted file mode 100644 index ad0ea8fbf2..0000000000 --- a/.github/workflows/clp-core-build.yaml +++ /dev/null @@ -1,376 +0,0 @@ -name: "clp-core-build" - -on: - pull_request: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - push: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -env: - BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted - # publishing of container images. - cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - clp_changed: "${{steps.filter.outputs.clp}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - # Consider changes between the current commit and `main` - # NOTE: If a pull request changes one of the images, then we need to (1) build the image - # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull - # request doesn't change an image, then we don't need to rebuild the image; instead we can - # use the published image which is based on `main`. So when determining what files have - # changed, we need to consider the delta between the current commit and `main` (rather - # than the current and previous commits) in order to detect if we need to rebuild the - # image (since it would be different from the published image). - base: "main" - filters: | - centos_stream_9_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - - "components/core/tools/scripts/lib_install/centos-stream-9/**" - ubuntu_jammy_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" - clp: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/cmake/**" - - "components/core/CMakeLists.txt" - - "components/core/src/**" - - "components/core/tests/**" - - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - centos-stream-9-deps-image: - name: "centos-stream-9-deps-image" - if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - ubuntu-jammy-deps-image: - name: "ubuntu-jammy-deps-image" - if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - centos-stream-9-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "centos-stream-9-deps-image" - - "filter-relevant-changes" - strategy: - matrix: - use_shared_libs: [true, false] - name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - ubuntu-jammy-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - strategy: - matrix: - include: - - use_shared_libs: true - upload_binaries: false - - use_shared_libs: false - upload_binaries: true - env: - OS_NAME: "ubuntu-jammy" - name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - - if: "matrix.upload_binaries == true" - id: "copy_binaries" - run: |- - output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - - mkdir -p "${output_dir}" - cd "$GITHUB_WORKSPACE/components/core/build" - tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable - shell: "bash" - - - if: "matrix.upload_binaries == true" - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{steps.copy_binaries.outputs.output_dir}}" - retention-days: 1 - - ubuntu-jammy-binaries-image: - name: "ubuntu-jammy-binaries-image" - # Run if the ancestor jobs were successful/skipped and building clp was successful. - if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" - needs: "ubuntu-jammy-binaries" - runs-on: "ubuntu-latest" - env: - OS_NAME: "ubuntu-jammy" - TMP_OUTPUT_DIR: "/tmp" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - - - name: "Untar binaries" - working-directory: >- - ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} - run: |- - tar xf clp.tar - rm clp.tar - - - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" - with: - registry: "ghcr.io" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" - - - name: "Sanitize the repo's name" - id: "sanitize_repo_name" - run: |- - # Docker doesn't support repository names with uppercase characters, so we convert to - # lowercase here. - lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') - echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" - shell: "bash" - - - id: "core_image_meta" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: >- - ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} - tags: "type=raw,value=${{github.ref_name}}" - - # Only publish the image if this workflow was triggered by a push to `main`. - # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. - - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" - push: true - tags: "${{steps.core_image_meta.outputs.tags}}" - labels: "${{steps.core_image_meta.outputs.labels}}" - - ubuntu-jammy-lint: - name: "ubuntu-jammy-lint" - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. - fetch-depth: 0 - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full - # on all files. - - if: "'schedule' != github.event_name" - name: "Restore lint:check-cpp-static-full cache" - id: "cache-restore-lint-check-cpp-static-full" - uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - - # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy - # violations. - key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. - # run_command: >- - # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) - task lint:check-cpp-full - - # Cache the source file checksums and the generated files (logs) for the - # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. - # NOTE: If we don't cache the generated files, the task will re-run to generate them. - - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" - name: "Update lint:check-cpp-static-full cache" - uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml deleted file mode 100644 index 82fc4a0e90..0000000000 --- a/.github/workflows/clp-docs.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: "clp-docs" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - build: - name: "build" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - lfs: "true" - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.10" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Build docs" - shell: "bash" - run: "task docs:site" - - # Upload the built docs so we can download and deploy them from y-scope/yscope-docs - - if: >- - contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) - && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) - && 'ubuntu-latest' == matrix.os - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "docs-html" - path: "build/docs/html" - if-no-files-found: "error" - - # Retain the artifact for a week in case there was a deployment issue - retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml deleted file mode 100644 index 17789906e8..0000000000 --- a/.github/workflows/clp-execution-image-build.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: "clp-execution-image-build" - -on: - pull_request: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - push: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - base: "main" - filters: | - ubuntu_jammy_image: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" - - ubuntu-jammy-execution-image: - name: "ubuntu-jammy-execution-image" - if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "./.github/actions/clp-execution-image-build" - with: - image_registry: "ghcr.io" - image_registry_username: "${{github.actor}}" - image_registry_password: "${{secrets.GITHUB_TOKEN}}" - platform_id: "ubuntu" - platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml deleted file mode 100644 index 5f800d8c23..0000000000 --- a/.github/workflows/clp-lint.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-lint" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - lint-check: - name: "lint-check" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.11" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Run lint task" - shell: "bash" - run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml deleted file mode 100644 index 0aa1533a98..0000000000 --- a/.github/workflows/clp-pr-title-checks.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: "clp-pr-title-checks" - -on: - pull_request_target: - # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the - # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: - # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the - # pull request triggered by this event. - # - Each job has `permissions` set to only those necessary. - types: ["edited", "opened", "reopened"] - branches: ["main"] - -permissions: {} - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - conventional-commits: - name: "conventional-commits" - permissions: - # For amannn/action-semantic-pull-request - pull-requests: "read" - runs-on: "ubuntu-latest" - steps: - - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" - env: - GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml deleted file mode 100644 index e605db9005..0000000000 --- a/.github/workflows/clp-s-generated-code-checks.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-s-generated-code-checks" - -on: - pull_request: - push: - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - antlr-code-committed: - name: "antlr-code-committed" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Check if the generated parsers are the latest" - shell: "bash" - run: - | - git status --porcelain \ - components/core/src/clp_s/search/kql/generated \ - components/core/src/clp_s/search/sql/generated \ - | grep . > /dev/null \ - && exit 1 \ - || exit 0 From e2434cb2c1f85931aaca05a18c6d6f336cd3129d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 15:21:18 -0400 Subject: [PATCH 15/47] Revert "Experiment with extra clang-tidy flags" This reverts commit cf2c37b97314433e4167b94b45b619002ed31ee4. --- taskfiles/lint.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index aef2da5d32..b07c4cad31 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -177,8 +177,6 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "-Drsize_t=size_t" - - "-Derrno_t=int" OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" @@ -766,8 +764,6 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "-Drsize_t=size_t" - - "-Derrno_t=int" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From 39961b4327a0767b6a328abddcda5591fc4afaf6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 15:21:45 -0400 Subject: [PATCH 16/47] Add more environment flags --- .github/workflows/clp-core-build-macos.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 654fe0b930..0337e8e0d5 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -79,6 +79,9 @@ jobs: echo "LLVM_PREFIX=$LLVM_PREFIX" echo "CC=$LLVM_PREFIX/bin/clang" echo "CXX=$LLVM_PREFIX/bin/clang++" + echo "LDFLAGS=-L$LLVM_PREFIX/lib" + echo "CPPFLAGS=-I$LLVM_PREFIX/include" + echo "LD_LIBRARY_PATH=$LLVM_PREFIX/lib" } >> "$GITHUB_ENV" - run: "./tools/scripts/deps-download/init.sh" From 5e9d8c6221fb79140cabf5dbdafdff348fbfb637 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 15:41:09 -0400 Subject: [PATCH 17/47] Experiment with environment variables --- .github/workflows/clp-core-build-macos.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 0337e8e0d5..caf09366af 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -80,8 +80,10 @@ jobs: echo "CC=$LLVM_PREFIX/bin/clang" echo "CXX=$LLVM_PREFIX/bin/clang++" echo "LDFLAGS=-L$LLVM_PREFIX/lib" + echo "CFLAGS=-I$LLVM_PREFIX/include" echo "CPPFLAGS=-I$LLVM_PREFIX/include" - echo "LD_LIBRARY_PATH=$LLVM_PREFIX/lib" + echo "LIBRARY_PATH=$LLVM_PREFIX/lib" + echo "DYLD_LIBRARY_PATH=$LLVM_PREFIX/lib" } >> "$GITHUB_ENV" - run: "./tools/scripts/deps-download/init.sh" From 9478ae8b3382e70eef9ed75b8b55b3c78fb17ced Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 15:57:44 -0400 Subject: [PATCH 18/47] Add more experiment with environment variables AR and RANLIB --- .github/workflows/clp-core-build-macos.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index caf09366af..fccc4df963 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -79,10 +79,10 @@ jobs: echo "LLVM_PREFIX=$LLVM_PREFIX" echo "CC=$LLVM_PREFIX/bin/clang" echo "CXX=$LLVM_PREFIX/bin/clang++" + echo "AR=$LLVM_PREFIX/bin/llvm-ar" + echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" echo "LDFLAGS=-L$LLVM_PREFIX/lib" - echo "CFLAGS=-I$LLVM_PREFIX/include" echo "CPPFLAGS=-I$LLVM_PREFIX/include" - echo "LIBRARY_PATH=$LLVM_PREFIX/lib" echo "DYLD_LIBRARY_PATH=$LLVM_PREFIX/lib" } >> "$GITHUB_ENV" From fca48575bf703671bd4362765200749d72068c2b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 16:16:34 -0400 Subject: [PATCH 19/47] Remove environment flags for flags --- .github/workflows/clp-core-build-macos.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index fccc4df963..7c8c44f977 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -81,9 +81,6 @@ jobs: echo "CXX=$LLVM_PREFIX/bin/clang++" echo "AR=$LLVM_PREFIX/bin/llvm-ar" echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" - echo "LDFLAGS=-L$LLVM_PREFIX/lib" - echo "CPPFLAGS=-I$LLVM_PREFIX/include" - echo "DYLD_LIBRARY_PATH=$LLVM_PREFIX/lib" } >> "$GITHUB_ENV" - run: "./tools/scripts/deps-download/init.sh" From 67d2363a06d32e1b57edc21a24fba05350d06d72 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 17:33:02 -0400 Subject: [PATCH 20/47] Experiment with clang-tidy argument --- taskfiles/lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index b07c4cad31..42fadf0461 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -764,6 +764,7 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" + - "--extra-arg=-isysroot$(xcrun --show-sdk-path)" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From 2d82296956b13488566ec9336a214ea09757e3c3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 23:43:46 -0400 Subject: [PATCH 21/47] Try add llvm to clang-tidy sysroot --- taskfiles/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 42fadf0461..836a4b3419 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -764,7 +764,7 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "--extra-arg=-isysroot$(xcrun --show-sdk-path)" + - "--extra-arg=-isysroot$LLVM_PREFIX/lib" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From 260b69d8ea5c88c33d907de1f1b07d506e81e76f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 23:58:25 -0400 Subject: [PATCH 22/47] Revert "Try add llvm to clang-tidy sysroot" This reverts commit 2d82296956b13488566ec9336a214ea09757e3c3. --- taskfiles/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 836a4b3419..42fadf0461 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -764,7 +764,7 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "--extra-arg=-isysroot$LLVM_PREFIX/lib" + - "--extra-arg=-isysroot$(xcrun --show-sdk-path)" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From c91dc84196208537e084c5eb9de74dce26a6721c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 23:59:36 -0400 Subject: [PATCH 23/47] Not use llvm toolchain in clp --- .../core/cmake/Toolchains/llvm-clang-16-toolchain.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake b/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake index 583e044652..b1c61c1b5e 100644 --- a/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake +++ b/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake @@ -17,5 +17,5 @@ endif() set(CMAKE_C_COMPILER "${LLVM_TOOLCHAIN_PREFIX}/bin/clang") set(CMAKE_CXX_COMPILER "${LLVM_TOOLCHAIN_PREFIX}/bin/clang++") -set(CMAKE_AR "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ar") -set(CMAKE_RANLIB "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ranlib") +#set(CMAKE_AR "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ar") +#set(CMAKE_RANLIB "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ranlib") From 693e2d03d5f69862c519a49ac49c9979b7376ff8 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 28 Jun 2025 12:58:34 -0400 Subject: [PATCH 24/47] Revert "Not use llvm toolchain in clp" This reverts commit c91dc84196208537e084c5eb9de74dce26a6721c. --- .../core/cmake/Toolchains/llvm-clang-16-toolchain.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake b/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake index b1c61c1b5e..583e044652 100644 --- a/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake +++ b/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake @@ -17,5 +17,5 @@ endif() set(CMAKE_C_COMPILER "${LLVM_TOOLCHAIN_PREFIX}/bin/clang") set(CMAKE_CXX_COMPILER "${LLVM_TOOLCHAIN_PREFIX}/bin/clang++") -#set(CMAKE_AR "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ar") -#set(CMAKE_RANLIB "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ranlib") +set(CMAKE_AR "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ar") +set(CMAKE_RANLIB "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ranlib") From 234a1f31fff3d6a235dae89d46f48fb9d3bfd1b6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 28 Jun 2025 13:01:24 -0400 Subject: [PATCH 25/47] Experiment with extra args --- taskfiles/lint.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 42fadf0461..2bfbff3ef6 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -764,7 +764,8 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "--extra-arg=-isysroot$(xcrun --show-sdk-path)" + - "--extra-arg=-Drsize=size_t" + - "--extra-arg=-Derrno_t=int" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From 99cdaccded6502dd40e33e793ffb0e8f46e284a7 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 28 Jun 2025 13:33:43 -0400 Subject: [PATCH 26/47] Remove errno_t=int from clang-tidy flags --- taskfiles/lint.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 2bfbff3ef6..bec7e6e3dd 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -765,7 +765,6 @@ tasks: --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - "--extra-arg=-Drsize=size_t" - - "--extra-arg=-Derrno_t=int" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From 0f1034aec1efde6c754f8ab754fd59b7b0ccafd6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 28 Jun 2025 14:03:20 -0400 Subject: [PATCH 27/47] Bug fix --- taskfiles/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index bec7e6e3dd..9c4a867dc8 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -764,7 +764,7 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "--extra-arg=-Drsize=size_t" + - "--extra-arg=-Drsize_t=size_t" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From db7cd48d006e59ed6ac92cf09997f73e56095446 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 28 Jun 2025 15:00:47 -0400 Subject: [PATCH 28/47] Revert "Temporarily remove other GH workflows" This reverts commit fa5dfc2d2f6d4d99106b0c3f2cda968f33e57a12. --- .github/workflows/clp-core-build.yaml | 376 ++++++++++++++++++ .github/workflows/clp-docs.yaml | 57 +++ .../workflows/clp-execution-image-build.yaml | 70 ++++ .github/workflows/clp-lint.yaml | 42 ++ .github/workflows/clp-pr-title-checks.yaml | 31 ++ .../clp-s-generated-code-checks.yaml | 42 ++ 6 files changed, 618 insertions(+) create mode 100644 .github/workflows/clp-core-build.yaml create mode 100644 .github/workflows/clp-docs.yaml create mode 100644 .github/workflows/clp-execution-image-build.yaml create mode 100644 .github/workflows/clp-lint.yaml create mode 100644 .github/workflows/clp-pr-title-checks.yaml create mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml new file mode 100644 index 0000000000..ad0ea8fbf2 --- /dev/null +++ b/.github/workflows/clp-core-build.yaml @@ -0,0 +1,376 @@ +name: "clp-core-build" + +on: + pull_request: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + push: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +env: + BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" + DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted + # publishing of container images. + cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + clp_changed: "${{steps.filter.outputs.clp}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + # Consider changes between the current commit and `main` + # NOTE: If a pull request changes one of the images, then we need to (1) build the image + # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull + # request doesn't change an image, then we don't need to rebuild the image; instead we can + # use the published image which is based on `main`. So when determining what files have + # changed, we need to consider the delta between the current commit and `main` (rather + # than the current and previous commits) in order to detect if we need to rebuild the + # image (since it would be different from the published image). + base: "main" + filters: | + centos_stream_9_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" + - "components/core/tools/scripts/lib_install/centos-stream-9/**" + ubuntu_jammy_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" + - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" + clp: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/cmake/**" + - "components/core/CMakeLists.txt" + - "components/core/src/**" + - "components/core/tests/**" + - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + + centos-stream-9-deps-image: + name: "centos-stream-9-deps-image" + if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + ubuntu-jammy-deps-image: + name: "ubuntu-jammy-deps-image" + if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + centos-stream-9-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "centos-stream-9-deps-image" + - "filter-relevant-changes" + strategy: + matrix: + use_shared_libs: [true, false] + name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + ubuntu-jammy-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + strategy: + matrix: + include: + - use_shared_libs: true + upload_binaries: false + - use_shared_libs: false + upload_binaries: true + env: + OS_NAME: "ubuntu-jammy" + name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + - if: "matrix.upload_binaries == true" + id: "copy_binaries" + run: |- + output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + mkdir -p "${output_dir}" + cd "$GITHUB_WORKSPACE/components/core/build" + tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable + shell: "bash" + + - if: "matrix.upload_binaries == true" + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{steps.copy_binaries.outputs.output_dir}}" + retention-days: 1 + + ubuntu-jammy-binaries-image: + name: "ubuntu-jammy-binaries-image" + # Run if the ancestor jobs were successful/skipped and building clp was successful. + if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" + needs: "ubuntu-jammy-binaries" + runs-on: "ubuntu-latest" + env: + OS_NAME: "ubuntu-jammy" + TMP_OUTPUT_DIR: "/tmp" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + + - name: "Untar binaries" + working-directory: >- + ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} + run: |- + tar xf clp.tar + rm clp.tar + + - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Sanitize the repo's name" + id: "sanitize_repo_name" + run: |- + # Docker doesn't support repository names with uppercase characters, so we convert to + # lowercase here. + lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') + echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" + shell: "bash" + + - id: "core_image_meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} + tags: "type=raw,value=${{github.ref_name}}" + + # Only publish the image if this workflow was triggered by a push to `main`. + # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. + - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" + push: true + tags: "${{steps.core_image_meta.outputs.tags}}" + labels: "${{steps.core_image_meta.outputs.labels}}" + + ubuntu-jammy-lint: + name: "ubuntu-jammy-lint" + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. + fetch-depth: 0 + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full + # on all files. + - if: "'schedule' != github.event_name" + name: "Restore lint:check-cpp-static-full cache" + id: "cache-restore-lint-check-cpp-static-full" + uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + + # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy + # violations. + key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. + # run_command: >- + # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + task lint:check-cpp-full + + # Cache the source file checksums and the generated files (logs) for the + # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. + # NOTE: If we don't cache the generated files, the task will re-run to generate them. + - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" + name: "Update lint:check-cpp-static-full cache" + uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml new file mode 100644 index 0000000000..82fc4a0e90 --- /dev/null +++ b/.github/workflows/clp-docs.yaml @@ -0,0 +1,57 @@ +name: "clp-docs" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + build: + name: "build" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + lfs: "true" + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.10" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Build docs" + shell: "bash" + run: "task docs:site" + + # Upload the built docs so we can download and deploy them from y-scope/yscope-docs + - if: >- + contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) + && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) + && 'ubuntu-latest' == matrix.os + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "docs-html" + path: "build/docs/html" + if-no-files-found: "error" + + # Retain the artifact for a week in case there was a deployment issue + retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml new file mode 100644 index 0000000000..17789906e8 --- /dev/null +++ b/.github/workflows/clp-execution-image-build.yaml @@ -0,0 +1,70 @@ +name: "clp-execution-image-build" + +on: + pull_request: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + push: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + base: "main" + filters: | + ubuntu_jammy_image: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" + + ubuntu-jammy-execution-image: + name: "ubuntu-jammy-execution-image" + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - uses: "./.github/actions/clp-execution-image-build" + with: + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml new file mode 100644 index 0000000000..5f800d8c23 --- /dev/null +++ b/.github/workflows/clp-lint.yaml @@ -0,0 +1,42 @@ +name: "clp-lint" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + lint-check: + name: "lint-check" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.11" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Run lint task" + shell: "bash" + run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml new file mode 100644 index 0000000000..0aa1533a98 --- /dev/null +++ b/.github/workflows/clp-pr-title-checks.yaml @@ -0,0 +1,31 @@ +name: "clp-pr-title-checks" + +on: + pull_request_target: + # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the + # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: + # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the + # pull request triggered by this event. + # - Each job has `permissions` set to only those necessary. + types: ["edited", "opened", "reopened"] + branches: ["main"] + +permissions: {} + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + conventional-commits: + name: "conventional-commits" + permissions: + # For amannn/action-semantic-pull-request + pull-requests: "read" + runs-on: "ubuntu-latest" + steps: + - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" + env: + GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml new file mode 100644 index 0000000000..e605db9005 --- /dev/null +++ b/.github/workflows/clp-s-generated-code-checks.yaml @@ -0,0 +1,42 @@ +name: "clp-s-generated-code-checks" + +on: + pull_request: + push: + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + antlr-code-committed: + name: "antlr-code-committed" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Check if the generated parsers are the latest" + shell: "bash" + run: + | + git status --porcelain \ + components/core/src/clp_s/search/kql/generated \ + components/core/src/clp_s/search/sql/generated \ + | grep . > /dev/null \ + && exit 1 \ + || exit 0 From 9e2c951104bc70d7e782c32e78a70eef864699e3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 28 Jun 2025 16:04:37 -0400 Subject: [PATCH 29/47] Add rsize clang-tidy flag in all clang-tidy tasks --- taskfiles/lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 9c4a867dc8..0af185443e 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -177,6 +177,7 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" + - "--extra-arg=-Drsize_t=size_t" OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" From fcb664e91cb3b08021f3cd43b8f33b37666cfc74 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 11:35:05 -0400 Subject: [PATCH 30/47] Test if setting GITHUB_ENV inside script works --- .github/workflows/clp-core-build-macos.yaml | 12 ------------ .../tools/scripts/lib_install/macos/install-all.sh | 7 +++++++ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 7c8c44f977..51eb6985d0 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -71,18 +71,6 @@ jobs: - name: "Install dependencies" run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" - - name: "Set environment to use Homebrew's llvm" - run: | - LLVM_PREFIX=$(brew --prefix llvm@16) - echo "$LLVM_PREFIX/bin" >> "$GITHUB_PATH" - { - echo "LLVM_PREFIX=$LLVM_PREFIX" - echo "CC=$LLVM_PREFIX/bin/clang" - echo "CXX=$LLVM_PREFIX/bin/clang++" - echo "AR=$LLVM_PREFIX/bin/llvm-ar" - echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" - } >> "$GITHUB_ENV" - - run: "./tools/scripts/deps-download/init.sh" shell: "bash" diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 57d77af764..3ee7f3ce87 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -48,6 +48,13 @@ brew install \ xz \ zstd +LLVM_PREFIX=$(brew --prefix llvm@16) +echo "LLVM_PREFIX=$LLVM_PREFIX" >> "$GITHUB_ENV" +echo "CC=$LLVM_PREFIX/bin/clang" >> "$GITHUB_ENV" +echo "CXX=$LLVM_PREFIX/bin/clang++" >> "$GITHUB_ENV" +echo "AR=$LLVM_PREFIX/bin/llvm-ar" >> "$GITHUB_ENV" +echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" >> "$GITHUB_ENV" + # Install pkg-config if it isn't already installed # NOTE: We might expect that pkg-config is installed through brew, so trying to install it again # would be harmless; however, in certain environments, like the macOS GitHub hosted runner, From e09f0b884905c3f21322d1885bbb899ac5c0712c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 13:04:55 -0400 Subject: [PATCH 31/47] Add check for GITHUB_ENV --- .../tools/scripts/lib_install/macos/install-all.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 3ee7f3ce87..8169051240 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -48,12 +48,14 @@ brew install \ xz \ zstd -LLVM_PREFIX=$(brew --prefix llvm@16) -echo "LLVM_PREFIX=$LLVM_PREFIX" >> "$GITHUB_ENV" -echo "CC=$LLVM_PREFIX/bin/clang" >> "$GITHUB_ENV" -echo "CXX=$LLVM_PREFIX/bin/clang++" >> "$GITHUB_ENV" -echo "AR=$LLVM_PREFIX/bin/llvm-ar" >> "$GITHUB_ENV" -echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" >> "$GITHUB_ENV" +if [-n "$GITHUB_ENV"]; then + LLVM_PREFIX=$(brew --prefix llvm@16) + echo "LLVM_PREFIX=$LLVM_PREFIX" >> "$GITHUB_ENV" + echo "CC=$LLVM_PREFIX/bin/clang" >> "$GITHUB_ENV" + echo "CXX=$LLVM_PREFIX/bin/clang++" >> "$GITHUB_ENV" + echo "AR=$LLVM_PREFIX/bin/llvm-ar" >> "$GITHUB_ENV" + echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" >> "$GITHUB_ENV" +fi # Install pkg-config if it isn't already installed # NOTE: We might expect that pkg-config is installed through brew, so trying to install it again From ba7cd2f201e24651119ecbf72272c97cd02a46ed Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 13:11:10 -0400 Subject: [PATCH 32/47] Fix test --- .../tools/scripts/lib_install/macos/install-all.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 8169051240..d55e56f13f 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -48,13 +48,15 @@ brew install \ xz \ zstd -if [-n "$GITHUB_ENV"]; then +if [-n "${GITHUB_ENV}"]; then LLVM_PREFIX=$(brew --prefix llvm@16) - echo "LLVM_PREFIX=$LLVM_PREFIX" >> "$GITHUB_ENV" - echo "CC=$LLVM_PREFIX/bin/clang" >> "$GITHUB_ENV" - echo "CXX=$LLVM_PREFIX/bin/clang++" >> "$GITHUB_ENV" - echo "AR=$LLVM_PREFIX/bin/llvm-ar" >> "$GITHUB_ENV" - echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" >> "$GITHUB_ENV" + { + echo "LLVM_PREFIX=$LLVM_PREFIX" + echo "CC=$LLVM_PREFIX/bin/clang" + echo "CXX=$LLVM_PREFIX/bin/clang++" + echo "AR=$LLVM_PREFIX/bin/llvm-ar" + echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" + } >> "$GITHUB_ENV" fi # Install pkg-config if it isn't already installed From 2ce75bd9de3b44baf86d8800f759e47667b9c8e9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 13:57:58 -0400 Subject: [PATCH 33/47] Remove test for GITHUB_ENV --- .../scripts/lib_install/macos/install-all.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index d55e56f13f..3065b1956e 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -48,16 +48,14 @@ brew install \ xz \ zstd -if [-n "${GITHUB_ENV}"]; then - LLVM_PREFIX=$(brew --prefix llvm@16) - { - echo "LLVM_PREFIX=$LLVM_PREFIX" - echo "CC=$LLVM_PREFIX/bin/clang" - echo "CXX=$LLVM_PREFIX/bin/clang++" - echo "AR=$LLVM_PREFIX/bin/llvm-ar" - echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" - } >> "$GITHUB_ENV" -fi +LLVM_PREFIX=$(brew --prefix llvm@16) +{ + echo "LLVM_PREFIX=$LLVM_PREFIX" + echo "CC=$LLVM_PREFIX/bin/clang" + echo "CXX=$LLVM_PREFIX/bin/clang++" + echo "AR=$LLVM_PREFIX/bin/llvm-ar" + echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" +} >> "$GITHUB_ENV" # Install pkg-config if it isn't already installed # NOTE: We might expect that pkg-config is installed through brew, so trying to install it again From accd3c49844cd2a7e66a6be33d860ab2ae6a6820 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 14:06:38 -0400 Subject: [PATCH 34/47] Add tests for GITHUB_ENV --- components/core/tools/scripts/lib_install/macos/install-all.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 3065b1956e..b467152ff1 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -48,6 +48,7 @@ brew install \ xz \ zstd +echo "$GITHUB_ENV" LLVM_PREFIX=$(brew --prefix llvm@16) { echo "LLVM_PREFIX=$LLVM_PREFIX" From a2220741a26d4d90ae2a4e94f8f91f01a667ac66 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 14:07:14 -0400 Subject: [PATCH 35/47] Temporarily remove all other workflows --- .github/workflows/clp-core-build.yaml | 376 ------------------ .github/workflows/clp-docs.yaml | 57 --- .../workflows/clp-execution-image-build.yaml | 70 ---- .github/workflows/clp-lint.yaml | 42 -- .github/workflows/clp-pr-title-checks.yaml | 31 -- .../clp-s-generated-code-checks.yaml | 42 -- 6 files changed, 618 deletions(-) delete mode 100644 .github/workflows/clp-core-build.yaml delete mode 100644 .github/workflows/clp-docs.yaml delete mode 100644 .github/workflows/clp-execution-image-build.yaml delete mode 100644 .github/workflows/clp-lint.yaml delete mode 100644 .github/workflows/clp-pr-title-checks.yaml delete mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml deleted file mode 100644 index ad0ea8fbf2..0000000000 --- a/.github/workflows/clp-core-build.yaml +++ /dev/null @@ -1,376 +0,0 @@ -name: "clp-core-build" - -on: - pull_request: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - push: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -env: - BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted - # publishing of container images. - cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - clp_changed: "${{steps.filter.outputs.clp}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - # Consider changes between the current commit and `main` - # NOTE: If a pull request changes one of the images, then we need to (1) build the image - # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull - # request doesn't change an image, then we don't need to rebuild the image; instead we can - # use the published image which is based on `main`. So when determining what files have - # changed, we need to consider the delta between the current commit and `main` (rather - # than the current and previous commits) in order to detect if we need to rebuild the - # image (since it would be different from the published image). - base: "main" - filters: | - centos_stream_9_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - - "components/core/tools/scripts/lib_install/centos-stream-9/**" - ubuntu_jammy_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" - clp: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/cmake/**" - - "components/core/CMakeLists.txt" - - "components/core/src/**" - - "components/core/tests/**" - - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - centos-stream-9-deps-image: - name: "centos-stream-9-deps-image" - if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - ubuntu-jammy-deps-image: - name: "ubuntu-jammy-deps-image" - if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - centos-stream-9-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "centos-stream-9-deps-image" - - "filter-relevant-changes" - strategy: - matrix: - use_shared_libs: [true, false] - name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - ubuntu-jammy-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - strategy: - matrix: - include: - - use_shared_libs: true - upload_binaries: false - - use_shared_libs: false - upload_binaries: true - env: - OS_NAME: "ubuntu-jammy" - name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - - if: "matrix.upload_binaries == true" - id: "copy_binaries" - run: |- - output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - - mkdir -p "${output_dir}" - cd "$GITHUB_WORKSPACE/components/core/build" - tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable - shell: "bash" - - - if: "matrix.upload_binaries == true" - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{steps.copy_binaries.outputs.output_dir}}" - retention-days: 1 - - ubuntu-jammy-binaries-image: - name: "ubuntu-jammy-binaries-image" - # Run if the ancestor jobs were successful/skipped and building clp was successful. - if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" - needs: "ubuntu-jammy-binaries" - runs-on: "ubuntu-latest" - env: - OS_NAME: "ubuntu-jammy" - TMP_OUTPUT_DIR: "/tmp" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - - - name: "Untar binaries" - working-directory: >- - ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} - run: |- - tar xf clp.tar - rm clp.tar - - - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" - with: - registry: "ghcr.io" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" - - - name: "Sanitize the repo's name" - id: "sanitize_repo_name" - run: |- - # Docker doesn't support repository names with uppercase characters, so we convert to - # lowercase here. - lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') - echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" - shell: "bash" - - - id: "core_image_meta" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: >- - ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} - tags: "type=raw,value=${{github.ref_name}}" - - # Only publish the image if this workflow was triggered by a push to `main`. - # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. - - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" - push: true - tags: "${{steps.core_image_meta.outputs.tags}}" - labels: "${{steps.core_image_meta.outputs.labels}}" - - ubuntu-jammy-lint: - name: "ubuntu-jammy-lint" - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. - fetch-depth: 0 - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full - # on all files. - - if: "'schedule' != github.event_name" - name: "Restore lint:check-cpp-static-full cache" - id: "cache-restore-lint-check-cpp-static-full" - uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - - # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy - # violations. - key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. - # run_command: >- - # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) - task lint:check-cpp-full - - # Cache the source file checksums and the generated files (logs) for the - # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. - # NOTE: If we don't cache the generated files, the task will re-run to generate them. - - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" - name: "Update lint:check-cpp-static-full cache" - uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml deleted file mode 100644 index 82fc4a0e90..0000000000 --- a/.github/workflows/clp-docs.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: "clp-docs" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - build: - name: "build" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - lfs: "true" - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.10" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Build docs" - shell: "bash" - run: "task docs:site" - - # Upload the built docs so we can download and deploy them from y-scope/yscope-docs - - if: >- - contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) - && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) - && 'ubuntu-latest' == matrix.os - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "docs-html" - path: "build/docs/html" - if-no-files-found: "error" - - # Retain the artifact for a week in case there was a deployment issue - retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml deleted file mode 100644 index 17789906e8..0000000000 --- a/.github/workflows/clp-execution-image-build.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: "clp-execution-image-build" - -on: - pull_request: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - push: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - base: "main" - filters: | - ubuntu_jammy_image: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" - - ubuntu-jammy-execution-image: - name: "ubuntu-jammy-execution-image" - if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "./.github/actions/clp-execution-image-build" - with: - image_registry: "ghcr.io" - image_registry_username: "${{github.actor}}" - image_registry_password: "${{secrets.GITHUB_TOKEN}}" - platform_id: "ubuntu" - platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml deleted file mode 100644 index 5f800d8c23..0000000000 --- a/.github/workflows/clp-lint.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-lint" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - lint-check: - name: "lint-check" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.11" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Run lint task" - shell: "bash" - run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml deleted file mode 100644 index 0aa1533a98..0000000000 --- a/.github/workflows/clp-pr-title-checks.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: "clp-pr-title-checks" - -on: - pull_request_target: - # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the - # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: - # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the - # pull request triggered by this event. - # - Each job has `permissions` set to only those necessary. - types: ["edited", "opened", "reopened"] - branches: ["main"] - -permissions: {} - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - conventional-commits: - name: "conventional-commits" - permissions: - # For amannn/action-semantic-pull-request - pull-requests: "read" - runs-on: "ubuntu-latest" - steps: - - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" - env: - GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml deleted file mode 100644 index e605db9005..0000000000 --- a/.github/workflows/clp-s-generated-code-checks.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-s-generated-code-checks" - -on: - pull_request: - push: - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - antlr-code-committed: - name: "antlr-code-committed" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Check if the generated parsers are the latest" - shell: "bash" - run: - | - git status --porcelain \ - components/core/src/clp_s/search/kql/generated \ - components/core/src/clp_s/search/sql/generated \ - | grep . > /dev/null \ - && exit 1 \ - || exit 0 From 146dc62eccaacd814be5648e40590a59b84709c9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 14:16:14 -0400 Subject: [PATCH 36/47] Add GITHUB_ENV check --- .../scripts/lib_install/macos/install-all.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index b467152ff1..5aeea87de8 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -48,15 +48,16 @@ brew install \ xz \ zstd -echo "$GITHUB_ENV" -LLVM_PREFIX=$(brew --prefix llvm@16) -{ - echo "LLVM_PREFIX=$LLVM_PREFIX" - echo "CC=$LLVM_PREFIX/bin/clang" - echo "CXX=$LLVM_PREFIX/bin/clang++" - echo "AR=$LLVM_PREFIX/bin/llvm-ar" - echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" -} >> "$GITHUB_ENV" +if [[ -n "${GITHUB_ENV}" ]]; then + LLVM_PREFIX=$(brew --prefix llvm@16) + { + echo "LLVM_PREFIX=$LLVM_PREFIX" + echo "CC=$LLVM_PREFIX/bin/clang" + echo "CXX=$LLVM_PREFIX/bin/clang++" + echo "AR=$LLVM_PREFIX/bin/llvm-ar" + echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" + } >> "$GITHUB_ENV" +fi # Install pkg-config if it isn't already installed # NOTE: We might expect that pkg-config is installed through brew, so trying to install it again From d7ff1cbf886407853f5f9935a8fed0f3271842d3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 14:21:13 -0400 Subject: [PATCH 37/47] Revert "Temporarily remove all other workflows" This reverts commit a2220741a26d4d90ae2a4e94f8f91f01a667ac66. --- .github/workflows/clp-core-build.yaml | 376 ++++++++++++++++++ .github/workflows/clp-docs.yaml | 57 +++ .../workflows/clp-execution-image-build.yaml | 70 ++++ .github/workflows/clp-lint.yaml | 42 ++ .github/workflows/clp-pr-title-checks.yaml | 31 ++ .../clp-s-generated-code-checks.yaml | 42 ++ 6 files changed, 618 insertions(+) create mode 100644 .github/workflows/clp-core-build.yaml create mode 100644 .github/workflows/clp-docs.yaml create mode 100644 .github/workflows/clp-execution-image-build.yaml create mode 100644 .github/workflows/clp-lint.yaml create mode 100644 .github/workflows/clp-pr-title-checks.yaml create mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml new file mode 100644 index 0000000000..ad0ea8fbf2 --- /dev/null +++ b/.github/workflows/clp-core-build.yaml @@ -0,0 +1,376 @@ +name: "clp-core-build" + +on: + pull_request: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + push: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +env: + BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" + DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted + # publishing of container images. + cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + clp_changed: "${{steps.filter.outputs.clp}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + # Consider changes between the current commit and `main` + # NOTE: If a pull request changes one of the images, then we need to (1) build the image + # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull + # request doesn't change an image, then we don't need to rebuild the image; instead we can + # use the published image which is based on `main`. So when determining what files have + # changed, we need to consider the delta between the current commit and `main` (rather + # than the current and previous commits) in order to detect if we need to rebuild the + # image (since it would be different from the published image). + base: "main" + filters: | + centos_stream_9_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" + - "components/core/tools/scripts/lib_install/centos-stream-9/**" + ubuntu_jammy_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" + - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" + clp: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/cmake/**" + - "components/core/CMakeLists.txt" + - "components/core/src/**" + - "components/core/tests/**" + - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + + centos-stream-9-deps-image: + name: "centos-stream-9-deps-image" + if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + ubuntu-jammy-deps-image: + name: "ubuntu-jammy-deps-image" + if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + centos-stream-9-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "centos-stream-9-deps-image" + - "filter-relevant-changes" + strategy: + matrix: + use_shared_libs: [true, false] + name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + ubuntu-jammy-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + strategy: + matrix: + include: + - use_shared_libs: true + upload_binaries: false + - use_shared_libs: false + upload_binaries: true + env: + OS_NAME: "ubuntu-jammy" + name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + - if: "matrix.upload_binaries == true" + id: "copy_binaries" + run: |- + output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + mkdir -p "${output_dir}" + cd "$GITHUB_WORKSPACE/components/core/build" + tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable + shell: "bash" + + - if: "matrix.upload_binaries == true" + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{steps.copy_binaries.outputs.output_dir}}" + retention-days: 1 + + ubuntu-jammy-binaries-image: + name: "ubuntu-jammy-binaries-image" + # Run if the ancestor jobs were successful/skipped and building clp was successful. + if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" + needs: "ubuntu-jammy-binaries" + runs-on: "ubuntu-latest" + env: + OS_NAME: "ubuntu-jammy" + TMP_OUTPUT_DIR: "/tmp" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + + - name: "Untar binaries" + working-directory: >- + ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} + run: |- + tar xf clp.tar + rm clp.tar + + - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Sanitize the repo's name" + id: "sanitize_repo_name" + run: |- + # Docker doesn't support repository names with uppercase characters, so we convert to + # lowercase here. + lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') + echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" + shell: "bash" + + - id: "core_image_meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} + tags: "type=raw,value=${{github.ref_name}}" + + # Only publish the image if this workflow was triggered by a push to `main`. + # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. + - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" + push: true + tags: "${{steps.core_image_meta.outputs.tags}}" + labels: "${{steps.core_image_meta.outputs.labels}}" + + ubuntu-jammy-lint: + name: "ubuntu-jammy-lint" + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. + fetch-depth: 0 + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full + # on all files. + - if: "'schedule' != github.event_name" + name: "Restore lint:check-cpp-static-full cache" + id: "cache-restore-lint-check-cpp-static-full" + uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + + # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy + # violations. + key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. + # run_command: >- + # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + task lint:check-cpp-full + + # Cache the source file checksums and the generated files (logs) for the + # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. + # NOTE: If we don't cache the generated files, the task will re-run to generate them. + - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" + name: "Update lint:check-cpp-static-full cache" + uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml new file mode 100644 index 0000000000..82fc4a0e90 --- /dev/null +++ b/.github/workflows/clp-docs.yaml @@ -0,0 +1,57 @@ +name: "clp-docs" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + build: + name: "build" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + lfs: "true" + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.10" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Build docs" + shell: "bash" + run: "task docs:site" + + # Upload the built docs so we can download and deploy them from y-scope/yscope-docs + - if: >- + contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) + && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) + && 'ubuntu-latest' == matrix.os + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "docs-html" + path: "build/docs/html" + if-no-files-found: "error" + + # Retain the artifact for a week in case there was a deployment issue + retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml new file mode 100644 index 0000000000..17789906e8 --- /dev/null +++ b/.github/workflows/clp-execution-image-build.yaml @@ -0,0 +1,70 @@ +name: "clp-execution-image-build" + +on: + pull_request: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + push: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + base: "main" + filters: | + ubuntu_jammy_image: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" + + ubuntu-jammy-execution-image: + name: "ubuntu-jammy-execution-image" + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - uses: "./.github/actions/clp-execution-image-build" + with: + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml new file mode 100644 index 0000000000..5f800d8c23 --- /dev/null +++ b/.github/workflows/clp-lint.yaml @@ -0,0 +1,42 @@ +name: "clp-lint" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + lint-check: + name: "lint-check" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.11" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Run lint task" + shell: "bash" + run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml new file mode 100644 index 0000000000..0aa1533a98 --- /dev/null +++ b/.github/workflows/clp-pr-title-checks.yaml @@ -0,0 +1,31 @@ +name: "clp-pr-title-checks" + +on: + pull_request_target: + # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the + # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: + # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the + # pull request triggered by this event. + # - Each job has `permissions` set to only those necessary. + types: ["edited", "opened", "reopened"] + branches: ["main"] + +permissions: {} + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + conventional-commits: + name: "conventional-commits" + permissions: + # For amannn/action-semantic-pull-request + pull-requests: "read" + runs-on: "ubuntu-latest" + steps: + - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" + env: + GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml new file mode 100644 index 0000000000..e605db9005 --- /dev/null +++ b/.github/workflows/clp-s-generated-code-checks.yaml @@ -0,0 +1,42 @@ +name: "clp-s-generated-code-checks" + +on: + pull_request: + push: + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + antlr-code-committed: + name: "antlr-code-committed" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Check if the generated parsers are the latest" + shell: "bash" + run: + | + git status --porcelain \ + components/core/src/clp_s/search/kql/generated \ + components/core/src/clp_s/search/sql/generated \ + | grep . > /dev/null \ + && exit 1 \ + || exit 0 From 2bbec2df843ba9a262ced140598e9dd09467b635 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 18:51:45 -0400 Subject: [PATCH 38/47] Use GITHUB_ACTIONS to check --- .../scripts/lib_install/macos/install-all.sh | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 5aeea87de8..4df699210d 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -18,8 +18,7 @@ curl \ --location \ --output "$cmake_formula_path" \ --show-error \ - https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce\ -/Formula/c/cmake.rb + https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce/Formula/c/cmake.rb brew install --formula "$cmake_formula_path" # Install a version of `task` < 3.43 to avoid https://github.com/y-scope/clp/issues/872 @@ -29,8 +28,7 @@ curl \ --location \ --output "$task_formula_path" \ --show-error \ - https://raw.githubusercontent.com/Homebrew/homebrew-core/356f8408263b6a06e8f5f83cad574773d8054e1c\ -/Formula/g/go-task.rb + https://raw.githubusercontent.com/Homebrew/homebrew-core/356f8408263b6a06e8f5f83cad574773d8054e1c/Formula/g/go-task.rb brew install --formula "$task_formula_path" rm -rf "$formula_dir" @@ -48,7 +46,7 @@ brew install \ xz \ zstd -if [[ -n "${GITHUB_ENV}" ]]; then +if [ "${GITHUB_ACTIONS:-}" == "true" ]; then LLVM_PREFIX=$(brew --prefix llvm@16) { echo "LLVM_PREFIX=$LLVM_PREFIX" @@ -56,17 +54,17 @@ if [[ -n "${GITHUB_ENV}" ]]; then echo "CXX=$LLVM_PREFIX/bin/clang++" echo "AR=$LLVM_PREFIX/bin/llvm-ar" echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" - } >> "$GITHUB_ENV" + } >>"$GITHUB_ENV" fi # Install pkg-config if it isn't already installed # NOTE: We might expect that pkg-config is installed through brew, so trying to install it again # would be harmless; however, in certain environments, like the macOS GitHub hosted runner, # pkg-config is installed by other means, meaning a brew install would cause conflicts. -if ! command -v pkg-config ; then - brew install pkg-config +if ! command -v pkg-config; then + brew install pkg-config fi # TODO: https://github.com/y-scope/clp/issues/795 -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" "${script_dir}/../check-cmake-version.sh" From 9cf5f4d89fdb491b4cc234035b005f368ce3d1d2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 19:03:39 -0400 Subject: [PATCH 39/47] Revert "Use GITHUB_ACTIONS to check" This reverts commit 2bbec2df843ba9a262ced140598e9dd09467b635. --- .../scripts/lib_install/macos/install-all.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 4df699210d..5aeea87de8 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -18,7 +18,8 @@ curl \ --location \ --output "$cmake_formula_path" \ --show-error \ - https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce/Formula/c/cmake.rb + https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce\ +/Formula/c/cmake.rb brew install --formula "$cmake_formula_path" # Install a version of `task` < 3.43 to avoid https://github.com/y-scope/clp/issues/872 @@ -28,7 +29,8 @@ curl \ --location \ --output "$task_formula_path" \ --show-error \ - https://raw.githubusercontent.com/Homebrew/homebrew-core/356f8408263b6a06e8f5f83cad574773d8054e1c/Formula/g/go-task.rb + https://raw.githubusercontent.com/Homebrew/homebrew-core/356f8408263b6a06e8f5f83cad574773d8054e1c\ +/Formula/g/go-task.rb brew install --formula "$task_formula_path" rm -rf "$formula_dir" @@ -46,7 +48,7 @@ brew install \ xz \ zstd -if [ "${GITHUB_ACTIONS:-}" == "true" ]; then +if [[ -n "${GITHUB_ENV}" ]]; then LLVM_PREFIX=$(brew --prefix llvm@16) { echo "LLVM_PREFIX=$LLVM_PREFIX" @@ -54,17 +56,17 @@ if [ "${GITHUB_ACTIONS:-}" == "true" ]; then echo "CXX=$LLVM_PREFIX/bin/clang++" echo "AR=$LLVM_PREFIX/bin/llvm-ar" echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" - } >>"$GITHUB_ENV" + } >> "$GITHUB_ENV" fi # Install pkg-config if it isn't already installed # NOTE: We might expect that pkg-config is installed through brew, so trying to install it again # would be harmless; however, in certain environments, like the macOS GitHub hosted runner, # pkg-config is installed by other means, meaning a brew install would cause conflicts. -if ! command -v pkg-config; then - brew install pkg-config +if ! command -v pkg-config ; then + brew install pkg-config fi # TODO: https://github.com/y-scope/clp/issues/795 -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/../check-cmake-version.sh" From b49789fb9a8b979ea5d1535b2b050ac419f974ce Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 19:06:01 -0400 Subject: [PATCH 40/47] Fix the spacing and shell equal --- components/core/tools/scripts/lib_install/macos/install-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 5aeea87de8..ee67fd8f11 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -48,7 +48,7 @@ brew install \ xz \ zstd -if [[ -n "${GITHUB_ENV}" ]]; then +if [ "${GITHUB_ACTIONS:-}" = "true" ]; then LLVM_PREFIX=$(brew --prefix llvm@16) { echo "LLVM_PREFIX=$LLVM_PREFIX" From 893aa84e797ab29455aea9c259aa3e673598ca2a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 19:24:29 -0400 Subject: [PATCH 41/47] Put constant first when checking variables in shell script --- components/core/tools/scripts/lib_install/macos/install-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index ee67fd8f11..d7f4ae9ad7 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -48,7 +48,7 @@ brew install \ xz \ zstd -if [ "${GITHUB_ACTIONS:-}" = "true" ]; then +if [ "true" = "${GITHUB_ACTIONS:-}" ]; then LLVM_PREFIX=$(brew --prefix llvm@16) { echo "LLVM_PREFIX=$LLVM_PREFIX" From c95039ccb39acdc6ab0620fa2c8eafb284f7a53d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 22:48:25 -0400 Subject: [PATCH 42/47] Temporarily remove other GH workflows --- .github/workflows/clp-core-build.yaml | 376 ------------------ .github/workflows/clp-docs.yaml | 57 --- .../workflows/clp-execution-image-build.yaml | 70 ---- .github/workflows/clp-lint.yaml | 42 -- .github/workflows/clp-pr-title-checks.yaml | 31 -- .../clp-s-generated-code-checks.yaml | 42 -- 6 files changed, 618 deletions(-) delete mode 100644 .github/workflows/clp-core-build.yaml delete mode 100644 .github/workflows/clp-docs.yaml delete mode 100644 .github/workflows/clp-execution-image-build.yaml delete mode 100644 .github/workflows/clp-lint.yaml delete mode 100644 .github/workflows/clp-pr-title-checks.yaml delete mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml deleted file mode 100644 index ad0ea8fbf2..0000000000 --- a/.github/workflows/clp-core-build.yaml +++ /dev/null @@ -1,376 +0,0 @@ -name: "clp-core-build" - -on: - pull_request: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - push: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -env: - BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted - # publishing of container images. - cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - clp_changed: "${{steps.filter.outputs.clp}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - # Consider changes between the current commit and `main` - # NOTE: If a pull request changes one of the images, then we need to (1) build the image - # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull - # request doesn't change an image, then we don't need to rebuild the image; instead we can - # use the published image which is based on `main`. So when determining what files have - # changed, we need to consider the delta between the current commit and `main` (rather - # than the current and previous commits) in order to detect if we need to rebuild the - # image (since it would be different from the published image). - base: "main" - filters: | - centos_stream_9_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - - "components/core/tools/scripts/lib_install/centos-stream-9/**" - ubuntu_jammy_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" - clp: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/cmake/**" - - "components/core/CMakeLists.txt" - - "components/core/src/**" - - "components/core/tests/**" - - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - centos-stream-9-deps-image: - name: "centos-stream-9-deps-image" - if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - ubuntu-jammy-deps-image: - name: "ubuntu-jammy-deps-image" - if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - centos-stream-9-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "centos-stream-9-deps-image" - - "filter-relevant-changes" - strategy: - matrix: - use_shared_libs: [true, false] - name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - ubuntu-jammy-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - strategy: - matrix: - include: - - use_shared_libs: true - upload_binaries: false - - use_shared_libs: false - upload_binaries: true - env: - OS_NAME: "ubuntu-jammy" - name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - - if: "matrix.upload_binaries == true" - id: "copy_binaries" - run: |- - output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - - mkdir -p "${output_dir}" - cd "$GITHUB_WORKSPACE/components/core/build" - tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable - shell: "bash" - - - if: "matrix.upload_binaries == true" - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{steps.copy_binaries.outputs.output_dir}}" - retention-days: 1 - - ubuntu-jammy-binaries-image: - name: "ubuntu-jammy-binaries-image" - # Run if the ancestor jobs were successful/skipped and building clp was successful. - if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" - needs: "ubuntu-jammy-binaries" - runs-on: "ubuntu-latest" - env: - OS_NAME: "ubuntu-jammy" - TMP_OUTPUT_DIR: "/tmp" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - - - name: "Untar binaries" - working-directory: >- - ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} - run: |- - tar xf clp.tar - rm clp.tar - - - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" - with: - registry: "ghcr.io" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" - - - name: "Sanitize the repo's name" - id: "sanitize_repo_name" - run: |- - # Docker doesn't support repository names with uppercase characters, so we convert to - # lowercase here. - lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') - echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" - shell: "bash" - - - id: "core_image_meta" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: >- - ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} - tags: "type=raw,value=${{github.ref_name}}" - - # Only publish the image if this workflow was triggered by a push to `main`. - # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. - - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" - push: true - tags: "${{steps.core_image_meta.outputs.tags}}" - labels: "${{steps.core_image_meta.outputs.labels}}" - - ubuntu-jammy-lint: - name: "ubuntu-jammy-lint" - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. - fetch-depth: 0 - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full - # on all files. - - if: "'schedule' != github.event_name" - name: "Restore lint:check-cpp-static-full cache" - id: "cache-restore-lint-check-cpp-static-full" - uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - - # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy - # violations. - key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. - # run_command: >- - # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) - task lint:check-cpp-full - - # Cache the source file checksums and the generated files (logs) for the - # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. - # NOTE: If we don't cache the generated files, the task will re-run to generate them. - - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" - name: "Update lint:check-cpp-static-full cache" - uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml deleted file mode 100644 index 82fc4a0e90..0000000000 --- a/.github/workflows/clp-docs.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: "clp-docs" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - build: - name: "build" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - lfs: "true" - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.10" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Build docs" - shell: "bash" - run: "task docs:site" - - # Upload the built docs so we can download and deploy them from y-scope/yscope-docs - - if: >- - contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) - && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) - && 'ubuntu-latest' == matrix.os - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "docs-html" - path: "build/docs/html" - if-no-files-found: "error" - - # Retain the artifact for a week in case there was a deployment issue - retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml deleted file mode 100644 index 17789906e8..0000000000 --- a/.github/workflows/clp-execution-image-build.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: "clp-execution-image-build" - -on: - pull_request: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - push: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - base: "main" - filters: | - ubuntu_jammy_image: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" - - ubuntu-jammy-execution-image: - name: "ubuntu-jammy-execution-image" - if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "./.github/actions/clp-execution-image-build" - with: - image_registry: "ghcr.io" - image_registry_username: "${{github.actor}}" - image_registry_password: "${{secrets.GITHUB_TOKEN}}" - platform_id: "ubuntu" - platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml deleted file mode 100644 index 5f800d8c23..0000000000 --- a/.github/workflows/clp-lint.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-lint" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - lint-check: - name: "lint-check" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.11" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Run lint task" - shell: "bash" - run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml deleted file mode 100644 index 0aa1533a98..0000000000 --- a/.github/workflows/clp-pr-title-checks.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: "clp-pr-title-checks" - -on: - pull_request_target: - # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the - # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: - # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the - # pull request triggered by this event. - # - Each job has `permissions` set to only those necessary. - types: ["edited", "opened", "reopened"] - branches: ["main"] - -permissions: {} - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - conventional-commits: - name: "conventional-commits" - permissions: - # For amannn/action-semantic-pull-request - pull-requests: "read" - runs-on: "ubuntu-latest" - steps: - - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" - env: - GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml deleted file mode 100644 index e605db9005..0000000000 --- a/.github/workflows/clp-s-generated-code-checks.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-s-generated-code-checks" - -on: - pull_request: - push: - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - antlr-code-committed: - name: "antlr-code-committed" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Check if the generated parsers are the latest" - shell: "bash" - run: - | - git status --porcelain \ - components/core/src/clp_s/search/kql/generated \ - components/core/src/clp_s/search/sql/generated \ - | grep . > /dev/null \ - && exit 1 \ - || exit 0 From b4665a574ba3878f146865c905fe402cfcffc422 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 22:51:32 -0400 Subject: [PATCH 43/47] Try unset GITHUB_ENV --- .github/workflows/clp-core-build-macos.yaml | 10 ++++++++++ taskfiles/lint.yaml | 2 -- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 51eb6985d0..3fa8c97c4b 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -105,6 +105,16 @@ jobs: # violations. key: "main-branch-${{matrix.os}}-lint:check-cpp-static-full" + - run: >- + { + echo "LLVM_PREFIX=" + echo "CC=" + echo "CXX=" + echo "AR=" + echo "RANLIB=" + } >> "$GITHUB_ENV" + shell: "bash" + # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. # - run: >- # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index b9eb7b4d7e..149a1ba5f8 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -159,7 +159,6 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "--extra-arg=-Drsize_t=size_t" OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" @@ -747,7 +746,6 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "--extra-arg=-Drsize_t=size_t" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From 618d297c1c282cf2efc56e7b59d7d9d6982fca67 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 23:45:48 -0400 Subject: [PATCH 44/47] Revert "Temporarily remove other GH workflows" This reverts commit c95039ccb39acdc6ab0620fa2c8eafb284f7a53d. --- .github/workflows/clp-core-build.yaml | 376 ++++++++++++++++++ .github/workflows/clp-docs.yaml | 57 +++ .../workflows/clp-execution-image-build.yaml | 70 ++++ .github/workflows/clp-lint.yaml | 42 ++ .github/workflows/clp-pr-title-checks.yaml | 31 ++ .../clp-s-generated-code-checks.yaml | 42 ++ 6 files changed, 618 insertions(+) create mode 100644 .github/workflows/clp-core-build.yaml create mode 100644 .github/workflows/clp-docs.yaml create mode 100644 .github/workflows/clp-execution-image-build.yaml create mode 100644 .github/workflows/clp-lint.yaml create mode 100644 .github/workflows/clp-pr-title-checks.yaml create mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml new file mode 100644 index 0000000000..ad0ea8fbf2 --- /dev/null +++ b/.github/workflows/clp-core-build.yaml @@ -0,0 +1,376 @@ +name: "clp-core-build" + +on: + pull_request: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + push: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +env: + BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" + DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted + # publishing of container images. + cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + clp_changed: "${{steps.filter.outputs.clp}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + # Consider changes between the current commit and `main` + # NOTE: If a pull request changes one of the images, then we need to (1) build the image + # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull + # request doesn't change an image, then we don't need to rebuild the image; instead we can + # use the published image which is based on `main`. So when determining what files have + # changed, we need to consider the delta between the current commit and `main` (rather + # than the current and previous commits) in order to detect if we need to rebuild the + # image (since it would be different from the published image). + base: "main" + filters: | + centos_stream_9_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" + - "components/core/tools/scripts/lib_install/centos-stream-9/**" + ubuntu_jammy_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" + - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" + clp: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/cmake/**" + - "components/core/CMakeLists.txt" + - "components/core/src/**" + - "components/core/tests/**" + - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + + centos-stream-9-deps-image: + name: "centos-stream-9-deps-image" + if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + ubuntu-jammy-deps-image: + name: "ubuntu-jammy-deps-image" + if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + centos-stream-9-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "centos-stream-9-deps-image" + - "filter-relevant-changes" + strategy: + matrix: + use_shared_libs: [true, false] + name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + ubuntu-jammy-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + strategy: + matrix: + include: + - use_shared_libs: true + upload_binaries: false + - use_shared_libs: false + upload_binaries: true + env: + OS_NAME: "ubuntu-jammy" + name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + - if: "matrix.upload_binaries == true" + id: "copy_binaries" + run: |- + output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + mkdir -p "${output_dir}" + cd "$GITHUB_WORKSPACE/components/core/build" + tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable + shell: "bash" + + - if: "matrix.upload_binaries == true" + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{steps.copy_binaries.outputs.output_dir}}" + retention-days: 1 + + ubuntu-jammy-binaries-image: + name: "ubuntu-jammy-binaries-image" + # Run if the ancestor jobs were successful/skipped and building clp was successful. + if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" + needs: "ubuntu-jammy-binaries" + runs-on: "ubuntu-latest" + env: + OS_NAME: "ubuntu-jammy" + TMP_OUTPUT_DIR: "/tmp" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + + - name: "Untar binaries" + working-directory: >- + ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} + run: |- + tar xf clp.tar + rm clp.tar + + - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Sanitize the repo's name" + id: "sanitize_repo_name" + run: |- + # Docker doesn't support repository names with uppercase characters, so we convert to + # lowercase here. + lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') + echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" + shell: "bash" + + - id: "core_image_meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} + tags: "type=raw,value=${{github.ref_name}}" + + # Only publish the image if this workflow was triggered by a push to `main`. + # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. + - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" + push: true + tags: "${{steps.core_image_meta.outputs.tags}}" + labels: "${{steps.core_image_meta.outputs.labels}}" + + ubuntu-jammy-lint: + name: "ubuntu-jammy-lint" + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. + fetch-depth: 0 + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full + # on all files. + - if: "'schedule' != github.event_name" + name: "Restore lint:check-cpp-static-full cache" + id: "cache-restore-lint-check-cpp-static-full" + uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + + # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy + # violations. + key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. + # run_command: >- + # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + task lint:check-cpp-full + + # Cache the source file checksums and the generated files (logs) for the + # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. + # NOTE: If we don't cache the generated files, the task will re-run to generate them. + - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" + name: "Update lint:check-cpp-static-full cache" + uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml new file mode 100644 index 0000000000..82fc4a0e90 --- /dev/null +++ b/.github/workflows/clp-docs.yaml @@ -0,0 +1,57 @@ +name: "clp-docs" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + build: + name: "build" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + lfs: "true" + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.10" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Build docs" + shell: "bash" + run: "task docs:site" + + # Upload the built docs so we can download and deploy them from y-scope/yscope-docs + - if: >- + contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) + && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) + && 'ubuntu-latest' == matrix.os + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "docs-html" + path: "build/docs/html" + if-no-files-found: "error" + + # Retain the artifact for a week in case there was a deployment issue + retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml new file mode 100644 index 0000000000..17789906e8 --- /dev/null +++ b/.github/workflows/clp-execution-image-build.yaml @@ -0,0 +1,70 @@ +name: "clp-execution-image-build" + +on: + pull_request: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + push: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + base: "main" + filters: | + ubuntu_jammy_image: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" + + ubuntu-jammy-execution-image: + name: "ubuntu-jammy-execution-image" + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - uses: "./.github/actions/clp-execution-image-build" + with: + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml new file mode 100644 index 0000000000..5f800d8c23 --- /dev/null +++ b/.github/workflows/clp-lint.yaml @@ -0,0 +1,42 @@ +name: "clp-lint" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + lint-check: + name: "lint-check" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.11" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Run lint task" + shell: "bash" + run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml new file mode 100644 index 0000000000..0aa1533a98 --- /dev/null +++ b/.github/workflows/clp-pr-title-checks.yaml @@ -0,0 +1,31 @@ +name: "clp-pr-title-checks" + +on: + pull_request_target: + # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the + # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: + # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the + # pull request triggered by this event. + # - Each job has `permissions` set to only those necessary. + types: ["edited", "opened", "reopened"] + branches: ["main"] + +permissions: {} + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + conventional-commits: + name: "conventional-commits" + permissions: + # For amannn/action-semantic-pull-request + pull-requests: "read" + runs-on: "ubuntu-latest" + steps: + - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" + env: + GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml new file mode 100644 index 0000000000..e605db9005 --- /dev/null +++ b/.github/workflows/clp-s-generated-code-checks.yaml @@ -0,0 +1,42 @@ +name: "clp-s-generated-code-checks" + +on: + pull_request: + push: + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + antlr-code-committed: + name: "antlr-code-committed" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Check if the generated parsers are the latest" + shell: "bash" + run: + | + git status --porcelain \ + components/core/src/clp_s/search/kql/generated \ + components/core/src/clp_s/search/sql/generated \ + | grep . > /dev/null \ + && exit 1 \ + || exit 0 From 2516396dc9ea6657c4899368de44c1f46da5309f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 23:47:11 -0400 Subject: [PATCH 45/47] Name the workflow step to unset env vars --- .github/workflows/clp-core-build-macos.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 3fa8c97c4b..f6e0dc9955 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -105,7 +105,9 @@ jobs: # violations. key: "main-branch-${{matrix.os}}-lint:check-cpp-static-full" - - run: >- + - name: "Unset LLVM environment variables" + shell: "bash" + run: >- { echo "LLVM_PREFIX=" echo "CC=" @@ -113,7 +115,6 @@ jobs: echo "AR=" echo "RANLIB=" } >> "$GITHUB_ENV" - shell: "bash" # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. # - run: >- From 6ddb51d5208fd0b71deef2bf3b22f9819989cb14 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 9 Jul 2025 12:54:21 -0400 Subject: [PATCH 46/47] Move env var to GH workflow steps --- .github/workflows/clp-core-build-macos.yaml | 15 +++++++++++++++ .../scripts/lib_install/macos/install-all.sh | 11 ----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index f6e0dc9955..63cea59a44 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -74,6 +74,21 @@ jobs: - run: "./tools/scripts/deps-download/init.sh" shell: "bash" + # NOTE: We set up the environment variables for LLVM so that the dependencies and CLP core + # are built with clang/clang++ from homebrew installed LLVM. Need to unset for clang-tidy. + # See https://github.com/y-scope/clp/issues/1080 for details and exploration of solutions. + - name: "Set up environment variables for LLVM" + shell: "bash" + run: >- + LLVM_PREFIX=$(brew --prefix llvm@16) + { + echo "LLVM_PREFIX=$LLVM_PREFIX" + echo "CC=$LLVM_PREFIX/bin/clang" + echo "CXX=$LLVM_PREFIX/bin/clang++" + echo "AR=$LLVM_PREFIX/bin/llvm-ar" + echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" + } >> "$GITHUB_ENV" + - run: "CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core" shell: "bash" diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index d7f4ae9ad7..57d77af764 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -48,17 +48,6 @@ brew install \ xz \ zstd -if [ "true" = "${GITHUB_ACTIONS:-}" ]; then - LLVM_PREFIX=$(brew --prefix llvm@16) - { - echo "LLVM_PREFIX=$LLVM_PREFIX" - echo "CC=$LLVM_PREFIX/bin/clang" - echo "CXX=$LLVM_PREFIX/bin/clang++" - echo "AR=$LLVM_PREFIX/bin/llvm-ar" - echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" - } >> "$GITHUB_ENV" -fi - # Install pkg-config if it isn't already installed # NOTE: We might expect that pkg-config is installed through brew, so trying to install it again # would be harmless; however, in certain environments, like the macOS GitHub hosted runner, From dcbf3625a6057b37e96c98ed27033d188e8efbca Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 9 Jul 2025 12:58:00 -0400 Subject: [PATCH 47/47] Bug fix --- .github/workflows/clp-core-build-macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 63cea59a44..e0f11419ff 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -79,7 +79,7 @@ jobs: # See https://github.com/y-scope/clp/issues/1080 for details and exploration of solutions. - name: "Set up environment variables for LLVM" shell: "bash" - run: >- + run: | LLVM_PREFIX=$(brew --prefix llvm@16) { echo "LLVM_PREFIX=$LLVM_PREFIX"