From 35c08137d82e5f8ecfe0089d421e3648cde98dd7 Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 15 Jun 2026 15:10:00 +0000 Subject: [PATCH 01/38] Add workflow for integration tests. --- .github/workflows/clp-integration-tests.yaml | 43 ++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/clp-integration-tests.yaml diff --git a/.github/workflows/clp-integration-tests.yaml b/.github/workflows/clp-integration-tests.yaml new file mode 100644 index 0000000000..cd7e05aeda --- /dev/null +++ b/.github/workflows/clp-integration-tests.yaml @@ -0,0 +1,43 @@ +name: "clp-integration-tests" + +on: + pull_request: + paths-ignore: &ignored_paths + - ".github/*" + - ".github/ISSUE_TEMPLATE/**" + - "docs/**" + push: + paths-ignore: *ignored_paths + schedule: + + # Run daily at 00:30 UTC (the 30 is to avoid periods of high load) + - cron: "30 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency; Exclude the `main` branch. + cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" + +jobs: + integration-tests: + + # Runs only on Ubuntu Noble (24.04) because the integration test suite can't run on macOS. + runs-on: "ubuntu-24.04" + steps: + - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 + with: + submodules: "recursive" + + - uses: "./tools/yscope-dev-utils/exports/github/actions/install-go-task" + with: + version: "3.48.0" + + - uses: "./tools/yscope-dev-utils/exports/github/actions/install-uv" + with: + version: "0.8" + + - name: "Run integration tests" + shell: "bash" + run: "task tests:integration" From bb52e058e29e7bc2856d47968de6b82ea423c112 Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 15 Jun 2026 16:24:14 +0000 Subject: [PATCH 02/38] Run the integration tests inside the published dependencies image --- .github/workflows/clp-integration-tests.yaml | 34 ++++++++++++-------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/clp-integration-tests.yaml b/.github/workflows/clp-integration-tests.yaml index cd7e05aeda..0dbc1a68fe 100644 --- a/.github/workflows/clp-integration-tests.yaml +++ b/.github/workflows/clp-integration-tests.yaml @@ -14,30 +14,38 @@ on: - cron: "30 0 * * *" workflow_dispatch: +env: + DEPS_IMAGE_NAME_PREFIX_X86: "clp-core-dependencies-x86-" + OS_NAME: "ubuntu-jammy" + concurrency: group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency; Exclude the `main` branch. + # Cancel in-progress jobs for efficiency; exclude the `main` branch. cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" jobs: integration-tests: - - # Runs only on Ubuntu Noble (24.04) because the integration test suite can't run on macOS. - runs-on: "ubuntu-24.04" + name: "integration-tests" + runs-on: "ubuntu-22.04" steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 with: submodules: "recursive" - - uses: "./tools/yscope-dev-utils/exports/github/actions/install-go-task" - with: - version: "3.48.0" + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" - - uses: "./tools/yscope-dev-utils/exports/github/actions/install-uv" + # Run the integration tests inside the published `ubuntu-jammy` dependencies image. + - name: "Build package and run integration tests" + uses: "./.github/actions/run-on-image" with: - version: "0.8" - - - name: "Run integration tests" - shell: "bash" - run: "task tests:integration" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + + # Use the published `main` image. + use_published_image: "true" + run_command: >- + CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + HOME=/tmp + task tests:integration From 1605df7f2f96cd730aebf09ed6889cf40d291b7c Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 15 Jun 2026 20:23:52 +0000 Subject: [PATCH 03/38] Add integration-tests workflow to artifact-build instead of its own workflow. --- .github/workflows/clp-artifact-build.yaml | 34 +++++++++++++ .github/workflows/clp-integration-tests.yaml | 51 -------------------- 2 files changed, 34 insertions(+), 51 deletions(-) delete mode 100644 .github/workflows/clp-integration-tests.yaml diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 9aeb978d0e..a082452c37 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -677,6 +677,40 @@ jobs: 'build/lint-clang-tidy/**/*' )}} + ubuntu-jammy-integration-tests: + name: "ubuntu-jammy-integration-tests" + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-x86_64-deps-image" + runs-on: *runner + steps: + - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + # TODO: We set HOME due to permission issues inside the container used in run-on-image. + # See: https://github.com/y-scope/clp/issues/1145 + run_command: >- + CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + HOME=/tmp + task tests:integration + package-image: name: "package-image-${{matrix.arch}}" # NOTE: The aarch64 job is skipped on PRs, so we accept both 'success' and 'skipped'. diff --git a/.github/workflows/clp-integration-tests.yaml b/.github/workflows/clp-integration-tests.yaml deleted file mode 100644 index 0dbc1a68fe..0000000000 --- a/.github/workflows/clp-integration-tests.yaml +++ /dev/null @@ -1,51 +0,0 @@ -name: "clp-integration-tests" - -on: - pull_request: - paths-ignore: &ignored_paths - - ".github/*" - - ".github/ISSUE_TEMPLATE/**" - - "docs/**" - push: - paths-ignore: *ignored_paths - schedule: - - # Run daily at 00:30 UTC (the 30 is to avoid periods of high load) - - cron: "30 0 * * *" - workflow_dispatch: - -env: - DEPS_IMAGE_NAME_PREFIX_X86: "clp-core-dependencies-x86-" - OS_NAME: "ubuntu-jammy" - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency; exclude the `main` branch. - cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" - -jobs: - integration-tests: - name: "integration-tests" - runs-on: "ubuntu-22.04" - steps: - - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - # Run the integration tests inside the published `ubuntu-jammy` dependencies image. - - name: "Build package and run integration tests" - uses: "./.github/actions/run-on-image" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" - - # Use the published `main` image. - use_published_image: "true" - run_command: >- - CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) - HOME=/tmp - task tests:integration From bb47ca899f3900ce6a6db43b7676b7c6aac4d5c7 Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 22 Jun 2026 12:04:32 -0400 Subject: [PATCH 04/38] Trying something --- .github/workflows/clp-artifact-build.yaml | 105 ++++++++++++++++++---- taskfile.yaml | 11 ++- 2 files changed, 96 insertions(+), 20 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index a082452c37..279afc6f8d 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -679,13 +679,11 @@ jobs: ubuntu-jammy-integration-tests: name: "ubuntu-jammy-integration-tests" - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + # Run if `package-image` built successfully, reusing its build artifact rather than rebuilding + # the package here. + if: "!cancelled() && !failure() && needs.package-image.result == 'success'" needs: - - "filter-relevant-changes" - - "ubuntu-jammy-x86_64-deps-image" + - "package-image" runs-on: *runner steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 @@ -696,20 +694,31 @@ jobs: run: "chown $(id -u):$(id -g) -R ." shell: "bash" - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" + - name: "Download the integration-test bundle" + uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - # TODO: We set HOME due to permission issues inside the container used in run-on-image. - # See: https://github.com/y-scope/clp/issues/1145 - run_command: >- - CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) - HOME=/tmp - task tests:integration + name: "clp-package-integration-bundle" + path: "/tmp/clp-package-integration-bundle" + + - name: "Load the runtime image and restore the build outputs" + shell: "bash" + run: |- + docker load --input "/tmp/clp-package-integration-bundle/clp-package-image.tar" + tar --extract \ + --file "/tmp/clp-package-integration-bundle/build.tar" \ + --directory "$GITHUB_WORKSPACE" + + - name: "Install uv" + shell: "bash" + run: |- + curl --fail --silent --show-error --location \ + "https://astral.sh/uv/0.8.0/install.sh" | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: "Run the integration tests" + working-directory: "integration-tests" + shell: "bash" + run: "uv run pytest" package-image: name: "package-image-${{matrix.arch}}" @@ -775,6 +784,64 @@ jobs: arch: "${{matrix.arch}}" platform_version_codename: "jammy" + # Build a local runtime image, assemble the package directory, and upload them (with the core + # binaries) as an artifact for the `ubuntu-jammy-integration-tests` job to consume. This lets + # the integration tests reuse this job's build instead of rebuilding the package. We only do + # this on amd64 since the integration tests run on x86_64. + - name: "Build the package image for the integration tests" + if: "matrix.arch == 'amd64'" + shell: "bash" + run: "./tools/docker-images/clp-package/build.sh" + + - name: "Assemble the package directory for the integration tests" + if: "matrix.arch == 'amd64'" + uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + # TODO: We set HOME due to permission issues inside the container used in run-on-image. + # See: https://github.com/y-scope/clp/issues/1145 + run_command: >- + HOME=/tmp + task package-assemble + + - name: "Stage the integration-test bundle" + if: "matrix.arch == 'amd64'" + id: "stage_integration_bundle" + shell: "bash" + run: |- + bundle_dir="/tmp/clp-package-integration-bundle" + echo "bundle_dir=${bundle_dir}" >> "$GITHUB_OUTPUT" + mkdir -p "${bundle_dir}" + + # Save the runtime image referenced by the assembled package's `clp-package-image.id`. + docker save "$(cat build/clp-package-image.id)" \ + --output "${bundle_dir}/clp-package-image.tar" + + # Bundle the core binaries and the assembled package directory. + tar --create --file "${bundle_dir}/build.tar" \ + --directory "$GITHUB_WORKSPACE" \ + build/core/clg \ + build/core/clo \ + build/core/clp \ + build/core/clp-s \ + build/core/indexer \ + build/core/log-converter \ + build/core/reducer-server \ + build/clp-package + + - name: "Upload the integration-test bundle" + if: "matrix.arch == 'amd64'" + uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 + with: + name: "clp-package-integration-bundle" + path: "${{steps.stage_integration_bundle.outputs.bundle_dir}}" + retention-days: 1 + package-image-multiarch-manifest: name: "package-image-multiarch-manifest" if: >- diff --git a/taskfile.yaml b/taskfile.yaml index 1f095c513b..6dc8ec6f3e 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -109,6 +109,16 @@ tasks: - "rm -f '{{.G_WEBUI_CHECKSUM_FILE}}'" package: + deps: + - "docker-images:package" + cmds: + - task: "package-assemble" + + # Assembles the package directory from already-built components and the package image's ID file. + # NOTE: This is split out from `package` so that callers which build the package image separately + # (e.g., the `package-image` CI job) can assemble the directory without rebuilding the image. It + # requires `{{.G_BUILD_DIR}}/clp-package-image.id` and the built components to already exist. + package-assemble: vars: CHECKSUM_FILE: "{{.G_PACKAGE_CHECKSUM_FILE}}" OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}" @@ -122,7 +132,6 @@ tasks: - "tools/deployment/package/**/*" generates: ["{{.CHECKSUM_FILE}}"] deps: - - "docker-images:package" - "generate-config-schemas" - task: "utils:checksum:validate" vars: From d4ac37600fca8cc0a22ee0cdbd36d1706edbb6f1 Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 6 Jul 2026 15:27:29 +0000 Subject: [PATCH 05/38] idk --- .github/workflows/clp-artifact-build.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 279afc6f8d..7875c4e1d1 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -715,6 +715,14 @@ jobs: "https://astral.sh/uv/0.8.0/install.sh" | sh echo "$HOME/.local/bin" >> "$GITHUB_PATH" + # The `mariadb` Python driver (a transitive dependency of `job-orchestration`) is only + # published as a source distribution and must compile against MariaDB Connector/C, which + # provides the `mariadb_config` helper. Install it so `uv` can build the driver. The tests run + # directly on the runner (not in a container) so `start-clp` can reach the host Docker daemon. + - name: "Install the MariaDB Connector/C development files" + shell: "bash" + run: "sudo apt-get update && sudo apt-get install --yes libmariadb-dev" + - name: "Run the integration tests" working-directory: "integration-tests" shell: "bash" From de7a3d5cb20d75a2486b46b65c309e324ead2897 Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 6 Jul 2026 17:59:18 +0000 Subject: [PATCH 06/38] idk2 --- .github/workflows/clp-artifact-build.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 7875c4e1d1..d35aa53d67 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -728,6 +728,29 @@ jobs: shell: "bash" run: "uv run pytest" + # Print the captured subprocess logs directly into the run output so failures are visible + # without downloading an artifact. + - name: "Dump the integration-test subprocess logs on failure" + if: "failure()" + shell: "bash" + run: |- + shopt -s nullglob + for log_file in build/integration_tests/test_logs/*/subprocess_output/*.log; do + echo "::group::${log_file}" + cat "${log_file}" + echo "::endgroup::" + done + + # Upload the per-run test logs (including the captured `start-clp.sh`/`stop-clp.sh` subprocess + # output) so failures can be diagnosed from the Actions run page without runner access. + - name: "Upload the integration-test logs" + if: "always()" + uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 + with: + name: "clp-integration-test-logs" + path: "build/integration_tests/test_logs" + retention-days: 3 + package-image: name: "package-image-${{matrix.arch}}" # NOTE: The aarch64 job is skipped on PRs, so we accept both 'success' and 'skipped'. From 397e4f1aeb2c22469e2962650e09604a3c08a0ca Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 6 Jul 2026 19:44:05 +0000 Subject: [PATCH 07/38] idk3 --- .github/workflows/clp-artifact-build.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index d35aa53d67..b1bb9cd631 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -824,6 +824,20 @@ jobs: shell: "bash" run: "./tools/docker-images/clp-package/build.sh" + # TEMPORARY DIAGNOSTIC: report the runtime image's Python version and the compiled + # `pydantic_core` extension baked into it. If the interpreter version and the `cpython-3XX` + # tag in the `.so` filename disagree, `python-libs` was built with a mismatched Python. + - name: "DIAGNOSTIC: inspect the runtime image's pydantic_core" + if: "matrix.arch == 'amd64'" + shell: "bash" + run: |- + image_id="$(cat build/clp-package-image.id)" + echo "== Runtime image Python version ==" + docker run --rm --entrypoint bash "$image_id" -c "python3 --version" + echo "== pydantic_core contents in the image ==" + docker run --rm --entrypoint bash "$image_id" \ + -c "ls -l /opt/clp/lib/python3/site-packages/pydantic_core/" + - name: "Assemble the package directory for the integration tests" if: "matrix.arch == 'amd64'" uses: "./.github/actions/run-on-image" From 52338b0de4f961fdbcb35cd4c8a70ab305330777 Mon Sep 17 00:00:00 2001 From: Quinn Date: Tue, 7 Jul 2026 19:51:57 +0000 Subject: [PATCH 08/38] Try a new approach --- .../clp-build-runtime-image/action.yaml | 29 +++- .github/workflows/clp-artifact-build.yaml | 142 ++---------------- taskfile.yaml | 11 +- 3 files changed, 37 insertions(+), 145 deletions(-) diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index 454dd19394..1976c4c4de 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -22,6 +22,9 @@ inputs: description: >- Platform version codename (e.g. jammy, noble) used as the runtime image base version. required: true + push_image: + description: "Whether to publish the package image" + required: true runs: using: "composite" @@ -30,6 +33,7 @@ runs: uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2" - name: "Login to Image Registry" + if: "inputs.push_image == 'true'" uses: "docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121" # v4.1.0 with: registry: "${{inputs.image_registry}}" @@ -78,8 +82,21 @@ runs: exit 1 fi + - name: "Compute image output properties" + id: "output-properties" + shell: "bash" + run: | + image_name="${{steps.compute-meta.outputs.IMAGE_NAME}}" + if [[ "${{inputs.push_image}}" == "true" ]]; then + echo "image_output=type=registry" >> "$GITHUB_OUTPUT" + else + # Suffix with arch so the amd64/arm64 tars don't collide. + image_path="/tmp/${image_name}-${{inputs.arch}}.tar" + echo "image_path=${image_path}" >> "$GITHUB_OUTPUT" + echo "image_output=type=docker,dest=${image_path}" >> "$GITHUB_OUTPUT" + fi + - name: "Build and Push" - if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: context: "./" @@ -88,8 +105,16 @@ runs: # Disable provenance to create a simple image instead of a manifest list. # This allows `docker manifest create` to combine the per-arch images later. provenance: false - push: true + outputs: "${{steps.output-properties.outputs.image_output}}" tags: "${{steps.extract-gh-meta.outputs.tags}}" labels: "${{steps.extract-gh-meta.outputs.labels}}" build-args: | UBUNTU_VERSION_CODENAME=${{inputs.platform_version_codename}} + + - name: "Upload package image as artifact" + if: "inputs.push_image == 'false'" + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "${{steps.compute-meta.outputs.IMAGE_NAME}}-${{inputs.arch}}" + path: "${{steps.output-properties.outputs.image_path}}" + retention-days: 1 diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index b1bb9cd631..4b07b23bae 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -679,8 +679,6 @@ jobs: ubuntu-jammy-integration-tests: name: "ubuntu-jammy-integration-tests" - # Run if `package-image` built successfully, reusing its build artifact rather than rebuilding - # the package here. if: "!cancelled() && !failure() && needs.package-image.result == 'success'" needs: - "package-image" @@ -690,66 +688,15 @@ jobs: with: submodules: "recursive" - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Download the integration-test bundle" - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - with: - name: "clp-package-integration-bundle" - path: "/tmp/clp-package-integration-bundle" - - - name: "Load the runtime image and restore the build outputs" - shell: "bash" - run: |- - docker load --input "/tmp/clp-package-integration-bundle/clp-package-image.tar" - tar --extract \ - --file "/tmp/clp-package-integration-bundle/build.tar" \ - --directory "$GITHUB_WORKSPACE" - - - name: "Install uv" - shell: "bash" - run: |- - curl --fail --silent --show-error --location \ - "https://astral.sh/uv/0.8.0/install.sh" | sh - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - # The `mariadb` Python driver (a transitive dependency of `job-orchestration`) is only - # published as a source distribution and must compile against MariaDB Connector/C, which - # provides the `mariadb_config` helper. Install it so `uv` can build the driver. The tests run - # directly on the runner (not in a container) so `start-clp` can reach the host Docker daemon. - - name: "Install the MariaDB Connector/C development files" - shell: "bash" - run: "sudo apt-get update && sudo apt-get install --yes libmariadb-dev" - - - name: "Run the integration tests" - working-directory: "integration-tests" - shell: "bash" - run: "uv run pytest" - - # Print the captured subprocess logs directly into the run output so failures are visible - # without downloading an artifact. - - name: "Dump the integration-test subprocess logs on failure" - if: "failure()" - shell: "bash" - run: |- - shopt -s nullglob - for log_file in build/integration_tests/test_logs/*/subprocess_output/*.log; do - echo "::group::${log_file}" - cat "${log_file}" - echo "::endgroup::" - done - - # Upload the per-run test logs (including the captured `start-clp.sh`/`stop-clp.sh` subprocess - # output) so failures can be diagnosed from the Actions run page without runner access. - - name: "Upload the integration-test logs" - if: "always()" - uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" with: - name: "clp-integration-test-logs" - path: "build/integration_tests/test_logs" - retention-days: 3 + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: "echo test" package-image: name: "package-image-${{matrix.arch}}" @@ -814,78 +761,7 @@ jobs: image_registry_password: "${{secrets.GITHUB_TOKEN}}" arch: "${{matrix.arch}}" platform_version_codename: "jammy" - - # Build a local runtime image, assemble the package directory, and upload them (with the core - # binaries) as an artifact for the `ubuntu-jammy-integration-tests` job to consume. This lets - # the integration tests reuse this job's build instead of rebuilding the package. We only do - # this on amd64 since the integration tests run on x86_64. - - name: "Build the package image for the integration tests" - if: "matrix.arch == 'amd64'" - shell: "bash" - run: "./tools/docker-images/clp-package/build.sh" - - # TEMPORARY DIAGNOSTIC: report the runtime image's Python version and the compiled - # `pydantic_core` extension baked into it. If the interpreter version and the `cpython-3XX` - # tag in the `.so` filename disagree, `python-libs` was built with a mismatched Python. - - name: "DIAGNOSTIC: inspect the runtime image's pydantic_core" - if: "matrix.arch == 'amd64'" - shell: "bash" - run: |- - image_id="$(cat build/clp-package-image.id)" - echo "== Runtime image Python version ==" - docker run --rm --entrypoint bash "$image_id" -c "python3 --version" - echo "== pydantic_core contents in the image ==" - docker run --rm --entrypoint bash "$image_id" \ - -c "ls -l /opt/clp/lib/python3/site-packages/pydantic_core/" - - - name: "Assemble the package directory for the integration tests" - if: "matrix.arch == 'amd64'" - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - # TODO: We set HOME due to permission issues inside the container used in run-on-image. - # See: https://github.com/y-scope/clp/issues/1145 - run_command: >- - HOME=/tmp - task package-assemble - - - name: "Stage the integration-test bundle" - if: "matrix.arch == 'amd64'" - id: "stage_integration_bundle" - shell: "bash" - run: |- - bundle_dir="/tmp/clp-package-integration-bundle" - echo "bundle_dir=${bundle_dir}" >> "$GITHUB_OUTPUT" - mkdir -p "${bundle_dir}" - - # Save the runtime image referenced by the assembled package's `clp-package-image.id`. - docker save "$(cat build/clp-package-image.id)" \ - --output "${bundle_dir}/clp-package-image.tar" - - # Bundle the core binaries and the assembled package directory. - tar --create --file "${bundle_dir}/build.tar" \ - --directory "$GITHUB_WORKSPACE" \ - build/core/clg \ - build/core/clo \ - build/core/clp \ - build/core/clp-s \ - build/core/indexer \ - build/core/log-converter \ - build/core/reducer-server \ - build/clp-package - - - name: "Upload the integration-test bundle" - if: "matrix.arch == 'amd64'" - uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 - with: - name: "clp-package-integration-bundle" - path: "${{steps.stage_integration_bundle.outputs.bundle_dir}}" - retention-days: 1 + push_image: "${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}" package-image-multiarch-manifest: name: "package-image-multiarch-manifest" diff --git a/taskfile.yaml b/taskfile.yaml index d7325db488..8bd3c7633d 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -118,16 +118,6 @@ tasks: - "rm -f '{{.G_WEBUI_CHECKSUM_FILE}}'" package: - deps: - - "docker-images:package" - cmds: - - task: "package-assemble" - - # Assembles the package directory from already-built components and the package image's ID file. - # NOTE: This is split out from `package` so that callers which build the package image separately - # (e.g., the `package-image` CI job) can assemble the directory without rebuilding the image. It - # requires `{{.G_BUILD_DIR}}/clp-package-image.id` and the built components to already exist. - package-assemble: vars: CHECKSUM_FILE: "{{.G_PACKAGE_CHECKSUM_FILE}}" OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}" @@ -140,6 +130,7 @@ tasks: - "tools/deployment/package/**/*" generates: ["{{.CHECKSUM_FILE}}"] deps: + - "docker-images:package" - "generate-config-schemas" - task: "utils:checksum:validate" vars: From 14c8a77e5bc4346329ec430a3de3ad5ad9d42dec Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 13 Jul 2026 14:35:27 +0000 Subject: [PATCH 09/38] Run the integration tests with a workflow. --- .github/workflows/clp-artifact-build.yaml | 25 ++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 4b07b23bae..abe901783f 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -682,12 +682,27 @@ jobs: if: "!cancelled() && !failure() && needs.package-image.result == 'success'" needs: - "package-image" + strategy: + matrix: + arch: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + && fromJSON('["amd64", "arm64"]') || fromJSON('["amd64"]')}} runs-on: *runner steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 with: submodules: "recursive" + - name: "Download the package artifact" + uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "clp-package-${{matrix.arch}}" + path: "/tmp/clp-package" + + - name: "Load the package image" + shell: "bash" + run: "docker load --input /tmp/clp-package/clp-package-${{matrix.arch}}.tar" + - uses: "./.github/actions/run-on-image" env: OS_NAME: "ubuntu-jammy" @@ -696,7 +711,15 @@ jobs: use_published_image: >- ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: "echo test" + run_command: "cd integration-tests && uv run pytest" + + - name: "Upload the integration-test logs" + if: "always()" + uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 + with: + name: "clp-integration-test-logs" + path: "build/integration_tests/test_logs" + retention-days: 3 package-image: name: "package-image-${{matrix.arch}}" From aa498c1d4d1e8aa4f22c06a5a3169c7c49c540df Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 13 Jul 2026 17:29:07 +0000 Subject: [PATCH 10/38] Continue --- .github/workflows/clp-artifact-build.yaml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index abe901783f..154fb373ab 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -697,11 +697,17 @@ jobs: uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" with: name: "clp-package-${{matrix.arch}}" - path: "/tmp/clp-package" + path: "/tmp/build/clp-package" - name: "Load the package image" shell: "bash" - run: "docker load --input /tmp/clp-package/clp-package-${{matrix.arch}}.tar" + run: "docker load --input /tmp/build/clp-package/clp-package-${{matrix.arch}}.tar" + + - name: "Download the binaries artifact" + uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + path: "/tmp/build/core" - uses: "./.github/actions/run-on-image" env: @@ -711,7 +717,12 @@ jobs: use_published_image: >- ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: "cd integration-tests && uv run pytest" + run_command: >- + cd integration-tests + && CLP_BUILD_DIR=/tmp/build + CLP_CORE_BINS_DIR=/tmp/build/core + CLP_PACKAGE_DIR=/tmp/build/clp-package + uv run pytest - name: "Upload the integration-test logs" if: "always()" From 4a4e700b7598c653692aff3a060c4ea0728c5d78 Mon Sep 17 00:00:00 2001 From: Quinn Date: Tue, 14 Jul 2026 15:19:24 +0000 Subject: [PATCH 11/38] Improvements --- .../clp-build-runtime-image/action.yaml | 2 +- .github/workflows/clp-artifact-build.yaml | 65 +++++++++++++++++-- taskfile.yaml | 31 ++++++--- 3 files changed, 81 insertions(+), 17 deletions(-) diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index 1976c4c4de..5bd4309182 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -54,7 +54,7 @@ runs: shell: "bash" run: | base_path="./tools/docker-images" - image_name="clp-package" + image_name="clp-package-image" dockerfile_path="$base_path/$image_name/Dockerfile" echo "DOCKERFILE=$dockerfile_path" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 154fb373ab..78f01a336d 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -17,6 +17,8 @@ on: env: DEPS_IMAGE_NAME_PREFIX_AARCH64: "clp-core-dependencies-aarch64-" BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" + PACKAGE_ARTIFACT_NAME_PREFIX: "clp-package-" + PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX: "clp-package-image-" PYTHON_WHEELS_ARTIFACT_NAME_PREFIX: "clp-python-wheels-" DEPS_IMAGE_NAME_PREFIX_X86: "clp-core-dependencies-x86-" @@ -679,9 +681,13 @@ jobs: ubuntu-jammy-integration-tests: name: "ubuntu-jammy-integration-tests" - if: "!cancelled() && !failure() && needs.package-image.result == 'success'" + if: >- + !cancelled() && !failure() && needs.package-image.result == 'success' + && needs.ubuntu-jammy-binaries.result == 'success' needs: + - "filter-relevant-changes" - "package-image" + - "ubuntu-jammy-binaries" strategy: matrix: arch: >- @@ -693,20 +699,37 @@ jobs: with: submodules: "recursive" - - name: "Download the package artifact" + - name: "Download the package image artifact" uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" with: - name: "clp-package-${{matrix.arch}}" + name: "${{env.PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}" path: "/tmp/build/clp-package" - name: "Load the package image" shell: "bash" - run: "docker load --input /tmp/build/clp-package/clp-package-${{matrix.arch}}.tar" + run: >- + docker load + --input + /tmp/build/clp-package/${{env.PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}.tar + + - name: "Download the package artifact" + uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.PACKAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}" + path: "/tmp/build/clp-package" + + - name: "Untar the package" + shell: "bash" + run: >- + tar --extract --file "/tmp/build/clp-package/clp-package.tar" + --directory "/tmp/build/clp-package" - name: "Download the binaries artifact" uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + env: + OS_NAME: "ubuntu-jammy" with: - name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" path: "/tmp/build/core" - uses: "./.github/actions/run-on-image" @@ -729,7 +752,7 @@ jobs: uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 with: name: "clp-integration-test-logs" - path: "build/integration_tests/test_logs" + path: "/tmp/build/integration_tests/test_logs" retention-days: 3 package-image: @@ -797,6 +820,36 @@ jobs: platform_version_codename: "jammy" push_image: "${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}" + - name: "Assemble the package directories" + uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: >- + ${{format('{0}ubuntu-jammy', + matrix.arch == 'amd64' && env.DEPS_IMAGE_NAME_PREFIX_X86 + || env.DEPS_IMAGE_NAME_PREFIX_AARCH64)}} + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + HOME=/tmp + task package-assemble OUTPUT_DIR=build/clp-package + + - name: "Tar the package directories" + shell: "bash" + run: >- + tar --create --file "build/clp-package.tar" + --directory "build/clp-package" . + + - name: "Upload the package tarball artifact" + uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 + with: + name: "${{env.PACKAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}" + path: "build/clp-package.tar" + retention-days: 1 + package-image-multiarch-manifest: name: "package-image-multiarch-manifest" if: >- diff --git a/taskfile.yaml b/taskfile.yaml index 8bd3c7633d..6c1ef55cf1 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -136,6 +136,26 @@ tasks: vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + cmds: + - task: "package-assemble" + vars: + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + - >- + rsync --archive + "{{.G_BUILD_DIR}}/clp-package-image.id" + "{{.OUTPUT_DIR}}" + - "echo '{{.G_PACKAGE_VERSION}}' > '{{.OUTPUT_DIR}}/VERSION'" + # This command must be last + - task: "utils:checksum:compute" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + + package-assemble: + requires: + vars: ["OUTPUT_DIR"] + deps: + - "generate-config-schemas" cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - >- @@ -161,16 +181,6 @@ tasks: rsync --archive "tools/deployment/package/" "{{.OUTPUT_DIR}}" - - >- - rsync --archive - "{{.G_BUILD_DIR}}/clp-package-image.id" - "{{.OUTPUT_DIR}}" - - "echo '{{.G_PACKAGE_VERSION}}' > '{{.OUTPUT_DIR}}/VERSION'" - # This command must be last - - task: "utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] core: cmds: @@ -238,6 +248,7 @@ tasks: COMPONENT: "{{.TASK}}" generate-config-schemas: + run: "once" vars: OUTPUT_DIR: "{{.G_CONFIG_SCHEMA_BUILD_DIR}}" cmds: From 8f6c8dcdc79370cdf2beeb46c14bd5c0297b0edf Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 16 Jul 2026 13:41:47 +0000 Subject: [PATCH 12/38] Fix path error --- .github/actions/clp-build-runtime-image/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index 5bd4309182..2fbfc303c0 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -54,7 +54,7 @@ runs: shell: "bash" run: | base_path="./tools/docker-images" - image_name="clp-package-image" + image_name="clp-package" dockerfile_path="$base_path/$image_name/Dockerfile" echo "DOCKERFILE=$dockerfile_path" >> "$GITHUB_OUTPUT" @@ -115,6 +115,6 @@ runs: if: "inputs.push_image == 'false'" uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" with: - name: "${{steps.compute-meta.outputs.IMAGE_NAME}}-${{inputs.arch}}" + name: "clp-package-image-${{inputs.arch}}" path: "${{steps.output-properties.outputs.image_path}}" retention-days: 1 From 205f613e3128f6825d2b3f4d41988e70a61fbdf0 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 16 Jul 2026 14:04:31 +0000 Subject: [PATCH 13/38] Separate core and package integration tests --- .github/workflows/clp-artifact-build.yaml | 69 +++++++++++++++---- integration-tests/.pytest.ini | 1 + .../test_clp_native_py_project_imports.py | 2 + 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 78f01a336d..4b6bd1683f 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -679,15 +679,65 @@ jobs: 'build/lint-clang-tidy/**/*' )}} - ubuntu-jammy-integration-tests: - name: "ubuntu-jammy-integration-tests" + ubuntu-jammy-integration-tests-core: + name: "ubuntu-jammy-integration-tests-core" + if: >- + !cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success' + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-binaries" + runs-on: *runner + steps: + - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 + with: + submodules: "recursive" + + - name: "Download the binaries artifact" + uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + env: + OS_NAME: "ubuntu-jammy" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + + - name: "Untar binaries" + env: + OS_NAME: "ubuntu-jammy" + working-directory: >- + /tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} + run: |- + tar xf clp.tar + rm clp.tar + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + cd integration-tests + && CLP_BUILD_DIR=/tmp + CLP_CORE_BINS_DIR=/tmp/clp + uv run pytest -m "smoke and core" + + - name: "Upload the integration-test logs" + if: "always()" + uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 + with: + name: "clp-integration-test-core-logs" + path: "/tmp/integration_tests/test_logs" + retention-days: 3 + + ubuntu-jammy-integration-tests-package: + name: "ubuntu-jammy-integration-tests-package" if: >- !cancelled() && !failure() && needs.package-image.result == 'success' - && needs.ubuntu-jammy-binaries.result == 'success' needs: - "filter-relevant-changes" - "package-image" - - "ubuntu-jammy-binaries" strategy: matrix: arch: >- @@ -724,14 +774,6 @@ jobs: tar --extract --file "/tmp/build/clp-package/clp-package.tar" --directory "/tmp/build/clp-package" - - name: "Download the binaries artifact" - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - env: - OS_NAME: "ubuntu-jammy" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "/tmp/build/core" - - uses: "./.github/actions/run-on-image" env: OS_NAME: "ubuntu-jammy" @@ -743,9 +785,8 @@ jobs: run_command: >- cd integration-tests && CLP_BUILD_DIR=/tmp/build - CLP_CORE_BINS_DIR=/tmp/build/core CLP_PACKAGE_DIR=/tmp/build/clp-package - uv run pytest + uv run pytest -m "smoke and package" - name: "Upload the integration-test logs" if: "always()" diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index 0cf24cf095..e4a42810c0 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -30,4 +30,5 @@ markers = core: mark tests that test the CLP core binaries package: mark tests that use the CLP package search: mark tests that test search + smoke: mark tests that are smoke tests startstop: mark tests that test start-stop diff --git a/integration-tests/tests/test_clp_native_py_project_imports.py b/integration-tests/tests/test_clp_native_py_project_imports.py index b3809843e0..b119452f42 100644 --- a/integration-tests/tests/test_clp_native_py_project_imports.py +++ b/integration-tests/tests/test_clp_native_py_project_imports.py @@ -1,11 +1,13 @@ """Smoke tests to validate that CLP Python projects can be imported without errors.""" +import pytest from clp_mcp_server.constants import QueryJobType from clp_package_utils.general import JobType from clp_py_utils.clp_config import StorageEngine from job_orchestration.scheduler.constants import CompressionJobStatus +@pytest.mark.smoke def test_clp_native_py_project_enum_classes() -> None: """ Verifies that the following CLP Python projects can be imported successfully by testing From 426adc61c99a8b4545c36e7229a8367dfc8f1dcc Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 16 Jul 2026 16:15:35 +0000 Subject: [PATCH 14/38] Fix a few little things --- .github/actions/run-on-image/action.yaml | 5 +++++ .github/workflows/clp-artifact-build.yaml | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-on-image/action.yaml b/.github/actions/run-on-image/action.yaml index 3d0c060873..9c1dd82a6f 100644 --- a/.github/actions/run-on-image/action.yaml +++ b/.github/actions/run-on-image/action.yaml @@ -11,6 +11,10 @@ inputs: run_command: description: "Command to execute" required: true + mount_host_tmp_dir: + description: "Whether to mount the host's `/tmp` at the same path in the container" + required: false + default: "false" runs: using: "composite" @@ -48,6 +52,7 @@ runs: ${{inputs.use_published_image == 'true' && '--pull=always' || ''}} --user $(id -u):$(id -g) --volume "$GITHUB_WORKSPACE":/mnt/repo + ${{inputs.mount_host_tmp_dir == 'true' && '--volume /tmp:/tmp' || ''}} --workdir /mnt/repo ${{steps.get_image_props.outputs.qualified_image_name}} bash -c "${{inputs.run_command}}" diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 4b6bd1683f..beebffb309 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -717,11 +717,12 @@ jobs: use_published_image: >- ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + mount_host_tmp_dir: "true" run_command: >- cd integration-tests && CLP_BUILD_DIR=/tmp CLP_CORE_BINS_DIR=/tmp/clp - uv run pytest -m "smoke and core" + uv run pytest -m "smoke or core" - name: "Upload the integration-test logs" if: "always()" @@ -782,11 +783,12 @@ jobs: use_published_image: >- ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + mount_host_tmp_dir: "true" run_command: >- cd integration-tests && CLP_BUILD_DIR=/tmp/build CLP_PACKAGE_DIR=/tmp/build/clp-package - uv run pytest -m "smoke and package" + uv run pytest -m "smoke or package" - name: "Upload the integration-test logs" if: "always()" From 540da756d7e0e9ac5bc16b868d6bff60b2ffd5d7 Mon Sep 17 00:00:00 2001 From: Quinn Date: Fri, 17 Jul 2026 17:36:35 +0000 Subject: [PATCH 15/38] Fix double-quote pytest bug --- .github/workflows/clp-artifact-build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index beebffb309..5f1e399f42 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -722,7 +722,7 @@ jobs: cd integration-tests && CLP_BUILD_DIR=/tmp CLP_CORE_BINS_DIR=/tmp/clp - uv run pytest -m "smoke or core" + uv run pytest -m 'smoke or core' - name: "Upload the integration-test logs" if: "always()" @@ -788,7 +788,7 @@ jobs: cd integration-tests && CLP_BUILD_DIR=/tmp/build CLP_PACKAGE_DIR=/tmp/build/clp-package - uv run pytest -m "smoke or package" + uv run pytest -m 'smoke or package' - name: "Upload the integration-test logs" if: "always()" From 7c5fe4347b80c8d83909f17d77a2db8787ffee98 Mon Sep 17 00:00:00 2001 From: Quinn Date: Sat, 18 Jul 2026 14:09:28 +0000 Subject: [PATCH 16/38] Fix file path stuff --- .github/actions/clp-build-runtime-image/action.yaml | 2 +- .github/workflows/clp-artifact-build.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index 2fbfc303c0..6cddf919ce 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -91,7 +91,7 @@ runs: echo "image_output=type=registry" >> "$GITHUB_OUTPUT" else # Suffix with arch so the amd64/arm64 tars don't collide. - image_path="/tmp/${image_name}-${{inputs.arch}}.tar" + image_path="/tmp/${image_name}-image-${{inputs.arch}}.tar" echo "image_path=${image_path}" >> "$GITHUB_OUTPUT" echo "image_output=type=docker,dest=${image_path}" >> "$GITHUB_OUTPUT" fi diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 5f1e399f42..33b6105327 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -721,7 +721,7 @@ jobs: run_command: >- cd integration-tests && CLP_BUILD_DIR=/tmp - CLP_CORE_BINS_DIR=/tmp/clp + CLP_CORE_BINS_DIR=/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} uv run pytest -m 'smoke or core' - name: "Upload the integration-test logs" From 0f861d3b70ab51dd4c6e1c09d4512abfcb60afe8 Mon Sep 17 00:00:00 2001 From: Quinn Date: Sat, 18 Jul 2026 16:45:18 +0000 Subject: [PATCH 17/38] Run core tests directly on runner --- .github/actions/run-on-image/action.yaml | 5 -- .github/workflows/clp-artifact-build.yaml | 84 ++++++++++------------- 2 files changed, 35 insertions(+), 54 deletions(-) diff --git a/.github/actions/run-on-image/action.yaml b/.github/actions/run-on-image/action.yaml index 9c1dd82a6f..3d0c060873 100644 --- a/.github/actions/run-on-image/action.yaml +++ b/.github/actions/run-on-image/action.yaml @@ -11,10 +11,6 @@ inputs: run_command: description: "Command to execute" required: true - mount_host_tmp_dir: - description: "Whether to mount the host's `/tmp` at the same path in the container" - required: false - default: "false" runs: using: "composite" @@ -52,7 +48,6 @@ runs: ${{inputs.use_published_image == 'true' && '--pull=always' || ''}} --user $(id -u):$(id -g) --volume "$GITHUB_WORKSPACE":/mnt/repo - ${{inputs.mount_host_tmp_dir == 'true' && '--volume /tmp:/tmp' || ''}} --workdir /mnt/repo ${{steps.get_image_props.outputs.qualified_image_name}} bash -c "${{inputs.run_command}}" diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 33b6105327..0aef332dfa 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -687,6 +687,9 @@ jobs: - "filter-relevant-changes" - "ubuntu-jammy-binaries" runs-on: *runner + env: + OS_NAME: "ubuntu-jammy" + BUILD_DIR: "build" steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 with: @@ -694,42 +697,30 @@ jobs: - name: "Download the binaries artifact" uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - env: - OS_NAME: "ubuntu-jammy" with: name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - name: "Untar binaries" - env: - OS_NAME: "ubuntu-jammy" working-directory: >- - /tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} + ${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} run: |- tar xf clp.tar rm clp.tar - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - mount_host_tmp_dir: "true" - run_command: >- - cd integration-tests - && CLP_BUILD_DIR=/tmp - CLP_CORE_BINS_DIR=/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} - uv run pytest -m 'smoke or core' + - name: "Run core integration tests" + working-directory: "integration-tests" + run: >- + CLP_BUILD_DIR=../${{env.BUILD_DIR}} + CLP_CORE_BINS_DIR=../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}/clp + uv run pytest -m 'smoke or core' - name: "Upload the integration-test logs" if: "always()" uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 with: name: "clp-integration-test-core-logs" - path: "/tmp/integration_tests/test_logs" + path: "${{env.BUILD_DIR}}/integration_tests/test_logs" retention-days: 3 ubuntu-jammy-integration-tests-package: @@ -745,57 +736,52 @@ jobs: ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && fromJSON('["amd64", "arm64"]') || fromJSON('["amd64"]')}} runs-on: *runner + env: + OS_NAME: "ubuntu-jammy" + BUILD_DIR: "build" steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 with: submodules: "recursive" + - name: "Download the package artifact" + uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.PACKAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}" + path: "${{env.BUILD_DIR}}/clp-package" + + - name: "Untar the package" + working-directory: "${{env.BUILD_DIR}}/clp-package" + run: |- + tar xf clp-package.tar + rm clp-package.tar + - name: "Download the package image artifact" uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" with: name: "${{env.PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}" - path: "/tmp/build/clp-package" + path: "${{env.BUILD_DIR}}/clp-package-image" - name: "Load the package image" shell: "bash" run: >- docker load --input - /tmp/build/clp-package/${{env.PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}.tar + ${{env.BUILD_DIR}}/clp-package-image/${{env.PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}.tar - - name: "Download the package artifact" - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - with: - name: "${{env.PACKAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}" - path: "/tmp/build/clp-package" - - - name: "Untar the package" - shell: "bash" + - name: "Run package integration tests" + working-directory: "integration-tests" run: >- - tar --extract --file "/tmp/build/clp-package/clp-package.tar" - --directory "/tmp/build/clp-package" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - mount_host_tmp_dir: "true" - run_command: >- - cd integration-tests - && CLP_BUILD_DIR=/tmp/build - CLP_PACKAGE_DIR=/tmp/build/clp-package - uv run pytest -m 'smoke or package' + CLP_BUILD_DIR=../${{env.BUILD_DIR}} + CLP_PACKAGE_DIR=../${{env.BUILD_DIR}}/clp-package + uv run pytest -m 'smoke or package' - name: "Upload the integration-test logs" if: "always()" uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 with: name: "clp-integration-test-logs" - path: "/tmp/build/integration_tests/test_logs" + path: "${{env.BUILD_DIR}}/integration_tests/test_logs" retention-days: 3 package-image: From 864dcb2556d4af2ed0027ea22ed1cda5365abac7 Mon Sep 17 00:00:00 2001 From: Quinn Date: Sat, 18 Jul 2026 17:08:06 +0000 Subject: [PATCH 18/38] Include clp-packge-image.id in tarball --- .github/actions/clp-build-runtime-image/action.yaml | 6 ++++++ .github/workflows/clp-artifact-build.yaml | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index 6cddf919ce..0433b640de 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -26,6 +26,11 @@ inputs: description: "Whether to publish the package image" required: true +outputs: + image_id: + description: "The ID of the built container image." + value: "${{steps.build-and-push.outputs.imageid}}" + runs: using: "composite" steps: @@ -97,6 +102,7 @@ runs: fi - name: "Build and Push" + id: "build-and-push" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: context: "./" diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 0aef332dfa..c3058970f7 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -840,7 +840,8 @@ jobs: HOME=/tmp task package-build-deps - - uses: "./.github/actions/clp-build-runtime-image" + - id: "build-runtime-image" + uses: "./.github/actions/clp-build-runtime-image" with: image_registry: "ghcr.io" image_registry_username: "${{github.actor}}" @@ -866,6 +867,12 @@ jobs: HOME=/tmp task package-assemble OUTPUT_DIR=build/clp-package + - name: "Write the package image ID" + shell: "bash" + run: >- + echo "${{steps.build-runtime-image.outputs.image_id}}" + > "build/clp-package/clp-package-image.id" + - name: "Tar the package directories" shell: "bash" run: >- From 98ffe9e90b7ae3010735a3cc2140c20f42893724 Mon Sep 17 00:00:00 2001 From: Quinn Date: Sat, 18 Jul 2026 13:54:36 -0400 Subject: [PATCH 19/38] Install uv before running tests --- .github/workflows/clp-artifact-build.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index c3058970f7..3e8fbe505c 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -695,6 +695,9 @@ jobs: with: submodules: "recursive" + - name: "Install uv" + uses: "astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990" # v8.3.2 + - name: "Download the binaries artifact" uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" with: @@ -744,6 +747,9 @@ jobs: with: submodules: "recursive" + - name: "Install uv" + uses: "astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990" # v8.3.2 + - name: "Download the package artifact" uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" with: From 0acc0b3cfa4427aa05e468fa68601e24ef18bded Mon Sep 17 00:00:00 2001 From: Quinn Date: Sat, 18 Jul 2026 15:31:17 -0400 Subject: [PATCH 20/38] Run the pytest commands on the jammy image --- .github/workflows/clp-artifact-build.yaml | 46 +++++++++++++++-------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 3e8fbe505c..20fb06dc6f 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -686,6 +686,7 @@ jobs: needs: - "filter-relevant-changes" - "ubuntu-jammy-binaries" + - "ubuntu-jammy-x86_64-deps-image" runs-on: *runner env: OS_NAME: "ubuntu-jammy" @@ -695,9 +696,6 @@ jobs: with: submodules: "recursive" - - name: "Install uv" - uses: "astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990" # v8.3.2 - - name: "Download the binaries artifact" uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" with: @@ -712,11 +710,20 @@ jobs: rm clp.tar - name: "Run core integration tests" - working-directory: "integration-tests" - run: >- - CLP_BUILD_DIR=../${{env.BUILD_DIR}} - CLP_CORE_BINS_DIR=../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}/clp - uv run pytest -m 'smoke or core' + uses: "./.github/actions/run-on-image" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + # TODO: We set HOME due to permission issues inside the container used in run-on-image. + # See: https://github.com/y-scope/clp/issues/1145 + run_command: >- + cd integration-tests + && CLP_BUILD_DIR=../${{env.BUILD_DIR}} + CLP_CORE_BINS_DIR=../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}/clp + HOME=/tmp + uv run pytest -m 'smoke or core' - name: "Upload the integration-test logs" if: "always()" @@ -733,6 +740,7 @@ jobs: needs: - "filter-relevant-changes" - "package-image" + - "ubuntu-jammy-x86_64-deps-image" strategy: matrix: arch: >- @@ -747,9 +755,6 @@ jobs: with: submodules: "recursive" - - name: "Install uv" - uses: "astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990" # v8.3.2 - - name: "Download the package artifact" uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" with: @@ -776,11 +781,20 @@ jobs: ${{env.BUILD_DIR}}/clp-package-image/${{env.PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}.tar - name: "Run package integration tests" - working-directory: "integration-tests" - run: >- - CLP_BUILD_DIR=../${{env.BUILD_DIR}} - CLP_PACKAGE_DIR=../${{env.BUILD_DIR}}/clp-package - uv run pytest -m 'smoke or package' + uses: "./.github/actions/run-on-image" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + # TODO: We set HOME due to permission issues inside the container used in run-on-image. + # See: https://github.com/y-scope/clp/issues/1145 + run_command: >- + cd integration-tests + && CLP_BUILD_DIR=../${{env.BUILD_DIR}} + CLP_PACKAGE_DIR=../${{env.BUILD_DIR}}/clp-package + HOME=/tmp + uv run pytest -m 'smoke or package' - name: "Upload the integration-test logs" if: "always()" From 981e81fd1acb09368ab92c91557cc27961b4be5e Mon Sep 17 00:00:00 2001 From: Quinn Date: Sat, 18 Jul 2026 22:05:58 -0400 Subject: [PATCH 21/38] Fix yet another path-related bug --- .github/workflows/clp-artifact-build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 20fb06dc6f..d6d80e7469 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -721,7 +721,7 @@ jobs: run_command: >- cd integration-tests && CLP_BUILD_DIR=../${{env.BUILD_DIR}} - CLP_CORE_BINS_DIR=../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}/clp + CLP_CORE_BINS_DIR=../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} HOME=/tmp uv run pytest -m 'smoke or core' @@ -800,7 +800,7 @@ jobs: if: "always()" uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 with: - name: "clp-integration-test-logs" + name: "clp-integration-test-package-logs" path: "${{env.BUILD_DIR}}/integration_tests/test_logs" retention-days: 3 From bd54f231d0aab2a274ec0ab1c09daa5fd13298e1 Mon Sep 17 00:00:00 2001 From: Quinn Date: Sun, 19 Jul 2026 12:34:20 -0400 Subject: [PATCH 22/38] Try running package tests directly on the runner --- .github/workflows/clp-artifact-build.yaml | 25 ++++++++--------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index d6d80e7469..59727e9e08 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -738,9 +738,7 @@ jobs: if: >- !cancelled() && !failure() && needs.package-image.result == 'success' needs: - - "filter-relevant-changes" - "package-image" - - "ubuntu-jammy-x86_64-deps-image" strategy: matrix: arch: >- @@ -748,7 +746,6 @@ jobs: && fromJSON('["amd64", "arm64"]') || fromJSON('["amd64"]')}} runs-on: *runner env: - OS_NAME: "ubuntu-jammy" BUILD_DIR: "build" steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 @@ -780,21 +777,15 @@ jobs: --input ${{env.BUILD_DIR}}/clp-package-image/${{env.PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}.tar + - name: "Install uv" + uses: "astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990" # v8.3.2 + - name: "Run package integration tests" - uses: "./.github/actions/run-on-image" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - # TODO: We set HOME due to permission issues inside the container used in run-on-image. - # See: https://github.com/y-scope/clp/issues/1145 - run_command: >- - cd integration-tests - && CLP_BUILD_DIR=../${{env.BUILD_DIR}} - CLP_PACKAGE_DIR=../${{env.BUILD_DIR}}/clp-package - HOME=/tmp - uv run pytest -m 'smoke or package' + working-directory: "integration-tests" + run: >- + CLP_BUILD_DIR=../${{env.BUILD_DIR}} + CLP_PACKAGE_DIR=../${{env.BUILD_DIR}}/clp-package + uv run pytest -m 'smoke or package' - name: "Upload the integration-test logs" if: "always()" From 18c78bb7396429134b2d439e7d1f4c73e0777c08 Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 20 Jul 2026 09:10:59 -0400 Subject: [PATCH 23/38] Install mariadb --- .github/workflows/clp-artifact-build.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 59727e9e08..d3759397a4 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -780,6 +780,9 @@ jobs: - name: "Install uv" uses: "astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990" # v8.3.2 + - name: "Install MariaDB Connector/C" + run: "sudo apt-get update && sudo apt-get install -y libmariadb-dev" + - name: "Run package integration tests" working-directory: "integration-tests" run: >- From 96d2b9d35fa062013b1aa2206ee8a7f27db2c2c9 Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 20 Jul 2026 11:06:33 -0400 Subject: [PATCH 24/38] Run core tests on bare runner --- .github/workflows/clp-artifact-build.yaml | 28 +++++++++++------------ integration-tests/tests/utils/classes.py | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index d3759397a4..7092c64bd3 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -709,21 +709,19 @@ jobs: tar xf clp.tar rm clp.tar - - name: "Run core integration tests" - uses: "./.github/actions/run-on-image" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - # TODO: We set HOME due to permission issues inside the container used in run-on-image. - # See: https://github.com/y-scope/clp/issues/1145 - run_command: >- - cd integration-tests - && CLP_BUILD_DIR=../${{env.BUILD_DIR}} - CLP_CORE_BINS_DIR=../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} - HOME=/tmp - uv run pytest -m 'smoke or core' + - name: "Install uv" + uses: "astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990" # v8.3.2 + + - name: "Install MariaDB Connector/C" + run: "sudo apt-get update && sudo apt-get install -y libmariadb-dev" + + - name: "Run package integration tests" + working-directory: "integration-tests" + run: >- + CLP_BUILD_DIR=../${{env.BUILD_DIR}} + CLP_CORE_BINS_DIR=../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} + HOME=/tmp + uv run pytest -m 'smoke or core' - name: "Upload the integration-test logs" if: "always()" diff --git a/integration-tests/tests/utils/classes.py b/integration-tests/tests/utils/classes.py index 89c847945f..390d239e49 100644 --- a/integration-tests/tests/utils/classes.py +++ b/integration-tests/tests/utils/classes.py @@ -16,7 +16,7 @@ from tests.conftest import get_test_log_dir from tests.utils.utils import validate_dir_exists, validate_file_exists -DEFAULT_CMD_TIMEOUT_SECONDS = 120.0 +DEFAULT_CMD_TIMEOUT_SECONDS = 300.0 logger = logging.getLogger(__name__) From 247ebdb2ecdd32cd058f977f6b54fea2db67edbe Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 20 Jul 2026 19:53:37 +0000 Subject: [PATCH 25/38] Add all binaries to the clp-core-binaries build --- .github/workflows/clp-artifact-build.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 7092c64bd3..4a04fecc0c 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -530,7 +530,8 @@ jobs: mkdir -p "${output_dir}" cd "$GITHUB_WORKSPACE/components/core/build" - tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable + tar cfvv "${output_dir}/clp.tar" \ + clg clo clp clp-s glt indexer log-converter make-dictionaries-readable reducer-server shell: "bash" - if: "matrix.upload_binaries == true" @@ -684,9 +685,7 @@ jobs: if: >- !cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success' needs: - - "filter-relevant-changes" - "ubuntu-jammy-binaries" - - "ubuntu-jammy-x86_64-deps-image" runs-on: *runner env: OS_NAME: "ubuntu-jammy" From 202ec315a7eedf3d19e65bd2bb76f91fdc305dd1 Mon Sep 17 00:00:00 2001 From: Quinn Date: Tue, 21 Jul 2026 15:40:24 +0000 Subject: [PATCH 26/38] Remove package test stuff from this PR (moved to 2407). --- .../clp-build-runtime-image/action.yaml | 35 +----- .github/workflows/clp-artifact-build.yaml | 107 +----------------- integration-tests/tests/utils/classes.py | 2 +- taskfile.yaml | 31 ++--- 4 files changed, 14 insertions(+), 161 deletions(-) diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index 0433b640de..454dd19394 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -22,14 +22,6 @@ inputs: description: >- Platform version codename (e.g. jammy, noble) used as the runtime image base version. required: true - push_image: - description: "Whether to publish the package image" - required: true - -outputs: - image_id: - description: "The ID of the built container image." - value: "${{steps.build-and-push.outputs.imageid}}" runs: using: "composite" @@ -38,7 +30,6 @@ runs: uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2" - name: "Login to Image Registry" - if: "inputs.push_image == 'true'" uses: "docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121" # v4.1.0 with: registry: "${{inputs.image_registry}}" @@ -87,22 +78,8 @@ runs: exit 1 fi - - name: "Compute image output properties" - id: "output-properties" - shell: "bash" - run: | - image_name="${{steps.compute-meta.outputs.IMAGE_NAME}}" - if [[ "${{inputs.push_image}}" == "true" ]]; then - echo "image_output=type=registry" >> "$GITHUB_OUTPUT" - else - # Suffix with arch so the amd64/arm64 tars don't collide. - image_path="/tmp/${image_name}-image-${{inputs.arch}}.tar" - echo "image_path=${image_path}" >> "$GITHUB_OUTPUT" - echo "image_output=type=docker,dest=${image_path}" >> "$GITHUB_OUTPUT" - fi - - name: "Build and Push" - id: "build-and-push" + if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: context: "./" @@ -111,16 +88,8 @@ runs: # Disable provenance to create a simple image instead of a manifest list. # This allows `docker manifest create` to combine the per-arch images later. provenance: false - outputs: "${{steps.output-properties.outputs.image_output}}" + push: true tags: "${{steps.extract-gh-meta.outputs.tags}}" labels: "${{steps.extract-gh-meta.outputs.labels}}" build-args: | UBUNTU_VERSION_CODENAME=${{inputs.platform_version_codename}} - - - name: "Upload package image as artifact" - if: "inputs.push_image == 'false'" - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "clp-package-image-${{inputs.arch}}" - path: "${{steps.output-properties.outputs.image_path}}" - retention-days: 1 diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 4a04fecc0c..58dbc87cb9 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -17,8 +17,6 @@ on: env: DEPS_IMAGE_NAME_PREFIX_AARCH64: "clp-core-dependencies-aarch64-" BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - PACKAGE_ARTIFACT_NAME_PREFIX: "clp-package-" - PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX: "clp-package-image-" PYTHON_WHEELS_ARTIFACT_NAME_PREFIX: "clp-python-wheels-" DEPS_IMAGE_NAME_PREFIX_X86: "clp-core-dependencies-x86-" @@ -730,71 +728,6 @@ jobs: path: "${{env.BUILD_DIR}}/integration_tests/test_logs" retention-days: 3 - ubuntu-jammy-integration-tests-package: - name: "ubuntu-jammy-integration-tests-package" - if: >- - !cancelled() && !failure() && needs.package-image.result == 'success' - needs: - - "package-image" - strategy: - matrix: - arch: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main' - && fromJSON('["amd64", "arm64"]') || fromJSON('["amd64"]')}} - runs-on: *runner - env: - BUILD_DIR: "build" - steps: - - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 - with: - submodules: "recursive" - - - name: "Download the package artifact" - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - with: - name: "${{env.PACKAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}" - path: "${{env.BUILD_DIR}}/clp-package" - - - name: "Untar the package" - working-directory: "${{env.BUILD_DIR}}/clp-package" - run: |- - tar xf clp-package.tar - rm clp-package.tar - - - name: "Download the package image artifact" - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - with: - name: "${{env.PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}" - path: "${{env.BUILD_DIR}}/clp-package-image" - - - name: "Load the package image" - shell: "bash" - run: >- - docker load - --input - ${{env.BUILD_DIR}}/clp-package-image/${{env.PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}.tar - - - name: "Install uv" - uses: "astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990" # v8.3.2 - - - name: "Install MariaDB Connector/C" - run: "sudo apt-get update && sudo apt-get install -y libmariadb-dev" - - - name: "Run package integration tests" - working-directory: "integration-tests" - run: >- - CLP_BUILD_DIR=../${{env.BUILD_DIR}} - CLP_PACKAGE_DIR=../${{env.BUILD_DIR}}/clp-package - uv run pytest -m 'smoke or package' - - - name: "Upload the integration-test logs" - if: "always()" - uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 - with: - name: "clp-integration-test-package-logs" - path: "${{env.BUILD_DIR}}/integration_tests/test_logs" - retention-days: 3 - package-image: name: "package-image-${{matrix.arch}}" # NOTE: The aarch64 job is skipped on PRs, so we accept both 'success' and 'skipped'. @@ -851,51 +784,13 @@ jobs: HOME=/tmp task package-build-deps - - id: "build-runtime-image" - uses: "./.github/actions/clp-build-runtime-image" + - uses: "./.github/actions/clp-build-runtime-image" with: image_registry: "ghcr.io" image_registry_username: "${{github.actor}}" image_registry_password: "${{secrets.GITHUB_TOKEN}}" arch: "${{matrix.arch}}" platform_version_codename: "jammy" - push_image: "${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}" - - - name: "Assemble the package directories" - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: >- - ${{format('{0}ubuntu-jammy', - matrix.arch == 'amd64' && env.DEPS_IMAGE_NAME_PREFIX_X86 - || env.DEPS_IMAGE_NAME_PREFIX_AARCH64)}} - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) - HOME=/tmp - task package-assemble OUTPUT_DIR=build/clp-package - - - name: "Write the package image ID" - shell: "bash" - run: >- - echo "${{steps.build-runtime-image.outputs.image_id}}" - > "build/clp-package/clp-package-image.id" - - - name: "Tar the package directories" - shell: "bash" - run: >- - tar --create --file "build/clp-package.tar" - --directory "build/clp-package" . - - - name: "Upload the package tarball artifact" - uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 - with: - name: "${{env.PACKAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}" - path: "build/clp-package.tar" - retention-days: 1 package-image-multiarch-manifest: name: "package-image-multiarch-manifest" diff --git a/integration-tests/tests/utils/classes.py b/integration-tests/tests/utils/classes.py index 390d239e49..89c847945f 100644 --- a/integration-tests/tests/utils/classes.py +++ b/integration-tests/tests/utils/classes.py @@ -16,7 +16,7 @@ from tests.conftest import get_test_log_dir from tests.utils.utils import validate_dir_exists, validate_file_exists -DEFAULT_CMD_TIMEOUT_SECONDS = 300.0 +DEFAULT_CMD_TIMEOUT_SECONDS = 120.0 logger = logging.getLogger(__name__) diff --git a/taskfile.yaml b/taskfile.yaml index 6e228c2b07..4ad86e152c 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -136,26 +136,6 @@ tasks: vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] - cmds: - - task: "package-assemble" - vars: - OUTPUT_DIR: "{{.OUTPUT_DIR}}" - - >- - rsync --archive - "{{.G_BUILD_DIR}}/clp-package-image.id" - "{{.OUTPUT_DIR}}" - - "echo '{{.G_PACKAGE_VERSION}}' > '{{.OUTPUT_DIR}}/VERSION'" - # This command must be last - - task: "utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] - - package-assemble: - requires: - vars: ["OUTPUT_DIR"] - deps: - - "generate-config-schemas" cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - >- @@ -181,6 +161,16 @@ tasks: rsync --archive "tools/deployment/package/" "{{.OUTPUT_DIR}}" + - >- + rsync --archive + "{{.G_BUILD_DIR}}/clp-package-image.id" + "{{.OUTPUT_DIR}}" + - "echo '{{.G_PACKAGE_VERSION}}' > '{{.OUTPUT_DIR}}/VERSION'" + # This command must be last + - task: "utils:checksum:compute" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] core: cmds: @@ -248,7 +238,6 @@ tasks: COMPONENT: "{{.TASK}}" generate-config-schemas: - run: "once" vars: OUTPUT_DIR: "{{.G_CONFIG_SCHEMA_BUILD_DIR}}" cmds: From 0320b7634e279b7f2ae1b6ae9c476781c11a905a Mon Sep 17 00:00:00 2001 From: Quinn Date: Tue, 21 Jul 2026 15:43:31 +0000 Subject: [PATCH 27/38] Fix long line --- .github/workflows/clp-artifact-build.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 58dbc87cb9..6458ca9028 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -714,10 +714,12 @@ jobs: - name: "Run package integration tests" working-directory: "integration-tests" - run: >- - CLP_BUILD_DIR=../${{env.BUILD_DIR}} - CLP_CORE_BINS_DIR=../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} - HOME=/tmp + run: |- + CLP_BUILD_DIR=\ + ../${{env.BUILD_DIR}} \ + CLP_CORE_BINS_DIR=\ + ../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} \ + HOME=/tmp \ uv run pytest -m 'smoke or core' - name: "Upload the integration-test logs" From 87ac4ec72d6395828f5fe569ad998a870014df9d Mon Sep 17 00:00:00 2001 From: Quinn Date: Tue, 21 Jul 2026 16:09:20 +0000 Subject: [PATCH 28/38] Fix command issue and add integration test directory to clp path filter. --- .github/workflows/clp-artifact-build.yaml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 6458ca9028..5bd70e7b03 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -109,6 +109,7 @@ jobs: - "components/core/src/**" - "components/core/tests/**" - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" + - "integration-tests/**" - "taskfile.yaml" - "taskfiles/**" - "tools/scripts/deps-download/**" @@ -680,10 +681,9 @@ jobs: ubuntu-jammy-integration-tests-core: name: "ubuntu-jammy-integration-tests-core" - if: >- - !cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success' - needs: - - "ubuntu-jammy-binaries" + # Run if the ancestor jobs were successful/skipped and building clp was successful. + if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" + needs: "ubuntu-jammy-binaries" runs-on: *runner env: OS_NAME: "ubuntu-jammy" @@ -714,13 +714,12 @@ jobs: - name: "Run package integration tests" working-directory: "integration-tests" - run: |- - CLP_BUILD_DIR=\ - ../${{env.BUILD_DIR}} \ - CLP_CORE_BINS_DIR=\ - ../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} \ - HOME=/tmp \ - uv run pytest -m 'smoke or core' + env: + CLP_BUILD_DIR: "../${{env.BUILD_DIR}}" + CLP_CORE_BINS_DIR: >- + ../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} + HOME: "/tmp" + run: "uv run pytest -m 'smoke or core'" - name: "Upload the integration-test logs" if: "always()" From 81a617c4e2b17669e148ea1ed56b0a6a87404a86 Mon Sep 17 00:00:00 2001 From: Quinn Date: Tue, 21 Jul 2026 16:18:43 +0000 Subject: [PATCH 29/38] Fix typos --- .github/workflows/clp-artifact-build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 5bd70e7b03..d5e890b6f3 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -712,7 +712,7 @@ jobs: - name: "Install MariaDB Connector/C" run: "sudo apt-get update && sudo apt-get install -y libmariadb-dev" - - name: "Run package integration tests" + - name: "Run core integration tests" working-directory: "integration-tests" env: CLP_BUILD_DIR: "../${{env.BUILD_DIR}}" @@ -721,7 +721,7 @@ jobs: HOME: "/tmp" run: "uv run pytest -m 'smoke or core'" - - name: "Upload the integration-test logs" + - name: "Upload the package integration test logs" if: "always()" uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 with: From 3481d923c1d655ec7a4aa7b65433c3493155215d Mon Sep 17 00:00:00 2001 From: Quinn Date: Tue, 21 Jul 2026 16:19:12 +0000 Subject: [PATCH 30/38] Fix typos --- .github/workflows/clp-artifact-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index d5e890b6f3..e597eb085d 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -721,7 +721,7 @@ jobs: HOME: "/tmp" run: "uv run pytest -m 'smoke or core'" - - name: "Upload the package integration test logs" + - name: "Upload the core integration test logs" if: "always()" uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 with: From 913713ff5b943ea1c8bc6ce538c5181f0c10d5cc Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 27 Jul 2026 19:22:42 +0000 Subject: [PATCH 31/38] Rabbit --- .github/workflows/clp-artifact-build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 600e080fb1..25980498bc 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -739,6 +739,7 @@ jobs: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 with: submodules: "recursive" + persist-credentials: false - name: "Download the binaries artifact" uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" From b3024351b3825ad4740d76e072d00c08b83c17af Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 30 Jul 2026 19:35:43 +0000 Subject: [PATCH 32/38] Get rid of unnecessary 'if:' --- .github/workflows/clp-artifact-build.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 25980498bc..ca35b2edac 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -729,7 +729,6 @@ jobs: ubuntu-jammy-integration-tests-core: name: "ubuntu-jammy-integration-tests-core" # Run if the ancestor jobs were successful/skipped and building clp was successful. - if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" needs: "ubuntu-jammy-binaries" runs-on: *runner env: From a048c195fe6313c807f2ed92fe5d00809640f4de Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 30 Jul 2026 17:11:15 -0400 Subject: [PATCH 33/38] Update docs. --- .github/workflows/clp-artifact-build.yaml | 1 + docs/src/dev-docs/tooling-gh-workflows.md | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index ca35b2edac..25980498bc 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -729,6 +729,7 @@ jobs: ubuntu-jammy-integration-tests-core: name: "ubuntu-jammy-integration-tests-core" # Run if the ancestor jobs were successful/skipped and building clp was successful. + if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" needs: "ubuntu-jammy-binaries" runs-on: *runner env: diff --git a/docs/src/dev-docs/tooling-gh-workflows.md b/docs/src/dev-docs/tooling-gh-workflows.md index 981da79537..5de003b392 100644 --- a/docs/src/dev-docs/tooling-gh-workflows.md +++ b/docs/src/dev-docs/tooling-gh-workflows.md @@ -8,7 +8,7 @@ builds, testing, and linting. We briefly describe each workflow below. This workflow is responsible for: 1. building (Linux) container images containing CLP-core's dependencies, -2. building CLP-core and running its unit tests, and +2. building CLP-core and running its unit & integration tests, and 3. building a container image containing CLP's package components. To minimize build times, the jobs in the workflow are organized in the directed acyclic graph (DAG) @@ -47,6 +47,7 @@ flowchart LR ubuntu-jammy-deps-image --> ubuntu-jammy-binaries ubuntu-jammy-deps-image --> package-image ubuntu-jammy-binaries --> ubuntu-jammy-binaries-image + ubuntu-jammy-binaries --> ubuntu-jammy-integration-tests-core ::: Arrows between jobs indicate a dependency. The jobs are as follows: @@ -80,6 +81,8 @@ Arrows between jobs indicate a dependency. The jobs are as follows: core's unit tests. * `ubuntu-jammy-binaries-image`: Builds an Ubuntu Jammy container image containing CLP-core's binaries built in the `ubuntu-jammy-binaries` job. +* `ubuntu-jammy-integration-tests-core`: Runs CLP-core's integration tests using the binaries built + in the `ubuntu-jammy-binaries` job. The test logs are uploaded at the end of the test run. When the PR or commit doesn't change any of the files that affect CLP's dependencies (or the dependency container images), then the dependency container images won't be rebuilt; instead the From a65e9880ab24986973e31738e823c23c18a2de8a Mon Sep 17 00:00:00 2001 From: Quinn Date: Tue, 4 Aug 2026 14:32:10 +0000 Subject: [PATCH 34/38] Address Kirk's review. --- .github/workflows/clp-artifact-build.yaml | 24 ++++++++++++++--------- docs/src/dev-docs/tooling-gh-workflows.md | 4 ++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 25980498bc..914a68920b 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -47,6 +47,7 @@ jobs: ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" clp_changed: "${{steps.filter.outputs.clp}}" yscope_clp_core_changed: "${{steps.filter.outputs.yscope-clp-core}}" + integration_tests_changed: "${{steps.filter.outputs.integration-tests}}" steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 with: @@ -103,7 +104,6 @@ jobs: - "components/core/src/**" - "components/core/tests/**" - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" - - "integration-tests/**" - "taskfile.yaml" - "taskfiles/**" - "tools/scripts/deps-download/**" @@ -114,6 +114,11 @@ jobs: - "taskfile.yaml" - "taskfiles/**" - "python-wheels/yscope-clp-core/**" + integration-tests: + - ".github/actions/**" + - ".github/workflows/clp-artifact-build.yaml" + - ".gitmodules" + - "integration-tests/**" centos-stream-9-deps-image: name: "centos-stream-9-deps-image" @@ -529,7 +534,11 @@ jobs: # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. if: >- success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + || (!cancelled() && !failure() && ( + needs.filter-relevant-changes.outputs.clp_changed == 'true' + || needs.filter-relevant-changes.outputs.integration_tests_changed == 'true' + ) + ) needs: - "filter-relevant-changes" - "ubuntu-jammy-x86_64-deps-image" @@ -727,9 +736,6 @@ jobs: )}} ubuntu-jammy-integration-tests-core: - name: "ubuntu-jammy-integration-tests-core" - # Run if the ancestor jobs were successful/skipped and building clp was successful. - if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" needs: "ubuntu-jammy-binaries" runs-on: *runner env: @@ -738,8 +744,8 @@ jobs: steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 with: - submodules: "recursive" persist-credentials: false + submodules: "recursive" - name: "Download the binaries artifact" uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" @@ -754,15 +760,15 @@ jobs: tar xf clp.tar rm clp.tar - - name: "Install uv" - uses: "astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990" # v8.3.2 + - uses: "./tools/yscope-dev-utils/exports/github/actions/install-uv" - name: "Install MariaDB Connector/C" run: "sudo apt-get update && sudo apt-get install -y libmariadb-dev" - - name: "Run core integration tests" + - name: "Run the integration tests for CLP's core" working-directory: "integration-tests" env: + # Tell pytest where the CLP binaries are, and where it should store the test logs. CLP_BUILD_DIR: "../${{env.BUILD_DIR}}" CLP_CORE_BINS_DIR: >- ../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} diff --git a/docs/src/dev-docs/tooling-gh-workflows.md b/docs/src/dev-docs/tooling-gh-workflows.md index 5de003b392..e619fd4ea0 100644 --- a/docs/src/dev-docs/tooling-gh-workflows.md +++ b/docs/src/dev-docs/tooling-gh-workflows.md @@ -8,7 +8,7 @@ builds, testing, and linting. We briefly describe each workflow below. This workflow is responsible for: 1. building (Linux) container images containing CLP-core's dependencies, -2. building CLP-core and running its unit & integration tests, and +2. building CLP-core and running its unit & integration tests, and 3. building a container image containing CLP's package components. To minimize build times, the jobs in the workflow are organized in the directed acyclic graph (DAG) @@ -82,7 +82,7 @@ Arrows between jobs indicate a dependency. The jobs are as follows: * `ubuntu-jammy-binaries-image`: Builds an Ubuntu Jammy container image containing CLP-core's binaries built in the `ubuntu-jammy-binaries` job. * `ubuntu-jammy-integration-tests-core`: Runs CLP-core's integration tests using the binaries built - in the `ubuntu-jammy-binaries` job. The test logs are uploaded at the end of the test run. + in the `ubuntu-jammy-binaries` job, and then uploads the logs from the tests. When the PR or commit doesn't change any of the files that affect CLP's dependencies (or the dependency container images), then the dependency container images won't be rebuilt; instead the From 5afd24a44de3295dc231c2f14c867f62a4a6ef6e Mon Sep 17 00:00:00 2001 From: Quinn Date: Tue, 4 Aug 2026 16:31:46 +0000 Subject: [PATCH 35/38] Further Kirk comments. --- .github/workflows/clp-artifact-build.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 914a68920b..fd4657a7e2 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -772,8 +772,7 @@ jobs: CLP_BUILD_DIR: "../${{env.BUILD_DIR}}" CLP_CORE_BINS_DIR: >- ../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} - HOME: "/tmp" - run: "uv run pytest -m 'smoke or core'" + run: "uv run pytest -m 'core'" - name: "Upload the core integration test logs" if: "always()" From 454b9c9481e2fe2634f67502a2cc02a8c45fdfd5 Mon Sep 17 00:00:00 2001 From: Quinn Date: Tue, 4 Aug 2026 16:33:13 +0000 Subject: [PATCH 36/38] Further Kirk comments. --- .github/workflows/clp-artifact-build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index fd4657a7e2..298e131161 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -761,6 +761,8 @@ jobs: rm clp.tar - uses: "./tools/yscope-dev-utils/exports/github/actions/install-uv" + with: + version: "0.8" - name: "Install MariaDB Connector/C" run: "sudo apt-get update && sudo apt-get install -y libmariadb-dev" From 72e56a0101aa6183f0db6317904160ddf6fc72a8 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 6 Aug 2026 09:21:24 -0400 Subject: [PATCH 37/38] Address Kirk's comments. --- .github/workflows/clp-artifact-build.yaml | 1 - integration-tests/.pytest.ini | 1 - integration-tests/tests/test_clp_native_py_project_imports.py | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 298e131161..ceb7882688 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -770,7 +770,6 @@ jobs: - name: "Run the integration tests for CLP's core" working-directory: "integration-tests" env: - # Tell pytest where the CLP binaries are, and where it should store the test logs. CLP_BUILD_DIR: "../${{env.BUILD_DIR}}" CLP_CORE_BINS_DIR: >- ../${{env.BUILD_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index e4a42810c0..0cf24cf095 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -30,5 +30,4 @@ markers = core: mark tests that test the CLP core binaries package: mark tests that use the CLP package search: mark tests that test search - smoke: mark tests that are smoke tests startstop: mark tests that test start-stop diff --git a/integration-tests/tests/test_clp_native_py_project_imports.py b/integration-tests/tests/test_clp_native_py_project_imports.py index b119452f42..6005df4400 100644 --- a/integration-tests/tests/test_clp_native_py_project_imports.py +++ b/integration-tests/tests/test_clp_native_py_project_imports.py @@ -7,7 +7,6 @@ from job_orchestration.scheduler.constants import CompressionJobStatus -@pytest.mark.smoke def test_clp_native_py_project_enum_classes() -> None: """ Verifies that the following CLP Python projects can be imported successfully by testing From 49c5c276d89fd7dee3260f2edd8c8df8fee608d3 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 6 Aug 2026 09:22:22 -0400 Subject: [PATCH 38/38] Address Kirk's comments. --- integration-tests/tests/test_clp_native_py_project_imports.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/tests/test_clp_native_py_project_imports.py b/integration-tests/tests/test_clp_native_py_project_imports.py index 6005df4400..b3809843e0 100644 --- a/integration-tests/tests/test_clp_native_py_project_imports.py +++ b/integration-tests/tests/test_clp_native_py_project_imports.py @@ -1,6 +1,5 @@ """Smoke tests to validate that CLP Python projects can be imported without errors.""" -import pytest from clp_mcp_server.constants import QueryJobType from clp_package_utils.general import JobType from clp_py_utils.clp_config import StorageEngine