feat: Add support for building execution image for ubuntu-noble.

.github/workflows/clp-core-build.yaml

@@ -46,6 +46,7 @@ jobs:
     outputs:
       centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}"
       ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}"
+      ubuntu_noble_image_changed: "${{steps.filter.outputs.ubuntu_noble_image}}"
       clp_changed: "${{steps.filter.outputs.clp}}"
     steps:
       - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
@@ -82,6 +83,12 @@ jobs:
               - "components/core/tools/scripts/lib_install/*.sh"
               - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**"
               - "components/core/tools/scripts/lib_install/ubuntu-jammy/**"
+            ubuntu_noble_image:
+              - ".github/actions/**"
+              - ".github/workflows/clp-core-build.yaml"
+              - "components/core/tools/scripts/lib_install/*.sh"
+              - "components/core/tools/docker-images/clp-env-base-ubuntu-noble/**"
+              - "components/core/tools/scripts/lib_install/ubuntu-noble/**"
             clp:
               - ".github/actions/**"
               - ".github/workflows/clp-core-build.yaml"
@@ -147,6 +154,32 @@ jobs:
             ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}
           token: "${{secrets.GITHUB_TOKEN}}"
 
+  ubuntu-noble-deps-image:
+    name: "ubuntu-noble-deps-image"
+    if: "needs.filter-relevant-changes.outputs.ubuntu_noble_image_changed == 'true'"
+    needs: "filter-relevant-changes"
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
+        with:
+          submodules: "recursive"
+
+      - name: "Work around actions/runner-images/issues/6775"
+        run: "chown $(id -u):$(id -g) -R ."
+        shell: "bash"
+
+      - uses: "./.github/actions/clp-core-build-containers"
+        env:
+          OS_NAME: "ubuntu-noble"
+        with:
+          image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          docker_context: "components/core"
+          docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\
+            /Dockerfile"
+          push_deps_image: >-
+            ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}
+          token: "${{secrets.GITHUB_TOKEN}}"
+
   centos-stream-9-binaries:
     # Run if the ancestor jobs succeeded OR they were skipped and clp was changed.
     if: >-
@@ -247,6 +280,67 @@ jobs:
           path: "${{steps.copy_binaries.outputs.output_dir}}"
           retention-days: 1
 
+  ubuntu-noble-binaries:
+    # Run if the ancestor jobs succeeded OR they were skipped and clp was changed.
+    if: >-
+      success()
+      || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true')
+    needs:
+      - "filter-relevant-changes"
+      - "ubuntu-noble-deps-image"
+    strategy:
+      matrix:
+        include:
+          - use_shared_libs: true
+            upload_binaries: false
+          - use_shared_libs: false
+            upload_binaries: true
+    env:
+      OS_NAME: "ubuntu-noble"
+    name: "ubuntu-noble-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
+        with:
+          submodules: "recursive"
+
+      - name: "Work around actions/runner-images/issues/6775"
+        run: "chown $(id -u):$(id -g) -R ."
+        shell: "bash"
+
+      - uses: "./.github/actions/run-on-image"
+        with:
+          image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_published_image: >-
+            ${{needs.filter-relevant-changes.outputs.ubuntu_noble_image_changed == 'false'
+            || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
+          run_command: >-
+            task deps:core
+            && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py
+            ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}}
+            --source-dir /mnt/repo/components/core
+            --build-dir /mnt/repo/components/core/build
+            --num-jobs $(getconf _NPROCESSORS_ONLN)
+
+      - if: "matrix.upload_binaries == true"
+        id: "copy_binaries"
+        run: |-
+          output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
+          echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT"
+
+          mkdir -p "${output_dir}"
+          cd "$GITHUB_WORKSPACE/components/core/build"
+          tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable
+        shell: "bash"
+
+      - if: "matrix.upload_binaries == true"
+        uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02"
+        with:
+          name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
+          path: "${{steps.copy_binaries.outputs.output_dir}}"
+          retention-days: 1
+
   ubuntu-jammy-binaries-image:
     name: "ubuntu-jammy-binaries-image"
     # Run if the ancestor jobs were successful/skipped and building clp was successful.
@@ -310,6 +404,69 @@ jobs:
           tags: "${{steps.core_image_meta.outputs.tags}}"
           labels: "${{steps.core_image_meta.outputs.labels}}"
 
+  ubuntu-noble-binaries-image:
+    name: "ubuntu-noble-binaries-image"
+    # Run if the ancestor jobs were successful/skipped and building clp was successful.
+    if: "!cancelled() && !failure() && needs.ubuntu-noble-binaries.result == 'success'"
+    needs: "ubuntu-noble-binaries"
+    runs-on: "ubuntu-latest"
+    env:
+      OS_NAME: "ubuntu-noble"
+      TMP_OUTPUT_DIR: "/tmp"
+    steps:
+      - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
+        with:
+          submodules: "recursive"
+
+      - name: "Work around actions/runner-images/issues/6775"
+        run: "chown $(id -u):$(id -g) -R ."
+        shell: "bash"
+
+      - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e"
+        with:
+          name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
+          path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
+
+      - name: "Untar binaries"
+        working-directory: >-
+          ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}
+        run: |-
+          tar xf clp.tar
+          rm clp.tar
+
+      - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772"
+        with:
+          registry: "ghcr.io"
+          username: "${{github.actor}}"
+          password: "${{secrets.GITHUB_TOKEN}}"
+
+      - name: "Sanitize the repo's name"
+        id: "sanitize_repo_name"
+        run: |-
+          # Docker doesn't support repository names with uppercase characters, so we convert to
+          # lowercase here.
+          lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')
+          echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT"
+        shell: "bash"
+
+      - id: "core_image_meta"
+        uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804"
+        with:
+          images: >-
+            ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}}
+          tags: "type=raw,value=${{github.ref_name}}"
+
+      # Only publish the image if this workflow was triggered by a push to `main`.
+      # NOTE: We run the rest of the job to test that the binaries were uploaded correctly.
+      - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'"
+        uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4"
+        with:
+          context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
+          file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile"
+          push: true
+          tags: "${{steps.core_image_meta.outputs.tags}}"
+          labels: "${{steps.core_image_meta.outputs.labels}}"
+
   ubuntu-jammy-lint:
     name: "ubuntu-jammy-lint"
     # Run if the ancestor jobs succeeded OR they were skipped and clp was changed.
@@ -372,3 +529,66 @@ jobs:
             .task/checksum/utils-cpp-lint-clang-tidy-*
             build/lint-clang-tidy
           key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}"
+
+  ubuntu-noble-lint:
+    name: "ubuntu-noble-lint"
+    # Run if the ancestor jobs succeeded OR they were skipped and clp was changed.
+    if: >-
+      success()
+      || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true')
+    needs:
+      - "filter-relevant-changes"
+      - "ubuntu-noble-deps-image"
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
+        with:
+          # Fetch history so that the `clang-tidy-diff` task can compare against the main branch.
+          fetch-depth: 0
+          submodules: "recursive"
+
+      - name: "Work around actions/runner-images/issues/6775"
+        run: "chown $(id -u):$(id -g) -R ."
+        shell: "bash"
+
+      # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full
+      # on all files.
+      - if: "'schedule' != github.event_name"
+        name: "Restore lint:check-cpp-static-full cache"
+        id: "cache-restore-lint-check-cpp-static-full"
+        uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684"
+        with:
+          path: |
+            .task/checksum/lint-check-cpp-static-full
+            .task/checksum/utils-cpp-lint-clang-tidy-*
+            build/lint-clang-tidy
+
+          # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy
+          # violations.
+          key: "main-branch-ubuntu-noble-lint:check-cpp-static-full"
+
+      - uses: "./.github/actions/run-on-image"
+        env:
+          OS_NAME: "ubuntu-noble"
+        with:
+          image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_published_image: >-
+            ${{needs.filter-relevant-changes.outputs.ubuntu_noble_image_changed == 'false'
+            || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
+          # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only.
+          # run_command: >-
+          #   task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}}
+          run_command: "task lint:check-cpp-full"
+
+      # Cache the source file checksums and the generated files (logs) for the
+      # lint:check-cpp-static-full task, but only if it runs successfully on the main branch.
+      # NOTE: If we don't cache the generated files, the task will re-run to generate them.
+      - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref"
+        name: "Update lint:check-cpp-static-full cache"
+        uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684"
+        with:
+          path: |
+            .task/checksum/lint-check-cpp-static-full
+            .task/checksum/utils-cpp-lint-clang-tidy-*
+            build/lint-clang-tidy
+          key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}"

.github/workflows/clp-execution-image-build.yaml

@@ -27,6 +27,7 @@ jobs:
     runs-on: "ubuntu-latest"
     outputs:
       ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}"
+      ubuntu_noble_image_changed: "${{steps.filter.outputs.ubuntu_noble_image}}"
     steps:
       - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
         with:
@@ -46,6 +47,10 @@ jobs:
               - ".github/actions/clp-execution-image-build/action.yaml"
               - ".github/workflows/clp-execution-image-build.yaml"
               - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*"
+            ubuntu_noble_image:
+              - ".github/actions/clp-execution-image-build/action.yaml"
+              - ".github/workflows/clp-execution-image-build.yaml"
+              - "tools/docker-images/clp-execution-base-ubuntu-noble/**/*"
 
   ubuntu-jammy-execution-image:
     name: "ubuntu-jammy-execution-image"
@@ -68,3 +73,25 @@ jobs:
           image_registry_password: "${{secrets.GITHUB_TOKEN}}"
           platform_id: "ubuntu"
           platform_version_id: "jammy"
+
+  ubuntu-noble-execution-image:
+    name: "ubuntu-noble-execution-image"
+    if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_noble_image_changed"
+    needs: "filter-relevant-changes"
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
+        with:
+          submodules: "recursive"
+
+      - name: "Workaround actions/runner-images/issues/6775"
+        shell: "bash"
+        run: "chown $(id -u):$(id -g) -R ."
+
+      - uses: "./.github/actions/clp-execution-image-build"
+        with:
+          image_registry: "ghcr.io"
+          image_registry_username: "${{github.actor}}"
+          image_registry_password: "${{secrets.GITHUB_TOKEN}}"
+          platform_id: "ubuntu"
+          platform_version_id: "noble"

components/clp-py-utils/clp_py_utils/sql_adapter.py

@@ -83,7 +83,7 @@ def create_mysql_connection(
 
     def create_mariadb_connection(
         self, disable_localhost_socket_connection: bool = False
-    ) -> mariadb.connection:
+    ) -> mariadb.Connection:
         try:
             connection = mariadb.connect(
                 **self.database_config.get_mysql_connection_params(

components/clp-py-utils/pyproject.toml

@@ -13,7 +13,7 @@ python = "^3.8 || ^3.10"
 boto3 = "^1.35.81"
 # mariadb version must be compatible with libmariadev installed in runtime env.
 # See https://mariadb.com/docs/server/connect/programming-languages/python/install/#Dependencies
-mariadb = "~1.0.11"
+mariadb = "~1.1.10"
 mysql-connector-python = "^8.2.0"
 pydantic = "^1.10.13"
 python-dotenv = "^1.0.1"

components/core/tools/docker-images/clp-core-ubuntu-noble/Dockerfile

@@ -0,0 +1,29 @@
+FROM ubuntu:noble AS base
+
+# Install runtime dependencies
+# TODO: Investigate why libssl-dev is a hidden dependency of clp-s
+RUN apt-get update \
+    && apt-get install -y \
+    libcurl4 \
+    libmariadb-dev \
+    libssl-dev
+
+# Remove cached files
+RUN apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+WORKDIR /clp
+
+ADD ./clg ./
+ADD ./clp ./
+ADD ./clp-s ./
+ADD ./glt ./
+ADD ./make-dictionaries-readable ./
+
+# Flatten the image
+FROM scratch
+COPY --from=base / /
+
+WORKDIR /clp
+
+CMD bash

components/core/tools/docker-images/clp-core-ubuntu-noble/build.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Exit on any error
+set -e
+
+# Error on undefined variable
+set -u
+
+cUsage="Usage: ${BASH_SOURCE[0]} <clp-core-build-dir>"
+if [ "$#" -lt 1 ]; then
+    echo "${cUsage}"
+    exit
+fi
+build_dir="$1"
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+build_cmd=(
+    docker build
+    --tag clp-core-x86-ubuntu-noble:dev
+    "$build_dir"
+    --file "${script_dir}/Dockerfile"
+)
+
+if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ;
+then
+    build_cmd+=(
+        --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)"
+        --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)"
+    )
+fi
+
+"${build_cmd[@]}"