diff --git a/.github/files/assemble-image-tags.js b/.github/files/assemble-image-tags.js new file mode 100644 index 0000000..96c1130 --- /dev/null +++ b/.github/files/assemble-image-tags.js @@ -0,0 +1,36 @@ +async function get_by_assoc(assoc, package_name, type, method) { + let containers + try { + core.info(`Looking up existing containers by ${type} ${assoc}/${package_name}`) + containers = (await method({[type]: assoc, package_type: "container", package_name})).data; + } catch (e) { + containers = []; + console.error(e); + } + return containers +} + +async function get_containers(assoc, package_name) { + let by_org = await get_by_assoc(assoc, package_name, "org", github.rest.packages.getAllPackageVersionsForPackageOwnedByOrg) + let by_user = await get_by_assoc(assoc, package_name, "username", github.rest.packages.getAllPackageVersionsForPackageOwnedByUser) + return by_org.concat(by_user) +} + +async function main(rockMetas){ + const owner = context.repo.owner + const metas = await Promise.all( + rockMetas.map( + async meta => { + const versions = await get_containers(owner, meta.name) + const rockVersion_ = meta.version + "-ck" + const patchRev = versions.reduce((partial, v) => + partial + v.metadata.container.tags.filter(t => t.startsWith(rockVersion_)).length, 0 + ) + meta.version = rockVersion_ + patchRev + core.info(`Number of containers tagged ${owner}/${meta.name}/${meta.version}: ${patchRev}`) + core.info(`Tagging image ${meta.image} with ${meta.version}`) + return meta + } + )) + core.setOutput('rock-metas', JSON.stringify(metas)) +} \ No newline at end of file diff --git a/.github/files/create-and-push-manifest.js b/.github/files/create-and-push-manifest.js new file mode 100644 index 0000000..4f9b0d3 --- /dev/null +++ b/.github/files/create-and-push-manifest.js @@ -0,0 +1,73 @@ +class RockImage { + constructor (image, arch) { + this.image = image + this.arch = arch + } + + async import_image() { + console.info(` ⏬ pull image: ${this.image}`) + await exec.exec("docker", ["pull", this.image]) + } + + async annotate(target) { + console.info(` 🖌️ annotate manifest: ${target} ${this.arch}`) + await exec.exec("docker", ["manifest", "annotate", target, this.image, `--arch=${this.arch}`]) + } +} + +class RockComponent { + constructor (name, version, dryRun) { + this.name = name + this.version = version + this.dryRun = dryRun + this.imageVer = `${this.name}:${this.version}` + this.images = [] + } + + async create_manifest(target) { + const archs = this.images.map(i => i.arch) + const images = this.images.map(i => i.image) + console.info(` 📄 create manifest: ${target} ${archs.join(",")}`) + await exec.exec("docker", ["manifest", "create", target, images.join(' ')]) + } + + async push_manifest(target) { + console.info(` ⏫ push manifest: ${target}`) + console.info(`docker manifest push ${target}`) + if (this.dryRun != true ){ + await exec.exec("docker", ["manifest", "push", target]) + } else { + console.info(`Not pushing manifest ${target} -- because dryRun: ${this.dryRun}`) + } + } + + async craft_manifest(target) { + for (const image of this.images) { + await image.import_image() + } + const targetImage = `${target.trim('/')}/${this.imageVer}` + await exec.exec("docker", ["manifest", "rm", targetImage], {ignoreReturnCode: true}) + await this.create_manifest(targetImage) + + for (const image of this.images) { + await image.annotate(targetImage) + } + await this.push_manifest(targetImage) + } +} + +async function main(rockMetas, registry, dryRun) { + const owner = context.repo.owner + const metas = rockMetas + const containers = {} + for (const meta of metas) { + if (!containers.hasOwnProperty(meta.name)) { + containers[meta.name] = new RockComponent(meta.name, meta.version, dryRun) + } + containers[meta.name].images.push(new RockImage(meta.image, meta.arch)) + } + for (const component of Object.values(containers)) { + console.info(`🖥️ Assemble Multiarch Image: ${component.name}`) + await component.craft_manifest(`${registry}/${owner}`) + } +} diff --git a/.github/workflows/assemble_multiarch_image.yaml b/.github/workflows/assemble_multiarch_image.yaml new file mode 100644 index 0000000..fdefdd1 --- /dev/null +++ b/.github/workflows/assemble_multiarch_image.yaml @@ -0,0 +1,58 @@ +name: Assemble Multiarch Manifest + +on: + workflow_call: + inputs: + rock-metas: + description: List of maps featuring the built {name, version, path, arch, image} + type: string + default: "[]" + registry: + description: Container Registrying top-level domain + type: string + default: ghcr.io + dry-run: + description: Don't actually push the manifest, just print what would be pushed + type: string + default: true + +jobs: + create-multiarch-manifest: + name: Create Mulitarch Manifest + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4.1.1 + - id: assemble-image-tags-js + uses: juliangruber/read-file-action@v1 + with: + path: .github/files/assemble-image-tags.js + trim: true + - name: Assemble Image Tags + id: assemble-image-tags + uses: actions/github-script@v7.0.1 + with: + script: | + const rockMetas = JSON.parse(`${{ inputs.rock-metas }}`) + ${{ steps.assemble-image-tags-js.outputs.content }} + await main(rockMetas) + - name: Login to Container Registry + uses: docker/login-action@v3.0.0 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: create-and-push-manifest-js + uses: juliangruber/read-file-action@v1 + with: + path: .github/files/create-and-push-manifest.js + trim: true + - name: Create and Push Manifests + id: create-and-push-manifest + uses: actions/github-script@v7.0.1 + with: + script: | + const registry = '${{ inputs.registry }}' + const dryRun = ${{ inputs.dry-run }} + const rockMetas = JSON.parse(`${{ steps.assemble-image-tags.outputs.rock-metas }}`) + ${{ steps.create-and-push-manifest-js.outputs.content }} + await main(rockMetas, registry, dryRun) \ No newline at end of file diff --git a/.github/workflows/build_rocks.yaml b/.github/workflows/build_rocks.yaml new file mode 100644 index 0000000..6cc93d4 --- /dev/null +++ b/.github/workflows/build_rocks.yaml @@ -0,0 +1,282 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +name: Build images + +on: + workflow_call: + inputs: + owner: + type: string + description: Registry owner to push the built images + default: "" + registry: + type: string + description: Registry to push the built images + default: "" + runs-on: + type: string + description: Image runner for building the images + default: ubuntu-22.04 + trivy-image-config: + type: string + description: Trivy YAML configuration for image testing that is checked in as part of the repo + working-directory: + type: string + description: The working directory for jobs + default: "./" + cache-action: + type: string + description: The cache action can either be "save" or "restore". + default: restore + multiarch-awareness: + type: boolean + description: Maintain the architecture labels on the container names + default: false + outputs: + images: + description: List of images built + value: ${{ jobs.get-rocks.outputs.images }} + rock-metas: + description: List of maps featuring the built {name, version, path, arch, image} + value: ${{ jobs.get-rocks.outputs.rock-metas }} + +jobs: + get-rocks: + name: Get rocks + runs-on: ubuntu-22.04 + outputs: + rock-paths: ${{ steps.gen-rock-paths-and-images.outputs.rock-paths }} + images: "${{ steps.gen-rock-paths-and-images.outputs.images }}" + rock-metas: ${{ steps.gen-rock-paths-and-images.outputs.rock-metas }} + steps: + - name: Validate inputs + run: | + if [ "${{ inputs.cache-action }}" != "save" ] && [ "${{ inputs.cache-action }}" != "restore" ]; then + echo "Invalid value for cache-action. It must be 'save' or 'restore'" + exit 1 + fi + - uses: actions/checkout@v4.1.1 + - name: Generate rock paths and images + id: gen-rock-paths-and-images + uses: actions/github-script@v7.0.1 + with: + script: | + const path = require('path') + const inputs = ${{ toJSON(inputs) }} + const workingDir = inputs['working-directory'] + const multiarch = inputs['multiarch-awareness'] + const rockcraftGlobber = await glob.create( + path.join(workingDir, '**/rockcraft.yaml') + ) + const rockPaths = [] + const images = [] + const rockMetas = [] + const defaultArch = 'amd64' + core.info(`Multiarch Awareness is ${multiarch ? "on" : "off" }`) + for (const rockcraftFile of await rockcraftGlobber.glob()) { + const rockPath = path.relative('.', path.dirname(rockcraftFile)) + core.info(`found rockcraft.yaml in ${rockPath}`) + const fileHash = await glob.hashFiles(path.join(rockPath, '**')) + const [rockName, rockVersion] = ( + await exec.getExecOutput('yq', ['.name,.version', rockcraftFile]) + ).stdout.trim().split("\n") + const platforms = ( + await exec.getExecOutput('yq', ['.platforms | keys', '-o=json', rockcraftFile]) + ).stdout.trim() + if (multiarch && platforms) { + const arches = JSON.parse(platforms) + for (arch of arches) { + const image = `${{ inputs.registry }}/${{ inputs.owner }}/${rockName}:${fileHash}-${arch}` + core.info(`generate multi-arch image name: ${image}`) + images.push(image) + rockMetas.push({name: rockName, version: rockVersion, path: rockPath, arch: arch, image: image}) + } + } else { + const image = `${{ inputs.registry }}/${{ inputs.owner }}/${rockName}:${fileHash}` + core.info(`generate image name: ${image}`) + images.push(image) + rockMetas.push({name: rockName, version: rockVersion, path: rockPath, arch: defaultArch, image: image}) + } + rockPaths.push(rockPath) + } + core.setOutput('rock-metas', JSON.stringify(rockMetas)) + core.setOutput('rock-paths', JSON.stringify(rockPaths)) + core.setOutput('images', JSON.stringify(images)) + + build-rocks: + name: Build rock + runs-on: ${{ inputs.runs-on }} + permissions: + contents: read + packages: write + needs: [get-rocks] + if: ${{ needs.get-rocks.outputs.rock-metas != '[]' }} + strategy: + matrix: + rock: ${{ fromJSON(needs.get-rocks.outputs.rock-metas) }} + steps: + - uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + - name: Extract rock information + run: | + IMAGE_ARCH="${{ matrix.rock.arch }}" + IMAGE_NAME="${{ matrix.rock.name }}" + IMAGE_BASE=$(yq '.base' "${{ matrix.rock.path }}/rockcraft.yaml") + IMAGE_BUILD_BASE=$(yq '.["build-base"] // .base' "${{ matrix.rock.path }}/rockcraft.yaml") + IMAGE_REF=${{ matrix.rock.image }} + INODE_NUM=$(ls -id ${{ matrix.rock.path }} | cut -f 1 -d " ") + ROCKCRAFT_CONTAINER_NAME=rockcraft-$IMAGE_NAME-on-$IMAGE_ARCH-for-$IMAGE_ARCH-$INODE_NUM + echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV + echo "IMAGE_BASE=$IMAGE_BASE" >> $GITHUB_ENV + echo "IMAGE_BUILD_BASE=$IMAGE_BUILD_BASE" >> $GITHUB_ENV + echo "IMAGE_REF=$IMAGE_REF" >> $GITHUB_ENV + echo "IMAGE_ARCH=$IMAGE_ARCH" >> $GITHUB_ENV + echo "ROCKCRAFT_CONTAINER_NAME=$ROCKCRAFT_CONTAINER_NAME" >> $GITHUB_ENV + - name: Generate rockcraft cache key + run: | + ROCKCRAFT_PATH="${{ matrix.rock.path }}" + ROCKCRAFT_PATH="${ROCKCRAFT_PATH%/}" + ROCKCRAFT_CACHE_KEY_BASE="$ROCKCRAFT_PATH/rockcraft-cache?name=${{ env.IMAGE_NAME }}&base=${{ env.IMAGE_BUILD_BASE }}&build-base=${{ env.IMAGE_BUILD_BASE }}" + ROCK_CACHE_KEY_BASE="$ROCKCRAFT_PATH/${{ env.IMAGE_NAME }}.rock?filehash=${{ hashFiles(format('{0}/{1}', matrix.rock.path, '**')) }}" + if [ "${{ inputs.multiarch-awareness }}" == "true" ]; then + ROCKCRAFT_CACHE_KEY_BASE="${ROCKCRAFT_CACHE_KEY_BASE}&arch=${{ env.IMAGE_ARCH }}" + ROCK_CACHE_KEY_BASE="${ROCK_CACHE_KEY_BASE}&arch=${{ env.IMAGE_ARCH }}" + fi + echo "ROCKCRAFT_CACHE_KEY=$ROCKCRAFT_CACHE_KEY_BASE&date=$(date +%Y-%m-%d)" >> $GITHUB_ENV + echo 'ROCKCRAFT_CACHE_ALT_KEYS<> $GITHUB_ENV + for d in {1..2} + do echo "$ROCKCRAFT_CACHE_KEY_BASE&date=$(date -d"-$d days" +%Y-%m-%d)" >> $GITHUB_ENV + done + echo 'EOF' >> $GITHUB_ENV + echo "ROCK_CACHE_KEY=$ROCK_CACHE_KEY_BASE=$(date +%Y-%m-%d)" >> $GITHUB_ENV + echo 'ROCK_CACHE_ALT_KEYS<> $GITHUB_ENV + for d in {1..2} + do echo "$ROCK_CACHE_KEY_BASE&date=$(date -d"-$d days" +%Y-%m-%d)" >> $GITHUB_ENV + done + echo 'EOF' >> $GITHUB_ENV + - name: Restore rock cache + if: inputs.cache-action == 'restore' + uses: actions/cache/restore@v4.0.0 + id: rock-cache + with: + path: ~/.rock-cache + key: ${{ env.ROCK_CACHE_KEY }} + restore-keys: ${{ env.ROCK_CACHE_ALT_KEYS }} + - name: Restore rockcraft container cache + if: steps.rock-cache.outputs.cache-hit != 'true' && inputs.cache-action == 'restore' + uses: actions/cache/restore@v4.0.0 + id: rockcraft-cache + with: + path: ~/.rockcraft-cache/ + key: ${{ env.ROCKCRAFT_CACHE_KEY }} + restore-keys: ${{ env.ROCKCRAFT_CACHE_ALT_KEYS }} + - name: Setup lxd + if: steps.rockcraft-cache.outputs.cache-hit == 'true' + run: | + sudo groupadd --force --system lxd + sudo usermod --append --groups lxd runner + sudo snap refresh lxd --channel latest/stable + sudo lxd init --auto + sudo iptables -P FORWARD ACCEPT + - name: Import rockcraft container cache + if: steps.rockcraft-cache.outputs.cache-hit == 'true' + working-directory: ${{ inputs.working-directory }} + run: | + sudo lxc project create rockcraft -c features.images=false -c features.profiles=false + sudo lxc --project rockcraft import ~/.rockcraft-cache/${{ env.IMAGE_NAME }}.tar ${{ env.ROCKCRAFT_CONTAINER_NAME }} + find . -exec touch '{}' ';' + - name: Build rock + if: steps.rock-cache.outputs.cache-hit != 'true' || inputs.cache-action == 'save' + uses: canonical/craft-actions/rockcraft-pack@main + with: + path: ${{ matrix.rock.path }} + - name: Generate rockcraft container cache + if: inputs.cache-action == 'save' + run: | + mkdir -p ~/.rockcraft-cache + mkdir -p ~/.rock-cache + touch ~/.rock-cache/.gitkeep + sudo lxc --project rockcraft export ${{ env.ROCKCRAFT_CONTAINER_NAME }} --compression none ~/.rockcraft-cache/${{ env.IMAGE_NAME }}.tar + - name: Delete rockcraft container cache + if: inputs.cache-action == 'save' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method DELETE \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${{ github.repository }}/actions/caches?key=$(printf %s "${{ env.ROCKCRAFT_CACHE_KEY }}"|jq -sRr @uri) || : + for key in $(echo $ROCKCRAFT_CACHE_ALT_KEYS) + do gh api \ + --method DELETE \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${{ github.repository }}/actions/caches?key=$(printf %s "$key"|jq -sRr @uri) || : + done + - name: Save rockcraft container cache + if: inputs.cache-action == 'save' + uses: actions/cache/save@v4.0.0 + with: + path: ~/.rockcraft-cache/ + key: ${{ env.ROCKCRAFT_CACHE_KEY }} + - name: Delete rock cache + if: inputs.cache-action == 'save' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method DELETE \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${{ github.repository }}/actions/caches?key=$(printf %s "${{ env.ROCK_CACHE_KEY }}"|jq -sRr @uri) || : + for key in $(echo $ROCK_CACHE_ALT_KEYS) + do gh api \ + --method DELETE \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${{ github.repository }}/actions/caches?key=$(printf %s "$key"|jq -sRr @uri) || : + done + - name: Save rock cache + if: inputs.cache-action == 'save' + uses: actions/cache/save@v4.0.0 + with: + path: ~/.rock-cache + key: ${{ env.ROCK_CACHE_KEY }} + - name: Upload rock to ${{ inputs.registry }} + if: steps.rock-cache.outputs.cache-hit != 'true' || inputs.cache-action == 'save' + run: | + skopeo --insecure-policy copy oci-archive:$(ls "${{ matrix.rock.path }}"/*.rock) docker://$IMAGE_REF --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" + - name: Run Github Trivy Image Action + uses: aquasecurity/trivy-action@0.16.1 + if: steps.rock-cache.outputs.cache-hit != 'true' || inputs.cache-action == 'save' + with: + image-ref: ${{ env.IMAGE_REF }} + trivy-config: ${{ inputs.trivy-image-config }} + exit-code: '1' + severity: 'CRITICAL,HIGH' + env: + TRIVY_USERNAME: ${{ github.actor }} + TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + - name: Check trivyignore + run: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.46.0 + if [ -f ".trivyignore" ] + then + output=$(trivy image $ROCK_IMAGE --severity HIGH,CRITICAL -q -f json --ignorefile "" | jq -r '.Results[].Vulnerabilities[].VulnerabilityID' 2>/dev/null || echo "No vulnerabilities found") + line=0 + while read CVE; + do + line=$(( line + 1 )) + if [[ "$output" != *"$CVE"* ]] && [[ ! "$CVE" =~ ^#.* ]] + then + echo "::notice file=.trivyignore,line=${line}::$CVE not present anymore, can be safely removed." + fi + done < .trivyignore + fi + env: + TRIVY_USERNAME: ${{ github.actor }} + TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + ROCK_IMAGE: ${{ env.IMAGE_REF }} diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml new file mode 100644 index 0000000..a49124a --- /dev/null +++ b/.github/workflows/pull_request.yaml @@ -0,0 +1,33 @@ +name: Push Multiarch Images +on: + pull_request: + push: + branches: + - main + +jobs: + get-runner-image: + name: Get runner image + uses: canonical/operator-workflows/.github/workflows/get_runner_image.yaml@main + with: + working-directory: ${{ inputs.working-directory }} + build-and-push-arch-specifics: + name: Push Arch Specific Images + uses: ./.github/workflows/build_rocks.yaml + needs: [get-runner-image] + with: + owner: ${{ github.repository_owner }} + registry: ghcr.io + runs-on: ${{ needs.get-runner-image.outputs.runs-on }} + trivy-image-config: "trivy.yaml" + working-directory: ${{ inputs.working-directory }} + multiarch-awareness: true + cache-action: ${{ (github.event_name == 'push') && 'save' || 'restore' }} + build-and-push-multiarch-manifest: + name: Push Multiarch Manifest + uses: ./.github/workflows/assemble_multiarch_image.yaml + needs: [build-and-push-arch-specifics] + with: + rock-metas: ${{ needs.build-and-push-arch-specifics.outputs.rock-metas }} + registry: ghcr.io + dry-run: ${{ github.event_name != 'push' }} diff --git a/trivy.yaml b/trivy.yaml new file mode 100644 index 0000000..6e79d6f --- /dev/null +++ b/trivy.yaml @@ -0,0 +1,3 @@ +timeout: 20m +scan: + offline-scan: true \ No newline at end of file