From 1327cedcf65be0592bd5e3454b77d9c86d9c27e5 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Mon, 15 Sep 2025 16:50:03 -0500 Subject: [PATCH 01/28] Initial commit with docker-build-push-image, docker-export-digest, and docker-import-digests-push-manifest --- actions/docker-build-push-image/action.yaml | 327 ++++++++++++++++++ .../dockerhub-setup-project-vars.sh | 7 + .../gar-setup-project-vars.sh | 56 +++ actions/docker-export-digest/action.yaml | 42 +++ .../action.yaml | 83 +++++ 5 files changed, 515 insertions(+) create mode 100644 actions/docker-build-push-image/action.yaml create mode 100644 actions/docker-build-push-image/dockerhub-setup-project-vars.sh create mode 100644 actions/docker-build-push-image/gar-setup-project-vars.sh create mode 100644 actions/docker-export-digest/action.yaml create mode 100644 actions/docker-import-digests-push-manifest/action.yaml diff --git a/actions/docker-build-push-image/action.yaml b/actions/docker-build-push-image/action.yaml new file mode 100644 index 000000000..df12f0622 --- /dev/null +++ b/actions/docker-build-push-image/action.yaml @@ -0,0 +1,327 @@ +name: Push to artifact registry +description: Composite action to push a docker image to GAR or DockerHub + +inputs: + build-args: + description: | + List of arguments necessary for the Docker image to be built. + default: "" + build-contexts: + description: | + List of additional build contexts (e.g., name=path) + required: false + buildkitd-config: + description: | + Configum buildkitium descriptium + default: "" + buildkitd-config-inline: + description: | + Inliumium configutorium buildkitium descriptium + default: "" + cache-from: + description: | + Where cache should be fetched from + required: false + default: "type=gha" + cache-to: + description: | + Where cache should be stored to + required: false + default: "type=gha,mode=max" + context: + description: | + Path to the Docker build context. + default: "." + delete-credentials-file: #TODO: Expand on this + description: | + Delete the credentials file after the action is finished. + If you want to keep the credentials file for a later step, set this to false. + default: "true" + dockerhub-repository: + description: | + Ipsum dockerhubium + default: "${{ github.repository }}" + docker-buildx-driver: + description: | + The driver to use for Docker Buildx + required: false + default: "docker-container" + file: + description: | + The dockerfile to use. + required: false + gar-registry: + description: | + Google Artifact Registry to store docker images in. + default: "us-docker.pkg.dev" + gar-repository: + description: | + Override the 'repo_name' used to construct the GAR repository name. + Only necessary when the GAR includes a repo name that doesn't match the GitHub repo name. + Default: `docker-${GitHub Repo Name}-${gar-environment}` + default: "" + gar-environment: + description: | + Environment for pushing artifacts (can be either dev or prod). + default: "dev" + gar-image: + description: | + Name of the image to build. + Default: `${GitHub Repo Name}` + default: "" + include-tags-in-push: + description: | # TODO: Expand + Can't push tags with a digest, need a way to disable + default: "true" + labels: + description: | + List of custom labels to add to the image as metadata. + required: false + load: + description: | + Whether to load the built image into the local docker daemon. + required: false + default: "false" + outputs: + description: | # TODO Desc + Ipsum factum explainum. + required: false + default: "" + platforms: + description: | + List of platforms to build the image for + required: false + push: + description: | + Whether to push the image to the configured registries. + required: false + registries: + description: | + List of registries to build images for. + default: "" + secrets: + description: | + Secrets to expose to the build. Only needed when authenticating to private repositories outside the repository in which the image is being built. + required: false + ssh: + description: | + List of SSH agent socket or keys to expose to the build + tags: + description: | + List of Docker tags to be pushed. + required: true + target: + description: | + Sets the target stage to build + required: false + +outputs: + annotations: + description: "Generated annotations (from docker/metadata-action)" + value: ${{ steps.meta.outputs.annotations }} + digest: + description: "Image digest (from docker/build-push-action)" + value: ${{ steps.build.outputs.digest }} + imageid: + description: "Image ID (from docker/build-push-action)" + value: ${{ steps.build.outputs.imageid }} + images: + description: "Comma separated list of the images that were built" + value: ${{ steps.setup-vars.outputs.images }} + json: + description: "JSON output of tags and labels (from docker/metadata-action)" + value: ${{ steps.meta.outputs.json }} + labels: + description: "Generated Docker labels (from docker/metadata-action)" + value: ${{ steps.meta.outputs.labels }} + metadata: + description: "Build result metadata (from docker/build-push-action)" + value: ${{ steps.build.outputs.metadata }} + metadatajson: + description: "Metadata JSON (from docker/metadata)" + value: ${{ steps.meta.outputs.json }} + version: + description: "Generated Docker image version (from docker/metadata-action)" + value: ${{ steps.meta.outputs.version }} + tags: + description: "Generated Docker tags (from docker/metadata-action)" + value: ${{ steps.meta.outputs.tags }} + +runs: + using: composite + steps: + - name: Checkout shared-workflows + env: + action_repo: ${{ github.action_repository }} + action_ref: ${{ github.action_ref }} + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + repository: ${{ env.action_repo }} + ref: ${{ env.action_ref }} + path: _shared-workflows-docker-build-push-image + persist-credentials: false + + - name: Set registries + id: registries + shell: bash + env: + REGISTRIES: ${{ inputs.registries }} + run: | + for r in ${REGISTRIES//,/ }; do + echo "include-${r}=true" | tee -a "${GITHUB_OUTPUT}" + done + + - name: Setup GAR variables + id: setup-gar-vars + if: ${{ steps.registries.outputs.include-gar == 'true' }} + shell: bash + env: + GAR_REPO: ${{ inputs.gar-repository }} + ENVIRONMENT: ${{ inputs.gar-environment }} + GAR_IMAGE: ${{ inputs.gar-image }} + REGISTRY: ${{ inputs.gar-registry }} + GH_REPO: ${{ github.repository }} + run: | + chmod +x ./_shared-workflows-docker-build-push-image/actions/docker-build-push-image/gar-setup-project-vars.sh + ./_shared-workflows-docker-build-push-image/actions/docker-build-push-image/gar-setup-project-vars.sh + + - name: Setup DockerHub variables + if: ${{ steps.registries.outputs.include-dockerhub == 'true' }} + id: setup-dockerhub-vars + shell: bash + env: + DOCKERHUB_IMAGE: ${{ inputs.dockerhub-repository }} + run: | + chmod +x ./_shared-workflows-docker-build-push-image/actions/docker-build-push-image/dockerhub-setup-project-vars.sh + ./_shared-workflows-docker-build-push-image/actions/docker-build-push-image/dockerhub-setup-project-vars.sh + + - name: Finalize build vars + id: setup-vars + shell: bash + env: + GAR_IMAGE: ${{ steps.setup-gar-vars.outputs.image }} + DOCKERHUB_IMAGE: ${{ steps.setup-dockerhub-vars.outputs.image }} + run: | + images="" + for image in "${GAR_IMAGE}" "${DOCKERHUB_IMAGE}" + do + # if an image is not the emptry string, add it to the list + if [ -n "${image}" ]; then + + # if we already have any images, prefix a comma + if [ -n "${images}" ]; then + images="${images},${image}" + else + images="${image}" + fi + fi + done + echo "images=${images}" | tee -a "${GITHUB_OUTPUT}" + + if [ -z "${images}" ]; then + echo "::warning::No registries have been selected, no images will be pushed" + echo "images=dry-run-image" | tee -a "${GITHUB_OUTPUT}" + fi + + - name: Login to GAR + if: ${{ inputs.push == 'true' && steps.registries.outputs.include-gar == 'true' }} + uses: ./_shared-workflows-docker-build-push-image/actions/login-to-gar + + - name: Login to DockerHub + if: ${{ inputs.push == 'true' && steps.registries.outputs.include-dockerhub == 'true' }} + uses: ./_shared-workflows-docker-build-push-image/actions/dockerhub-login # TODO: Write up an issue to convert this to actions/login-to-dockerhub + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: ${{ steps.setup-vars.outputs.images }} + tags: ${{ inputs.tags }} + + - name: Setup buildkitd-config + id: buildkitd-config + shell: sh + env: + BUILDKITD_CONFIG_INLINE: ${{ inputs.buildkitd-config-inline }} + BUILDKITD_CONFIG: ${{ inputs.buildkitd-config }} + DEFAULT_BUILDKITD_CONFIG: ${{ runner.environment == 'self-hosted' && '/etc/buildkitd.toml' || '' }} + run: | + # This step does the following: + # if buildkitd-config-inline != "", use that + # elif buildkitd-config != "", use that + # else, use buildkitd-config default config if on self hosted runners + + buildkitd_config="" + buildkitd_config_inline="" + + if [ -n "${BUILDKITD_CONFIG_INLINE}" ]; then + buildkitd_config_inline="${BUILDKITD_CONFIG_INLINE}" + elif [ -n "${BUILDKITD_CONFIG}" ]; then + buildkitd_config="${BUILDKITD_CONFIG}" + else + buildkitd_config="${DEFAULT_BUILDKITD_CONFIG}" + fi + + echo "buildkitd-config-inline=${buildkitd_config_inline}" | tee -a "${GITHUB_OUTPUT}" + echo "buildkitd-config=${buildkitd_config}" | tee -a "${GITHUB_OUTPUT}" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + with: + driver: ${{ inputs.docker-buildx-driver }} + version: latest # see https://github.com/docker/build-push-action/issues/1345#issuecomment-2770572479 + buildkitd-config: ${{ steps.buildkitd-config.outputs.buildkitd-config }} + buildkitd-config-inline: ${{ steps.buildkitd-config.outputs.buildkitd-config-inline }} + + # The `context` input is flagged by Zizmor as a [sink]. This means that with + # the upstream action the user's input to the input ends up in an output, + # and so if it's not handled properly, it could lead to a template injection + # attack. In this action, we do pass this back out via our `metadata` + # output. However, we consider ourselves a proxy, so in that case our job is + # to warn users but not to take any action. + # + # [sink]: https://github.blog/security/application-security/how-to-secure-your-github-actions-workflows-with-codeql/#models + - name: Build the container # zizmor: ignore[template-injection] + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + id: build + with: + build-args: ${{ inputs.build-args }} + build-contexts: ${{ inputs.build-contexts }} + cache-from: ${{ inputs.cache-from }} + cache-to: ${{ inputs.cache-to }} + context: ${{ inputs.context }} + file: ${{ inputs.file }} + labels: ${{ inputs.labels }} + load: ${{ inputs.load == 'true' }} + platforms: ${{ inputs.platforms }} + outputs: ${{ inputs.outputs }} + push: ${{ inputs.push == 'true' }} + secrets: ${{ inputs.secrets }} + ssh: ${{ inputs.ssh }} + tags: ${{ inputs.include-tags-in-push == 'true' && steps.meta.outputs.tags || steps.setup-vars.outputs.images }} + target: ${{ inputs.target }} + + - name: Cleanup checkout directory + if: ${{ !cancelled() }} + shell: bash + run: | + # Check that the directory looks OK before removing it + if ! [ -d "_shared-workflows-docker-build-push-image/.git" ]; then + echo "::warning Not removing shared workflows directory: doesn't look like a git repository" + exit 0 + fi + + rm -rf _shared-workflows-docker-build-push-image + + - name: Delete Google Application Credentials file + if: ${{ inputs.delete-credentials-file == 'true' && env.GOOGLE_APPLICATION_CREDENTIALS != '' }} + shell: sh + run: | + if [ -f "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + rm -f "${GOOGLE_APPLICATION_CREDENTIALS}" + echo "::notice::Successfully deleted credentials file" + else + echo "::warning::Credentials file not found at ${GOOGLE_APPLICATION_CREDENTIALS}" + fi + env: + GOOGLE_APPLICATION_CREDENTIALS: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} diff --git a/actions/docker-build-push-image/dockerhub-setup-project-vars.sh b/actions/docker-build-push-image/dockerhub-setup-project-vars.sh new file mode 100644 index 000000000..fad0655b5 --- /dev/null +++ b/actions/docker-build-push-image/dockerhub-setup-project-vars.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Required env vars (provided in workflow yaml) +: "${DOCKERHUB_IMAGE:?DOCKERHUB_IMAGE env var is required}" + +echo "image=${DOCKERHUB_IMAGE}" | tee -a "${GITHUB_OUTPUT}" diff --git a/actions/docker-build-push-image/gar-setup-project-vars.sh b/actions/docker-build-push-image/gar-setup-project-vars.sh new file mode 100644 index 000000000..970bb11b6 --- /dev/null +++ b/actions/docker-build-push-image/gar-setup-project-vars.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Optional env vars +: "${GAR_REPO:=}" # default, empty +: "${GAR_IMAGE:=}" # default, empty + +# Required env vars (provided in workflow yaml) +: "${ENVIRONMENT:?ENVIRONMENT env var is required}" +: "${REGISTRY:?REGISTRY env var is required}" +: "${GH_REPO:?GH_REPO env var is required}" + +gh_repo_name="$(echo "${GH_REPO}" | awk -F'/' '{print $2}')" # ex: grafana/repo -> repo + + +######################################## +# Resolve name of Google Artifact Repository +######################################## +gar_repo_name="${GAR_REPO}" +if [ -z "${gar_repo_name}" ]; then + gar_repo_name="docker-${gh_repo_name//_/-}-${ENVIRONMENT}" +fi +echo "gar_repo_name=${gar_repo_name}" + +######################################## +# Resolve Image Name +######################################## +gar_image="${GAR_IMAGE}" +if [ -z "${gar_image}" ]; then + gar_image="${gh_repo_name}" +fi +echo "gar_image=${gar_image}" + +######################################## +# Resolve project +######################################## +case "${ENVIRONMENT}" in + dev) + gar_project="grafanalabs-dev" + ;; + prod) + gar_project="grafanalabs-global" + ;; + *) + echo "❌ Invalid ENVIRONMENT: $ENVIRONMENT (must be 'dev' or 'prod')" >&2 + exit 1 + ;; +esac +echo "gar_project=${gar_project}" + +######################################## +# Build image path +######################################## +gar_image="${REGISTRY}/${gar_project}/${gar_repo_name}/${gar_image}" + +echo "image=${gar_image}" | tee -a "${GITHUB_OUTPUT}" diff --git a/actions/docker-export-digest/action.yaml b/actions/docker-export-digest/action.yaml new file mode 100644 index 000000000..8ccc5c9f7 --- /dev/null +++ b/actions/docker-export-digest/action.yaml @@ -0,0 +1,42 @@ +name: Export and Upload Docker Manifest +description: Composite action to export and upload a docker manifest + +inputs: + digest: + description: | # TODO Desc + Digest from docker/build-push action + required: true + platform: + description: | # TODO: HERE + Docker platform, ex: linux/arm64 + required: true + +runs: + using: composite + steps: + - name: Prepare + id: prepare + shell: bash + env: + PLATFORM: ${{ inputs.platform }} + run: | + echo "platform-pair=${PLATFORM//\//-}" | tee -a "${GITHUB_OUTPUT}" + + - name: Export digest + shell: sh + run: | + mkdir -p ${{ runner.temp }}/digests + digest="${STEPS_BUILD_OUTPUTS_DIGEST}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + + ls -lah ${{ runner.temp }}/digests + env: + STEPS_BUILD_OUTPUTS_DIGEST: ${{ inputs.digest }} + + - name: Upload digest + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: digests-${{ steps.prepare.outputs.platform-pair }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml new file mode 100644 index 000000000..eb5f3083a --- /dev/null +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -0,0 +1,83 @@ +name: Download and Merge Digests into Manifest +description: Composite action to export and upload a docker manifest + +inputs: # TODO: Fill in descriptions + images: + description: | + Lorem ipsum explainum + required: true + gar-environment: + description: | + Lorem ipsum explainum + default: "dev" + push-to-gar: # TODO: switch this to list style repositories input + description: | + Push resulting manifest to Google Artifact Registry + default: "false" + push-to-dockerhub: + description: | + Push resulting manifest to DockerHub + default: "false" + docker-metadata-json: + description: | + IPSUM Descriptionum + default: "" + +runs: + using: composite + steps: + - name: Download digests + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + path: ${{ runner.temp }}/digests + pattern: digests-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + with: + driver: docker-container + version: latest # see https://github.com/docker/build-push-action/issues/1345#issuecomment-2770572479 + + - name: Login to GAR + if: ${{ inputs.push-to-gar == 'true' }} + uses: grafana/shared-workflows/actions/login-to-gar@rwhitaker/multi-arch-builds + with: + environment: ${{ inputs.gar-environment }} + delete_credentials_file: false + + - name: Login to DockerHub + if: ${{ inputs.push-to-dockerhub == 'true' }} + uses: grafana/shared-workflows/actions/dockerhub-login@rwhitaker/multi-arch-builds + + # TODO: Add --dry-run if not configured to push anywhere + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests + shell: bash + env: + IMAGES: ${{ inputs.images }} + DOCKER_METADATA_OUTPUT_JSON: ${{ inputs.docker-metadata-json }} + run: | + echo "ls -lah ${{ runner.temp }}/digests" + ls -lah ${{ runner.temp }}/digests + + for image in ${IMAGES//,/ }; do + echo docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf "${image}@sha256:%s " *) + + docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf "${image}@sha256:%s " *) + done + + - name: Inspect image + shell: bash + env: + DOCKER_METADATA_OUTPUT_JSON: ${{ inputs.docker-metadata-json }} + run: | + for tag in $(jq -r '.tags[]' <<< "$DOCKER_METADATA_OUTPUT_JSON"); do + echo "" + echo "Inspecting $tag" + docker buildx imagetools inspect "$tag" + done From d30836f1ca8d2287225b89402679362bd66330e3 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Mon, 15 Sep 2025 18:46:34 -0500 Subject: [PATCH 02/28] Initial commit with READMEs --- actions/docker-build-push-image/README.md | 81 +++++++++++++++++++ actions/docker-export-digest/README.md | 73 +++++++++++++++++ .../README.md | 76 +++++++++++++++++ 3 files changed, 230 insertions(+) create mode 100644 actions/docker-build-push-image/README.md create mode 100644 actions/docker-export-digest/README.md create mode 100644 actions/docker-import-digests-push-manifest/README.md diff --git a/actions/docker-build-push-image/README.md b/actions/docker-build-push-image/README.md new file mode 100644 index 000000000..5fde9e4ca --- /dev/null +++ b/actions/docker-build-push-image/README.md @@ -0,0 +1,81 @@ +# docker-build-push-image + +This is a composite GitHub Action, used to build and push docker images to private Grafana registries. +It builds registry URLs for Grafana's registries, authenticates to them, and then +uses [docker/build-push-action](https://github.com/docker/build-push-action) to build and push the image(s). + +# TODO: do we need QEMU? + + + +```yaml +name: Build a Docker Image + +on: + push: + branches: + - main + +jobs: + build-push-image: + permissions: + contents: read + id-token: write + steps: + - uses: grafana/shared-workflows/actions/docker-build-push-image@main # TODO: Fix version once released + with: + platforms: linux/arm64,linux/amd64 + tags: | + ${{ github.sha }} + main + push: true + registries: "gar,dockerhub" +``` + + + +## Inputs + +| Name | Type | Description | +|---------------------------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `build-args` | String | List of arguments necessary for the Docker image to be built. | +| `build-contexts` | String | List of additional build contexts (e.g., name=path) | +| `buildkitd-config` | String | Configum buildkitium descriptium | +| `buildkitd-config-inline` | String | Inliumium configutorium buildkitium descriptium | +| `cache-from` | String | Where cache should be fetched from | +| `cache-to` | String | Where cache should be stored to | +| `context` | String | Path to the Docker build context. | +| `delete-credentials-file` | Boolean | Delete the credentials file after the action is finished. If you want to keep the credentials file for a later step, set this to false. | +| `dockerhub-repository` | String | Ipsum dockerhubium | +| `docker-buildx-driver` | String | The driver to use for Docker Buildx | +| `file` | String | The dockerfile to use. | +| `gar-registry` | String | Google Artifact Registry to store docker images in. | +| `gar-repository` | String | Override the 'repo_name' used to construct the GAR repository name. Only necessary when the GAR includes a repo name that doesn't match the GitHub repo name. Default: `docker-${GitHub Repo Name}-${gar-environment}` | +| `gar-environment` | String | Environment for pushing artifacts (can be either dev or prod). | +| `gar-image` | String | Name of the image to build. Default: `${GitHub Repo Name}` | +| `include-tags-in-push` | Boolean | Can't push tags with a digest, need a way to disable | +| `labels` | String | List of custom labels to add to the image as metadata. | +| `load` | Boolean | Whether to load the built image into the local docker daemon. | +| `outputs` | String | Ipsum factum explainum. | +| `platforms` | String | List of platforms to build the image for | +| `push` | String | Whether to push the image to the configured registries. | +| `registries` | String | List of registries to build images for. | +| `secrets` | String | Secrets to expose to the build. Only needed when authenticating to private repositories outside the repository in which the image is being built. | +| `ssh` | String | List of SSH agent socket or keys to expose to the build | +| `tags` | String | List of Docker tags to be pushed. | +| `target` | String | Sets the target stage to build | + +## Outputs + +| Name | Type | Description | +|----------------|--------|--------------------------------------------------------------| +| `annotations` | String | Generated annotations (from docker/metadata-action) | +| `digest` | String | Image digest (from docker/build-push-action) | +| `imageid` | String | Image ID (from docker/build-push-action) | +| `images` | String | Comma separated list of the images that were built | +| `json` | String | JSON output of tags and labels (from docker/metadata-action) | +| `labels` | String | Generated Docker labels (from docker/metadata-action) | +| `metadata` | String | Build result metadata (from docker/build-push-action) | +| `metadatajson` | String | Metadata JSON (from docker/metadata) | +| `version` | String | Generated Docker image version (from docker/metadata-action) | +| `tags` | String | Generated Docker tags (from docker/metadata-action) | diff --git a/actions/docker-export-digest/README.md b/actions/docker-export-digest/README.md new file mode 100644 index 000000000..467ef905d --- /dev/null +++ b/actions/docker-export-digest/README.md @@ -0,0 +1,73 @@ +# docker-export-digest + +This is a composite GitHub Action used to export a docker digest as a workflow artifact, so it can be merged and pushed +as part of a manifest. + +[//]: # (TODO: Link to docs for the following) +This is meant to work in conjuction with `docker-build-push-image` and `docker-import-digests-push-manifest`. + +It builds registry URLs for Grafana's registries, authenticates to them, and then +uses [docker/build-push-action](https://github.com/docker/build-push-action) to build and push the image(s). + +# TODO: do we need QEMU? + + + +```yaml +name: Build a Docker Image + +on: + push: + branches: + - main + +jobs: + build-push-image: + outputs: + images: ${{ steps.build.outputs.images }} + permissions: + contents: read + id-token: write + steps: + - name: Build Docker Image + id: build + uses: grafana/shared-workflows/actions/docker-build-push-image@main # TODO: Fix version once released + with: + platforms: linux/arm64 + tags: | + ${{ github.sha }} + main + push: true + registries: "gar,dockerhub" + include-tags-in-push: false + outputs: "type=image,push-by-digest=true,name-canonical=true,push=true" + - name: Export and upload digest + uses: grafana/shared-workflows/actions/docker-export-digest@rwhitaker/multi-arch-builds + with: + digest: ${{ steps.build.outputs.digest }} + platform: linux/arm64 + merge-digest: + if: ${{ inputs.push == 'true' }} + runs-on: ubuntu-arm64-small + needs: build-and-push + permissions: + contents: read + id-token: write + steps: + - name: Download Multi-Arch Digests, Construct and Upload Manifest + uses: grafana/shared-workflows/actions/docker-import-digests-push-manifest@main # TODO: Pin sha + with: + images: ${{ needs.build-push-image.outputs.images }} + gar-environment: 'dev' + registries: "gar,dockerhub" + docker-metadata-json: ${{ needs.build-and-push.outputs.metadatajson }} +``` + + + +## Inputs + +| Name | Type | Description | +|------------|--------|--------------------------------------| +| `digest` | String | Digest from docker/build-push action | +| `platform` | String | Docker platform, ex: linux/arm64 | diff --git a/actions/docker-import-digests-push-manifest/README.md b/actions/docker-import-digests-push-manifest/README.md new file mode 100644 index 000000000..824fdcaa0 --- /dev/null +++ b/actions/docker-import-digests-push-manifest/README.md @@ -0,0 +1,76 @@ +# docker-import-digest-push-manifest + +This is a composite GitHub Action used to export a docker digest as a workflow artifact, so it can be merged and pushed +as part of a manifest. + +[//]: # (TODO: Link to docs for the following) +This is meant to work in conjuction with `docker-build-push-image` and `docker-import-digests-push-manifest`. + +It builds registry URLs for Grafana's registries, authenticates to them, and then +uses [docker/build-push-action](https://github.com/docker/build-push-action) to build and push the image(s). + +# TODO: do we need QEMU? + + + +```yaml +name: Build a Docker Image + +on: + push: + branches: + - main + +jobs: + build-push-image: + outputs: + images: ${{ steps.build.outputs.images }} + permissions: + contents: read + id-token: write + steps: + - name: Build Docker Image + id: build + uses: grafana/shared-workflows/actions/docker-build-push-image@main # TODO: Fix version once released + with: + platforms: linux/arm64 + tags: | + ${{ github.sha }} + main + push: true + registries: "gar,dockerhub" + include-tags-in-push: false + outputs: "type=image,push-by-digest=true,name-canonical=true,push=true" + - name: Export and upload digest + uses: grafana/shared-workflows/actions/docker-export-digest@rwhitaker/multi-arch-builds + with: + digest: ${{ steps.build.outputs.digest }} + platform: linux/arm64 + merge-digest: + if: ${{ inputs.push == 'true' }} + runs-on: ubuntu-arm64-small + needs: build-and-push + permissions: + contents: read + id-token: write + steps: + - name: Download Multi-Arch Digests, Construct and Upload Manifest + uses: grafana/shared-workflows/actions/docker-import-digests-push-manifest@main # TODO: Pin sha + with: + images: ${{ needs.build-push-image.outputs.images }} + gar-environment: 'dev' + registries: "gar,dockerhub" + docker-metadata-json: ${{ needs.build-and-push.outputs.metadatajson }} +``` + + + +## Inputs + +| Name | Type | Description | +|------------------------|---------|-----------------------------------------------------| +| `images` | String | Lorem ipsum explainum | +| `gar-environment` | String | Lorem ipsum explainum | +| `push-to-gar` | Boolean | Push resulting manifest to Google Artifact Registry | +| `push-to-dockerhub` | Boolean | Push resulting manifest to DockerHub | +| `docker-metadata-json` | String | IPSUM Descriptionum | From d884f81382219b102993e187c2142cb480761ea9 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Thu, 18 Sep 2025 18:45:31 -0500 Subject: [PATCH 03/28] Add input/output descriptions for docker-build-push-image, tidy them up, and alphabetize --- actions/docker-build-push-image/README.md | 70 +++++++----- actions/docker-build-push-image/action.yaml | 119 ++++++++++---------- 2 files changed, 102 insertions(+), 87 deletions(-) diff --git a/actions/docker-build-push-image/README.md b/actions/docker-build-push-image/README.md index 5fde9e4ca..f70b99d95 100644 --- a/actions/docker-build-push-image/README.md +++ b/actions/docker-build-push-image/README.md @@ -36,34 +36,34 @@ jobs: ## Inputs -| Name | Type | Description | -|---------------------------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `build-args` | String | List of arguments necessary for the Docker image to be built. | -| `build-contexts` | String | List of additional build contexts (e.g., name=path) | -| `buildkitd-config` | String | Configum buildkitium descriptium | -| `buildkitd-config-inline` | String | Inliumium configutorium buildkitium descriptium | -| `cache-from` | String | Where cache should be fetched from | -| `cache-to` | String | Where cache should be stored to | -| `context` | String | Path to the Docker build context. | -| `delete-credentials-file` | Boolean | Delete the credentials file after the action is finished. If you want to keep the credentials file for a later step, set this to false. | -| `dockerhub-repository` | String | Ipsum dockerhubium | -| `docker-buildx-driver` | String | The driver to use for Docker Buildx | -| `file` | String | The dockerfile to use. | -| `gar-registry` | String | Google Artifact Registry to store docker images in. | -| `gar-repository` | String | Override the 'repo_name' used to construct the GAR repository name. Only necessary when the GAR includes a repo name that doesn't match the GitHub repo name. Default: `docker-${GitHub Repo Name}-${gar-environment}` | -| `gar-environment` | String | Environment for pushing artifacts (can be either dev or prod). | -| `gar-image` | String | Name of the image to build. Default: `${GitHub Repo Name}` | -| `include-tags-in-push` | Boolean | Can't push tags with a digest, need a way to disable | -| `labels` | String | List of custom labels to add to the image as metadata. | -| `load` | Boolean | Whether to load the built image into the local docker daemon. | -| `outputs` | String | Ipsum factum explainum. | -| `platforms` | String | List of platforms to build the image for | -| `push` | String | Whether to push the image to the configured registries. | -| `registries` | String | List of registries to build images for. | -| `secrets` | String | Secrets to expose to the build. Only needed when authenticating to private repositories outside the repository in which the image is being built. | -| `ssh` | String | List of SSH agent socket or keys to expose to the build | -| `tags` | String | List of Docker tags to be pushed. | -| `target` | String | Sets the target stage to build | +| Name | Type | Description | +|-------------------------------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `build-args` | String | List of arguments necessary for the Docker image to be built. Passed to `docker/build-push-action`. | +| `build-contexts` | String | List of additional build contexts (e.g., name=path). Passed to `docker/build-push-action`. | +| `buildkitd-config` | String | The buildkitd config file to use. Defaults to `/etc/buildkitd.toml` if you're using Grafana's self-hosted runners. Passed to `docker/setup-buildx-action`. | +| `buildkitd-config-inline` | String | The buildkitd inline config to use. Passed to `docker/setup-buildx-action`. | +| `cache-from` | String | Where cache should be fetched from. Passed to `docker/build-push-action`. | +| `cache-to` | String | Where cache should be stored to. Passed to `docker/build-push-action`. | +| `context` | String | Path to the Docker build context. Passed to `docker/build-push-action`. | +| `docker-buildx-driver` | String | The driver to use for Docker Buildx. Passed to `docker/setup-buildx-action`. | +| `dockerhub-repository` | String | Ipsum dockerhubium | +| `file` | String | The dockerfile to use. Passed to `docker/build-push-action`. | +| `gar-delete-credentials-file` | Boolean | Delete the Google credentials file after the action is finished. If you want to keep the credentials file for a later step, set this to false. | +| `gar-environment` | String | Environment for pushing artifacts (can be either dev or prod). This sets the GAR Project (gar-project) to either `grafanalabs-dev` or `grafanalabs-global`. | +| `gar-image` | String | Name of the image to build. Default: `${GitHub Repo Name}`. | +| `gar-registry` | String | Google Artifact Registry to store docker images in. | +| `gar-repository` | String | Override the 'repo_name' used to construct the GAR repository name. Only necessary when the GAR includes a repo name that doesn't match the GitHub repo name. Default: `docker-${GitHub Repo Name}-${gar-environment}` | +| `include-tags-in-push` | Boolean | Disables the pushing of tags, and instead includes just a list of images as docker tags. Used when pushing docker digests instead of docker tags. | +| `labels` | String | List of custom labels to add to the image as metadata (passed to `docker/build-push-action`). Passed to `docker/build-push-action`. | +| `load` | Boolean | Whether to load the built image into the local docker daemon (passed to `docker/build-push-action`). Passed to `docker/build-push-action`. | +| `outputs` | String | List of docker output destinations. Passed to `docker/build-push-action`. | +| `platforms` | String | List of platforms to build the image for. Passed to `docker/build-push-action`. | +| `push` | String | Whether to push the image to the configured registries. Passed to `docker/build-push-action`. | +| `registries` | String | CSV list of registries to build images for. Accepted registries are "gar" and "dockerhub". | +| `secrets` | String | Secrets to expose to the build. Only needed when authenticating to private repositories outside the repository in which the image is being built. Passed to `docker/build-push-action`. | +| `ssh` | String | List of SSH agent socket or keys to expose to the build Passed to `docker/build-push-action`. | +| `tags` | String | List of Docker tags to be pushed. Passed to `docker/build-push-action`. | +| `target` | String | Sets the target stage to build. Passed to `docker/build-push-action`. | ## Outputs @@ -77,5 +77,17 @@ jobs: | `labels` | String | Generated Docker labels (from docker/metadata-action) | | `metadata` | String | Build result metadata (from docker/build-push-action) | | `metadatajson` | String | Metadata JSON (from docker/metadata) | -| `version` | String | Generated Docker image version (from docker/metadata-action) | | `tags` | String | Generated Docker tags (from docker/metadata-action) | +| `version` | String | Generated Docker image version (from docker/metadata-action) | + + +## How we construct Google Artifact Registry Images + +The full GAR image is constructed as follows, where `gar-project` is determined by `inputs.gar-environment`. + +"${{ inputs.gar-registry }}/${{ gar-project }}/${{ inputs.gar-repository }}/${{ inputs.gar-image }}" + +## How we construct DockerHub Images + +The full DockerHub image is constructed as follows: + diff --git a/actions/docker-build-push-image/action.yaml b/actions/docker-build-push-image/action.yaml index df12f0622..f5aeb0b50 100644 --- a/actions/docker-build-push-image/action.yaml +++ b/actions/docker-build-push-image/action.yaml @@ -5,51 +5,62 @@ inputs: build-args: description: | List of arguments necessary for the Docker image to be built. - default: "" + Passed to `docker/build-push-action`. build-contexts: description: | - List of additional build contexts (e.g., name=path) - required: false + List of additional build contexts (e.g., name=path). + Passed to `docker/build-push-action`. buildkitd-config: description: | - Configum buildkitium descriptium - default: "" + The buildkitd config file to use. Defaults to `/etc/buildkitd.toml` if you're using + Grafana's self-hosted runners. + Passed to `docker/setup-buildx-action`. buildkitd-config-inline: description: | - Inliumium configutorium buildkitium descriptium - default: "" + The buildkitd inline config to use. + Passed to `docker/setup-buildx-action`. cache-from: description: | - Where cache should be fetched from - required: false - default: "type=gha" + Where cache should be fetched from. + Passed to `docker/build-push-action`. + default: type=gha cache-to: description: | - Where cache should be stored to - required: false - default: "type=gha,mode=max" + Where cache should be stored to. + Passed to `docker/build-push-action`. + default: type=gha,mode=max context: description: | Path to the Docker build context. - default: "." - delete-credentials-file: #TODO: Expand on this + Passed to `docker/build-push-action`. + default: . + docker-buildx-driver: description: | - Delete the credentials file after the action is finished. - If you want to keep the credentials file for a later step, set this to false. - default: "true" + The driver to use for Docker Buildx. + Passed to `docker/setup-buildx-action`. + default: docker-container dockerhub-repository: description: | Ipsum dockerhubium - default: "${{ github.repository }}" - docker-buildx-driver: - description: | - The driver to use for Docker Buildx - required: false - default: "docker-container" + default: ${{ github.repository }} file: description: | The dockerfile to use. - required: false + Passed to `docker/build-push-action`. + gar-delete-credentials-file: + description: | + Delete the Google credentials file after the action is finished. + If you want to keep the credentials file for a later step, set this to false. + default: "true" + gar-environment: + description: | + Environment for pushing artifacts (can be either dev or prod). + This sets the GAR Project (gar-project) to either `grafanalabs-dev` or `grafanalabs-global`. + default: dev + gar-image: + description: | + Name of the image to build. + Default: `${GitHub Repo Name}`. gar-registry: description: | Google Artifact Registry to store docker images in. @@ -59,61 +70,53 @@ inputs: Override the 'repo_name' used to construct the GAR repository name. Only necessary when the GAR includes a repo name that doesn't match the GitHub repo name. Default: `docker-${GitHub Repo Name}-${gar-environment}` - default: "" - gar-environment: - description: | - Environment for pushing artifacts (can be either dev or prod). - default: "dev" - gar-image: - description: | - Name of the image to build. - Default: `${GitHub Repo Name}` - default: "" include-tags-in-push: - description: | # TODO: Expand - Can't push tags with a digest, need a way to disable + description: | + Disables the pushing of tags, and instead includes just a list of images as docker tags. + Used when pushing docker digests instead of docker tags. default: "true" labels: description: | - List of custom labels to add to the image as metadata. - required: false + List of custom labels to add to the image as metadata (passed to `docker/build-push-action`). + Passed to `docker/build-push-action`. load: description: | - Whether to load the built image into the local docker daemon. - required: false + Whether to load the built image into the local docker daemon (passed to `docker/build-push-action`). + Passed to `docker/build-push-action`. default: "false" outputs: - description: | # TODO Desc - Ipsum factum explainum. - required: false - default: "" + description: | + List of docker output destinations. + Passed to `docker/build-push-action`. platforms: description: | - List of platforms to build the image for - required: false + List of platforms to build the image for. + Passed to `docker/build-push-action`. push: description: | Whether to push the image to the configured registries. - required: false + Passed to `docker/build-push-action`. registries: description: | - List of registries to build images for. - default: "" + CSV list of registries to build images for. + Accepted registries are "gar" and "dockerhub". secrets: description: | Secrets to expose to the build. Only needed when authenticating to private repositories outside the repository in which the image is being built. - required: false + Passed to `docker/build-push-action`. ssh: description: | List of SSH agent socket or keys to expose to the build + Passed to `docker/build-push-action`. tags: description: | List of Docker tags to be pushed. + Passed to `docker/build-push-action`. required: true target: description: | - Sets the target stage to build - required: false + Sets the target stage to build. + Passed to `docker/build-push-action`. outputs: annotations: @@ -139,13 +142,13 @@ outputs: value: ${{ steps.build.outputs.metadata }} metadatajson: description: "Metadata JSON (from docker/metadata)" - value: ${{ steps.meta.outputs.json }} - version: - description: "Generated Docker image version (from docker/metadata-action)" - value: ${{ steps.meta.outputs.version }} + value: ${{ steps.meta.outputs.json }}s tags: description: "Generated Docker tags (from docker/metadata-action)" value: ${{ steps.meta.outputs.tags }} + version: + description: "Generated Docker image version (from docker/metadata-action)" + value: ${{ steps.meta.outputs.version }} runs: using: composite @@ -314,7 +317,7 @@ runs: rm -rf _shared-workflows-docker-build-push-image - name: Delete Google Application Credentials file - if: ${{ inputs.delete-credentials-file == 'true' && env.GOOGLE_APPLICATION_CREDENTIALS != '' }} + if: ${{ inputs.gar-delete-credentials-file == 'true' && env.GOOGLE_APPLICATION_CREDENTIALS != '' }} shell: sh run: | if [ -f "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then From 4526d819bc91888f32c0717721b1e6d991c5bff3 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Fri, 19 Sep 2025 10:27:56 -0500 Subject: [PATCH 04/28] docs updates and adding in QEMU to docker-build-push-action --- actions/docker-build-push-image/TEST.md | 30 +++++++++++++++++++ actions/docker-build-push-image/action.yaml | 3 ++ actions/docker-export-digest/README.md | 8 ++--- actions/docker-export-digest/action.yaml | 9 +++--- .../README.md | 12 ++++---- .../action.yaml | 28 +++++++---------- 6 files changed, 57 insertions(+), 33 deletions(-) create mode 100644 actions/docker-build-push-image/TEST.md diff --git a/actions/docker-build-push-image/TEST.md b/actions/docker-build-push-image/TEST.md new file mode 100644 index 000000000..9e362e664 --- /dev/null +++ b/actions/docker-build-push-image/TEST.md @@ -0,0 +1,30 @@ +Design decisions + +docker-build-push-image + +1. This should handle both gar and dockerhub logins, with a framework in place that makes it easy to add other + registries in the future. Which registries we build images for and login to should be configurable. +2. This should handle both pushing tags and digests (for multiarch builds). The idea is that we can use this as a direct + replacement for both `push-to-gar-docker` and `build-push-to-dockerhub`, while also making it flexible enough that we + can use it for true multiarch builds. +2. To configure which registries we build and push to, an input is provided that takes a CSV of + registries (`gar,dockerhub`). +3. For each registry that we configure, a script is included that takes the inputs for that registry and outputs an + image to the + GITHUB_OUTPUT. Each image (without tags) is then appended to the `images` output. Inputs for each registry begin with + the same prefix (ex: `gar-config-value`) +4. For each registry that we configure, we only login to that registry if it's in the `registries` input list. +5. This action outputs both a tagged image list and an image list without tags. This is because when you're pushing a + manifest, tags cannot be + included; however when you're pushing tags, you have to include them. So we include both. +6. If `include-tags-in-push=true`, then we push the tag list, if `include-tags-in-push=false` then we push the image + list. +7. To provide access to Grafana's Docker mirror, if a workflow is running on a self-hosted runner, and no + buildkitd-config is provided, then we default to `/etc/buildkitd.toml`. This means if you're running this action from + inside a container you need to either mount that directory, or explicitly override it by setting `buidkitd-config` + or `buidkitd-config-inline`. +5. +6. +7. (Ex: images=us-docker.pkg.dev/grafanalabs-dev/gar-registry/image-name,docker.io/grafana/dockerhub-image) +5. +3. We only authenticate to registries that are included in the `` diff --git a/actions/docker-build-push-image/action.yaml b/actions/docker-build-push-image/action.yaml index f5aeb0b50..b9508df9a 100644 --- a/actions/docker-build-push-image/action.yaml +++ b/actions/docker-build-push-image/action.yaml @@ -268,6 +268,9 @@ runs: echo "buildkitd-config-inline=${buildkitd_config_inline}" | tee -a "${GITHUB_OUTPUT}" echo "buildkitd-config=${buildkitd_config}" | tee -a "${GITHUB_OUTPUT}" + - name: Set up QEMU + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 with: diff --git a/actions/docker-export-digest/README.md b/actions/docker-export-digest/README.md index 467ef905d..c7a3739ba 100644 --- a/actions/docker-export-digest/README.md +++ b/actions/docker-export-digest/README.md @@ -67,7 +67,7 @@ jobs: ## Inputs -| Name | Type | Description | -|------------|--------|--------------------------------------| -| `digest` | String | Digest from docker/build-push action | -| `platform` | String | Docker platform, ex: linux/arm64 | +| Name | Type | Description | +|------------|--------|------------------------------------------------------------------------------------------------------------| +| `digest` | String | Docker digest. This is included as an output for `docker-build-push-image` and `docker/build-push-action`. | +| `platform` | String | Docker platform, ex: linux/arm64. | diff --git a/actions/docker-export-digest/action.yaml b/actions/docker-export-digest/action.yaml index 8ccc5c9f7..e994ffb84 100644 --- a/actions/docker-export-digest/action.yaml +++ b/actions/docker-export-digest/action.yaml @@ -3,12 +3,13 @@ description: Composite action to export and upload a docker manifest inputs: digest: - description: | # TODO Desc - Digest from docker/build-push action + description: | + Docker digest. + This is included as an output for `docker-build-push-image` and `docker/build-push-action`. required: true platform: - description: | # TODO: HERE - Docker platform, ex: linux/arm64 + description: | + Docker platform, ex: linux/arm64. required: true runs: diff --git a/actions/docker-import-digests-push-manifest/README.md b/actions/docker-import-digests-push-manifest/README.md index 824fdcaa0..c185c0212 100644 --- a/actions/docker-import-digests-push-manifest/README.md +++ b/actions/docker-import-digests-push-manifest/README.md @@ -67,10 +67,8 @@ jobs: ## Inputs -| Name | Type | Description | -|------------------------|---------|-----------------------------------------------------| -| `images` | String | Lorem ipsum explainum | -| `gar-environment` | String | Lorem ipsum explainum | -| `push-to-gar` | Boolean | Push resulting manifest to Google Artifact Registry | -| `push-to-dockerhub` | Boolean | Push resulting manifest to DockerHub | -| `docker-metadata-json` | String | IPSUM Descriptionum | +| Name | Type | Description | +|------------------------|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `docker-metadata-json` | String | Docker metadata JSON, from `docker-build-push-image` or `docker/build-push-action`. | +| `gar-environment` | String | Environment for pushing artifacts (can be either dev or prod). This sets the GAR Project to either `grafanalabs-dev` or `grafanalabs-global`. | +| `images` | String | CSV of Docker images to push. These images should not include tags. Ex: us-docker.pkg.dev/grafanalabs-dev/gar-registry/image-name,docker.io/grafana/dockerhub-image | diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index eb5f3083a..0493e8892 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -1,27 +1,21 @@ name: Download and Merge Digests into Manifest description: Composite action to export and upload a docker manifest -inputs: # TODO: Fill in descriptions - images: +inputs: + docker-metadata-json: description: | - Lorem ipsum explainum - required: true + Docker metadata JSON, from `docker-build-push-image` or `docker/build-push-action`. + default: "" gar-environment: description: | - Lorem ipsum explainum + Environment for pushing artifacts (can be either dev or prod). + This sets the GAR Project to either `grafanalabs-dev` or `grafanalabs-global`. default: "dev" - push-to-gar: # TODO: switch this to list style repositories input - description: | - Push resulting manifest to Google Artifact Registry - default: "false" - push-to-dockerhub: - description: | - Push resulting manifest to DockerHub - default: "false" - docker-metadata-json: + images: description: | - IPSUM Descriptionum - default: "" + CSV of Docker images to push. These images should not include tags. + Ex: us-docker.pkg.dev/grafanalabs-dev/gar-registry/image-name,docker.io/grafana/dockerhub-image + required: true runs: using: composite @@ -40,14 +34,12 @@ runs: version: latest # see https://github.com/docker/build-push-action/issues/1345#issuecomment-2770572479 - name: Login to GAR - if: ${{ inputs.push-to-gar == 'true' }} uses: grafana/shared-workflows/actions/login-to-gar@rwhitaker/multi-arch-builds with: environment: ${{ inputs.gar-environment }} delete_credentials_file: false - name: Login to DockerHub - if: ${{ inputs.push-to-dockerhub == 'true' }} uses: grafana/shared-workflows/actions/dockerhub-login@rwhitaker/multi-arch-builds # TODO: Add --dry-run if not configured to push anywhere From b37f49dcd85a1f75fbb616ee31eba382bdd83785 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Fri, 19 Sep 2025 10:43:40 -0500 Subject: [PATCH 05/28] Docs updates and including dockerhub registry input --- actions/docker-build-push-image/TEST.md | 4 ++++ actions/docker-build-push-image/action.yaml | 8 +++++++- .../dockerhub-setup-project-vars.sh | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/actions/docker-build-push-image/TEST.md b/actions/docker-build-push-image/TEST.md index 9e362e664..dc288c5e6 100644 --- a/actions/docker-build-push-image/TEST.md +++ b/actions/docker-build-push-image/TEST.md @@ -1,5 +1,9 @@ Design decisions +## Whole workflow + +1. All actions are prefixed with `docker-`, to improve discoverability. + docker-build-push-image 1. This should handle both gar and dockerhub logins, with a framework in place that makes it easy to add other diff --git a/actions/docker-build-push-image/action.yaml b/actions/docker-build-push-image/action.yaml index b9508df9a..4446a44fc 100644 --- a/actions/docker-build-push-image/action.yaml +++ b/actions/docker-build-push-image/action.yaml @@ -39,9 +39,14 @@ inputs: The driver to use for Docker Buildx. Passed to `docker/setup-buildx-action`. default: docker-container + dockerhub-registry: + description: | + DockerHub Registry to store docker images in. + default: "docker.io" dockerhub-repository: description: | - Ipsum dockerhubium + DockerHub Repository to store docker images in. + Default: ${{ github.repository }} default: ${{ github.repository }} file: description: | @@ -194,6 +199,7 @@ runs: shell: bash env: DOCKERHUB_IMAGE: ${{ inputs.dockerhub-repository }} + DOCKERHUB_REGISTRY: ${{ inputs.dockerhub-registry }} run: | chmod +x ./_shared-workflows-docker-build-push-image/actions/docker-build-push-image/dockerhub-setup-project-vars.sh ./_shared-workflows-docker-build-push-image/actions/docker-build-push-image/dockerhub-setup-project-vars.sh diff --git a/actions/docker-build-push-image/dockerhub-setup-project-vars.sh b/actions/docker-build-push-image/dockerhub-setup-project-vars.sh index fad0655b5..4ae48d842 100644 --- a/actions/docker-build-push-image/dockerhub-setup-project-vars.sh +++ b/actions/docker-build-push-image/dockerhub-setup-project-vars.sh @@ -4,4 +4,4 @@ set -euo pipefail # Required env vars (provided in workflow yaml) : "${DOCKERHUB_IMAGE:?DOCKERHUB_IMAGE env var is required}" -echo "image=${DOCKERHUB_IMAGE}" | tee -a "${GITHUB_OUTPUT}" +echo "image=${DOCKERHUB_REGISTRY}/${DOCKERHUB_IMAGE}" | tee -a "${GITHUB_OUTPUT}" From 810ea7b62af43ab693c15cd96f0ffc24ba720c76 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Fri, 19 Sep 2025 10:46:53 -0500 Subject: [PATCH 06/28] Action doesn't like the template syntax in an input description? --- actions/docker-build-push-image/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/docker-build-push-image/action.yaml b/actions/docker-build-push-image/action.yaml index 4446a44fc..f978cd544 100644 --- a/actions/docker-build-push-image/action.yaml +++ b/actions/docker-build-push-image/action.yaml @@ -46,7 +46,7 @@ inputs: dockerhub-repository: description: | DockerHub Repository to store docker images in. - Default: ${{ github.repository }} + Default: github.repository default: ${{ github.repository }} file: description: | From d1f1ef8815531237fead2b7ae4700d8dbf79990d Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Fri, 19 Sep 2025 15:10:13 -0500 Subject: [PATCH 07/28] docker-import-digests-push-manifest: add registries input and dry-run logic --- .../action.yaml | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index 0493e8892..c0b665921 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -16,10 +16,27 @@ inputs: CSV of Docker images to push. These images should not include tags. Ex: us-docker.pkg.dev/grafanalabs-dev/gar-registry/image-name,docker.io/grafana/dockerhub-image required: true + registries: + description: | + CSV list of registries to build images for. + Accepted registries are "gar" and "dockerhub". runs: using: composite steps: + - name: Set registries + id: registries + shell: bash + env: + REGISTRIES: ${{ inputs.registries }} + run: | + if [ -n "${REGISTRIES}" ]; then + echo "push=true" | tee -a "${GITHUB_OUTPUT}" + for r in ${REGISTRIES//,/ }; do + echo "include-${r}=true" | tee -a "${GITHUB_OUTPUT}" + done + done + - name: Download digests uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: @@ -34,12 +51,14 @@ runs: version: latest # see https://github.com/docker/build-push-action/issues/1345#issuecomment-2770572479 - name: Login to GAR + if: ${{ steps.registries.outputs.include-gar == 'true' }} uses: grafana/shared-workflows/actions/login-to-gar@rwhitaker/multi-arch-builds with: environment: ${{ inputs.gar-environment }} delete_credentials_file: false - name: Login to DockerHub + if: ${{ steps.registries.outputs.include-dockerhub == 'true' }} uses: grafana/shared-workflows/actions/dockerhub-login@rwhitaker/multi-arch-builds # TODO: Add --dry-run if not configured to push anywhere @@ -49,18 +68,27 @@ runs: env: IMAGES: ${{ inputs.images }} DOCKER_METADATA_OUTPUT_JSON: ${{ inputs.docker-metadata-json }} + PUSH: ${{ steps.registries.outputs.push }} run: | + set -euo pipefail echo "ls -lah ${{ runner.temp }}/digests" ls -lah ${{ runner.temp }}/digests + if [ "${PUSH}" == "true" ]; then + DRY_RUN="" + else + echo "No images to push, setting --dry-run=true" + DRY_RUN="--dry-run" + fi + for image in ${IMAGES//,/ }; do echo docker buildx imagetools create \ $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf "${image}@sha256:%s " *) + $(printf "${image}@sha256:%s " *) $DRY_RUN docker buildx imagetools create \ $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf "${image}@sha256:%s " *) + $(printf "${image}@sha256:%s " *) $DRY_RUN done - name: Inspect image From 5e053bcd0996532e27ae4ac901a0c3d9525c7007 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Fri, 19 Sep 2025 15:17:27 -0500 Subject: [PATCH 08/28] docker-import-digests-push-manifest: add registries input and dry-run logic --- actions/docker-import-digests-push-manifest/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index c0b665921..0504ea1a2 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -35,7 +35,7 @@ runs: for r in ${REGISTRIES//,/ }; do echo "include-${r}=true" | tee -a "${GITHUB_OUTPUT}" done - done + fi - name: Download digests uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 From 2bee1ac4dfd8abe715aa654a91e4b9e172e55660 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Fri, 19 Sep 2025 15:38:42 -0500 Subject: [PATCH 09/28] docker-import-digests-push-manifest: Add some error handling --- .../action.yaml | 52 ++++++++----------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index 0504ea1a2..9e57050d6 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -16,27 +16,14 @@ inputs: CSV of Docker images to push. These images should not include tags. Ex: us-docker.pkg.dev/grafanalabs-dev/gar-registry/image-name,docker.io/grafana/dockerhub-image required: true - registries: + push: description: | - CSV list of registries to build images for. - Accepted registries are "gar" and "dockerhub". + Whether to push the manifest to the configured registries. + default: "false" runs: using: composite steps: - - name: Set registries - id: registries - shell: bash - env: - REGISTRIES: ${{ inputs.registries }} - run: | - if [ -n "${REGISTRIES}" ]; then - echo "push=true" | tee -a "${GITHUB_OUTPUT}" - for r in ${REGISTRIES//,/ }; do - echo "include-${r}=true" | tee -a "${GITHUB_OUTPUT}" - done - fi - - name: Download digests uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: @@ -51,45 +38,52 @@ runs: version: latest # see https://github.com/docker/build-push-action/issues/1345#issuecomment-2770572479 - name: Login to GAR - if: ${{ steps.registries.outputs.include-gar == 'true' }} + if: ${{ inputs.push == 'true' }} uses: grafana/shared-workflows/actions/login-to-gar@rwhitaker/multi-arch-builds with: environment: ${{ inputs.gar-environment }} delete_credentials_file: false - name: Login to DockerHub - if: ${{ steps.registries.outputs.include-dockerhub == 'true' }} + if: ${{ inputs.push == 'true' }} uses: grafana/shared-workflows/actions/dockerhub-login@rwhitaker/multi-arch-builds - # TODO: Add --dry-run if not configured to push anywhere - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests shell: bash env: IMAGES: ${{ inputs.images }} DOCKER_METADATA_OUTPUT_JSON: ${{ inputs.docker-metadata-json }} - PUSH: ${{ steps.registries.outputs.push }} + PUSH: ${{ inputs.push }} run: | set -euo pipefail echo "ls -lah ${{ runner.temp }}/digests" ls -lah ${{ runner.temp }}/digests - if [ "${PUSH}" == "true" ]; then + DRY_RUN="" else echo "No images to push, setting --dry-run=true" DRY_RUN="--dry-run" fi - for image in ${IMAGES//,/ }; do - echo docker buildx imagetools create \ - $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf "${image}@sha256:%s " *) $DRY_RUN + if [ -n "${IMAGES}" ]; then + for image in ${IMAGES//,/ }; do + echo docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf "${image}@sha256:%s " *) - docker buildx imagetools create \ - $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf "${image}@sha256:%s " *) $DRY_RUN - done + if [ "${PUSH}" == "true" ]; then + docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf "${image}@sha256:%s " *) + else + echo Skipping `docker buildx imagetools create` because push is set to false. + done + done + else + echo "No images to push, skipping imagetools creation" + fi - name: Inspect image shell: bash From 9cc653dc7c9f954d2a5ad570cdc9ef2e3554244d Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Fri, 19 Sep 2025 15:43:21 -0500 Subject: [PATCH 10/28] docker-import-digests-push-manifest: Add some error handling --- actions/docker-import-digests-push-manifest/action.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index 9e57050d6..ff31ff195 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -60,13 +60,6 @@ runs: echo "ls -lah ${{ runner.temp }}/digests" ls -lah ${{ runner.temp }}/digests - - DRY_RUN="" - else - echo "No images to push, setting --dry-run=true" - DRY_RUN="--dry-run" - fi - if [ -n "${IMAGES}" ]; then for image in ${IMAGES//,/ }; do echo docker buildx imagetools create \ From f8f110f711562978c24f787aeec99b919dbe2569 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Fri, 19 Sep 2025 15:46:13 -0500 Subject: [PATCH 11/28] docker-import-digests-push-manifest: Add some error handling --- actions/docker-import-digests-push-manifest/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index ff31ff195..11b324a75 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -72,7 +72,7 @@ runs: $(printf "${image}@sha256:%s " *) else echo Skipping `docker buildx imagetools create` because push is set to false. - done + fi done else echo "No images to push, skipping imagetools creation" From 0476051beb70210b6a4e1314bab2e5a63fb68e56 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Fri, 19 Sep 2025 16:09:00 -0500 Subject: [PATCH 12/28] docker-import-digests-push-manifest: Add some error handling --- actions/docker-import-digests-push-manifest/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index 11b324a75..2df3e5b71 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -80,6 +80,7 @@ runs: - name: Inspect image shell: bash + if: ${{ inputs.push == 'true' }} env: DOCKER_METADATA_OUTPUT_JSON: ${{ inputs.docker-metadata-json }} run: | From 691ec3047dac12073b1e0e3814d9d3f88dc61abc Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Fri, 19 Sep 2025 17:22:42 -0500 Subject: [PATCH 13/28] docker-import-digests-push-manifest: Add some error handling --- actions/docker-import-digests-push-manifest/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index 2df3e5b71..528b55c0c 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -71,7 +71,7 @@ runs: $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ $(printf "${image}@sha256:%s " *) else - echo Skipping `docker buildx imagetools create` because push is set to false. + echo Skipping "docker buildx imagetools create" because push is set to false. fi done else From df2148c79ab020dc60675c7bfc90aaca598bfdd7 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Fri, 19 Sep 2025 18:06:29 -0500 Subject: [PATCH 14/28] Updated error message --- actions/docker-import-digests-push-manifest/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index 528b55c0c..8b44a9c2c 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -71,7 +71,7 @@ runs: $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ $(printf "${image}@sha256:%s " *) else - echo Skipping "docker buildx imagetools create" because push is set to false. + echo Skipping command because push is set to false. fi done else From 6e2c378eb2138bf8dc1ce2e3e4896afc84024c53 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Sat, 20 Sep 2025 13:59:16 -0500 Subject: [PATCH 15/28] Add warnings when no images are pushed. --- actions/docker-build-push-image/action.yaml | 5 +++++ actions/docker-import-digests-push-manifest/action.yaml | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/actions/docker-build-push-image/action.yaml b/actions/docker-build-push-image/action.yaml index f978cd544..eb5977e82 100644 --- a/actions/docker-build-push-image/action.yaml +++ b/actions/docker-build-push-image/action.yaml @@ -210,6 +210,7 @@ runs: env: GAR_IMAGE: ${{ steps.setup-gar-vars.outputs.image }} DOCKERHUB_IMAGE: ${{ steps.setup-dockerhub-vars.outputs.image }} + PUSH: ${{ inputs.push }} run: | images="" for image in "${GAR_IMAGE}" "${DOCKERHUB_IMAGE}" @@ -232,6 +233,10 @@ runs: echo "images=dry-run-image" | tee -a "${GITHUB_OUTPUT}" fi + if [ "${PUSH}" != "true" ]; then + echo "::warning::push=${PUSH}, no images will be pushed" + fi + - name: Login to GAR if: ${{ inputs.push == 'true' && steps.registries.outputs.include-gar == 'true' }} uses: ./_shared-workflows-docker-build-push-image/actions/login-to-gar diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index 8b44a9c2c..01aaaa7f1 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -60,6 +60,10 @@ runs: echo "ls -lah ${{ runner.temp }}/digests" ls -lah ${{ runner.temp }}/digests + if [ "${PUSH}" != "true" ]; then + echo "::warning::push=${PUSH}, no images will be pushed" + fi + if [ -n "${IMAGES}" ]; then for image in ${IMAGES//,/ }; do echo docker buildx imagetools create \ @@ -71,11 +75,11 @@ runs: $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ $(printf "${image}@sha256:%s " *) else - echo Skipping command because push is set to false. + echo Skipping command since push is set to false. fi done else - echo "No images to push, skipping imagetools creation" + echo "::warning::No images to push, skipping imagetools creation" fi - name: Inspect image From 84dbe66a00f3fd345e01168d18ace531570ad2d8 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Sat, 20 Sep 2025 15:16:36 -0500 Subject: [PATCH 16/28] Add better handling for default buildkitd config --- actions/docker-build-push-image/action.yaml | 21 +++++++++++++++---- .../action.yaml | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/actions/docker-build-push-image/action.yaml b/actions/docker-build-push-image/action.yaml index eb5977e82..c4f7ddc18 100644 --- a/actions/docker-build-push-image/action.yaml +++ b/actions/docker-build-push-image/action.yaml @@ -1,4 +1,4 @@ -name: Push to artifact registry +name: Build and Push Docker Image description: Composite action to push a docker image to GAR or DockerHub inputs: @@ -254,11 +254,12 @@ runs: - name: Setup buildkitd-config id: buildkitd-config - shell: sh + shell: bash env: BUILDKITD_CONFIG_INLINE: ${{ inputs.buildkitd-config-inline }} BUILDKITD_CONFIG: ${{ inputs.buildkitd-config }} - DEFAULT_BUILDKITD_CONFIG: ${{ runner.environment == 'self-hosted' && '/etc/buildkitd.toml' || '' }} + DEFAULT_BUILDKITD_CONFIG: /etc/buildkitd.toml + RUNNER_ENVIRONMENT: ${{ runner.environment }} run: | # This step does the following: # if buildkitd-config-inline != "", use that @@ -269,11 +270,23 @@ runs: buildkitd_config_inline="" if [ -n "${BUILDKITD_CONFIG_INLINE}" ]; then + echo "Inline config was provided" buildkitd_config_inline="${BUILDKITD_CONFIG_INLINE}" elif [ -n "${BUILDKITD_CONFIG}" ]; then + echo "Config file was provided" buildkitd_config="${BUILDKITD_CONFIG}" else - buildkitd_config="${DEFAULT_BUILDKITD_CONFIG}" + echo "No configs were provided, building default config setting" + if [ "${RUNNER_ENVIRONMENT}" != "self-hosted" ]; then + echo "Not on self hosted runner, no config will be applied" + else + if [ -f "${DEFAULT_BUILDKITD_CONFIG}" ]; then + echo "${DEFAULT_BUILDKITD_CONFIG} exists, using that as default config." + buildkitd_config="${DEFAULT_BUILDKITD_CONFIG}" + else + echo "${DEFAULT_BUILDKITD_CONFIG} does not exist, no config will be applied" + fi + fi fi echo "buildkitd-config-inline=${buildkitd_config_inline}" | tee -a "${GITHUB_OUTPUT}" diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index 01aaaa7f1..43ac68b9b 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -1,4 +1,4 @@ -name: Download and Merge Digests into Manifest +name: Download and Merge Docker Digests into Manifest description: Composite action to export and upload a docker manifest inputs: From abf9d2a08be77cf714d2e0061658dd4b1da5a7c5 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Sat, 20 Sep 2025 15:28:45 -0500 Subject: [PATCH 17/28] Add warning when default buildkitd config doesn't exist --- actions/docker-build-push-image/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/docker-build-push-image/action.yaml b/actions/docker-build-push-image/action.yaml index c4f7ddc18..15eea5968 100644 --- a/actions/docker-build-push-image/action.yaml +++ b/actions/docker-build-push-image/action.yaml @@ -284,7 +284,7 @@ runs: echo "${DEFAULT_BUILDKITD_CONFIG} exists, using that as default config." buildkitd_config="${DEFAULT_BUILDKITD_CONFIG}" else - echo "${DEFAULT_BUILDKITD_CONFIG} does not exist, no config will be applied" + echo "::warning::${DEFAULT_BUILDKITD_CONFIG} does not exist, no config will be applied" fi fi fi From fc3c93fa927bda4d4df89acf0b5d071568e8ea02 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Tue, 23 Sep 2025 15:21:06 -0500 Subject: [PATCH 18/28] Various docs updates --- actions/docker-build-push-image/README.md | 51 ++++++++++- actions/docker-build-push-image/TEST.md | 85 +++++++++++++------ actions/docker-build-push-image/action.yaml | 2 +- actions/docker-export-digest/README.md | 11 ++- .../README.md | 15 ++-- 5 files changed, 120 insertions(+), 44 deletions(-) diff --git a/actions/docker-build-push-image/README.md b/actions/docker-build-push-image/README.md index f70b99d95..c2a9f15a7 100644 --- a/actions/docker-build-push-image/README.md +++ b/actions/docker-build-push-image/README.md @@ -2,9 +2,22 @@ This is a composite GitHub Action, used to build and push docker images to private Grafana registries. It builds registry URLs for Grafana's registries, authenticates to them, and then -uses [docker/build-push-action](https://github.com/docker/build-push-action) to build and push the image(s). +uses [docker/build-push-action] to build and push the image(s). -# TODO: do we need QEMU? +This action can work 1 of 2 ways: + +1. It can be run on a single runner, and if multiple `platforms` are configured then buildx/QEMU emulation is used. +2. It can be used in conjunction with [docker-export-digest] and [docker-import-digests-push-manifest] to push untagged + images whose digests are later exported and merged into a tagged docker manifest. For true multi-arch builds. + +This can push to the following registries: +1. Google Artifact Registry +2. DockerHub + +[docker/build-push-action]: https://github.com/docker/build-push-action +[docker-build-push-image]: ../docker-build-push-image/README.md +[docker-export-digest]: ../docker-export-digest/README.md +[docker-import-digests-push-manifest]: ../docker-import-digests-push-manifest/README.md @@ -80,7 +93,6 @@ jobs: | `tags` | String | Generated Docker tags (from docker/metadata-action) | | `version` | String | Generated Docker image version (from docker/metadata-action) | - ## How we construct Google Artifact Registry Images The full GAR image is constructed as follows, where `gar-project` is determined by `inputs.gar-environment`. @@ -91,3 +103,36 @@ The full GAR image is constructed as follows, where `gar-project` is determined The full DockerHub image is constructed as follows: +"${{ inputs.dockerhub-registry }}/${{ inputs.dockerhub-repository }}" + +## Adding New Registries + +This is currently configured to push to: + +Each registry is setup as follows: + +- All inputs for a registry share the same prefix (ex: `gar-image`, `gar-repository`). +- Inputs that are used for a specific registry are _not_ required by the workflow. Instead, validation is done in a step + specific to that registry. +- To calculate which registries have been configured, we loop through `inputs.registries`, and for each registry + configured we set the outputs `include-`. Those flags can be used to create steps that only execute when X + registry is configured. +- Each registry has a Setup step. This step takes the inputs specific to that registry and generates an untagged, + `image` name for that specific registry. +- The `setup-vars` step then loops through each configured image and creates a full list of images to push. +- That's it! That list of images to push is fed to `docker/build-push-action` along with the configured tags, and each + tagged image is pushed to each registry. + +So then the full checklist of work to do to implement a new registry is: + +- [ ] Add (and document) any inputs that you need to capture. Use the same prefix for all inputs, and all inputs must + _not_ be required. +- [ ] Add a step before `setup-vars` that takes those input values and constructs a valid untagged image name for the + registry you'll be pushing to. Then set that as an output. + Ex: `echo "image=${DOCKERHUB_REGISTRY}/${DOCKERHUB_IMAGE}" | tee -a "${GITHUB_OUTPUT}"` +- [ ] Add your image into the `setup-vars` step by passing the output image into an env variable, and adding it to the + list of images to be parsed. Use the existing repos as examples. +- [ ] Add a login step that depends + on `${{ inputs.push == 'true' && steps.registries.outputs.include- == 'true' }}`, where yourRegistry is + the value that will be passed into the `registries` input. Again, use existing repos as examples. +- [ ] Celebrate diff --git a/actions/docker-build-push-image/TEST.md b/actions/docker-build-push-image/TEST.md index dc288c5e6..823318707 100644 --- a/actions/docker-build-push-image/TEST.md +++ b/actions/docker-build-push-image/TEST.md @@ -2,33 +2,66 @@ Design decisions ## Whole workflow -1. All actions are prefixed with `docker-`, to improve discoverability. +1. The guide I used to build multi-arch images is + here: https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners +1. The names of all the new actions are prefixed with `docker-`, to improve discoverability. +1. I've tried to use docker's official actions ecosystem as much as possible. Most inputs are directly passed + through to the underlying docker actions, unless some sort of custom logic is necessary. +1. I've avoided repeating actions in places where I can. An example of this is one area where I differentiated from the + official guide (linked in item #1). Docker shows the metadata action being used twice (once when building the + image and once when pushing the manifest). Instead of setting up the metadata action a second time, I capture the + output from the original build, and use it as an input when pushing the manifests. +1. I tried to make this as reusable as possible. An example of this is GAR images: previously we made some assumptions + about the naming convention for GAR repos. Now you can now configure the entire image URL/path, so if your GAR bucket + doesn't match our naming convention this will (probably) still work. -docker-build-push-image +# docker-build-push-image -1. This should handle both gar and dockerhub logins, with a framework in place that makes it easy to add other - registries in the future. Which registries we build images for and login to should be configurable. -2. This should handle both pushing tags and digests (for multiarch builds). The idea is that we can use this as a direct +1. This handles both gar and dockerhub logins, with a framework in place that makes it easy to add other + registries in the future. How to add new registries is documented in the README. +2. This handles both pushing tags and digests (for multiarch builds). The idea is that we can use this as a direct replacement for both `push-to-gar-docker` and `build-push-to-dockerhub`, while also making it flexible enough that we - can use it for true multiarch builds. -2. To configure which registries we build and push to, an input is provided that takes a CSV of + can use it for true multiarch builds, and good ole-fashioned emulation builds. +3. To configure which registries we build and push to, an input is provided that takes a CSV of registries (`gar,dockerhub`). -3. For each registry that we configure, a script is included that takes the inputs for that registry and outputs an - image to the - GITHUB_OUTPUT. Each image (without tags) is then appended to the `images` output. Inputs for each registry begin with - the same prefix (ex: `gar-config-value`) -4. For each registry that we configure, we only login to that registry if it's in the `registries` input list. -5. This action outputs both a tagged image list and an image list without tags. This is because when you're pushing a - manifest, tags cannot be - included; however when you're pushing tags, you have to include them. So we include both. -6. If `include-tags-in-push=true`, then we push the tag list, if `include-tags-in-push=false` then we push the image - list. -7. To provide access to Grafana's Docker mirror, if a workflow is running on a self-hosted runner, and no - buildkitd-config is provided, then we default to `/etc/buildkitd.toml`. This means if you're running this action from - inside a container you need to either mount that directory, or explicitly override it by setting `buidkitd-config` - or `buidkitd-config-inline`. -5. -6. -7. (Ex: images=us-docker.pkg.dev/grafanalabs-dev/gar-registry/image-name,docker.io/grafana/dockerhub-image) -5. -3. We only authenticate to registries that are included in the `` +4. For each registry that we configure, a script is included that takes the inputs for that registry and outputs an + image to the GITHUB_OUTPUT. Each image (without tags) is then appended to the `images` output. Inputs for each + registry begin with the same prefix (ex: `gar-config-value`) +5. For each registry that we configure, we only login to that registry if it's in the `registries` input list. +6. This action outputs both a tagged image list and an image list without tags. This is because when you're pushing a + manifest, tags cannot be included; however when you're pushing tags, you have to include them. So we offer the + ability to do either. +7. If `include-tags-in-push=true`, then we push the tags list, if `include-tags-in-push=false` then we push the untagged + images. When untagged images are pushed you must collect the docker digests and merge them into a manifest in a + following step/job. +8. To provide access to Grafana's Docker mirror, if a workflow is running on a self-hosted runner, and no + buildkitd-config is provided, then we set the default to `/etc/buildkitd.toml` if it exists. +9. In the event `/etc/buildkitd.toml` does not exist then we will not set a default buildkitd-config. This allows the + action to work by default when run inside a container. +10. Most inputs are passed directly to the underlying `docker/build-push-action`, with the exception of tags. The tags + are built depending on the input values of `include-tags-in-push` and `registries`. +11. If this action is run with `push=false` then we still successfully build images. However a warning is logged saying + that we're not pushing anything. +12. If this action is run without any registries configured then we still successfully build an image. We construct a + fake image name and log a warning that we're not pushing anything. + +## docker-export-digest + +1. The `digest` input can be collected from `docker-build-push-image` or `docker/build-push-image`; both of which + output a docker digest. + +## docker-import-digests-push-manifest + +1. The `docker-metadata-json` and `images` inputs can be captured from `docker-build-push-image`. +2. The `gar-environment` and `push` match inputs that are also used by `docker-build-push-image`. +3. For simplicity, I did not replicate the `setup-vars` logic from `docker-build-push-image`, and instead we log in to + both GAR and DockerHub when `push=true`. + +## Potential Follow-up Items + +1. Should we convert naming to match new standards? + - `actions/dockerhub-login` -> `actions/login-to-dockerhub` +2. Deprecate `actions/push-to-gar-docker` and `actions/build-push-to-dockerhub` in favor + of `actions/docker-build-push-image` +3. Integrated GHCR registry into `docker-build-push-image`. + diff --git a/actions/docker-build-push-image/action.yaml b/actions/docker-build-push-image/action.yaml index 15eea5968..24e730ac4 100644 --- a/actions/docker-build-push-image/action.yaml +++ b/actions/docker-build-push-image/action.yaml @@ -243,7 +243,7 @@ runs: - name: Login to DockerHub if: ${{ inputs.push == 'true' && steps.registries.outputs.include-dockerhub == 'true' }} - uses: ./_shared-workflows-docker-build-push-image/actions/dockerhub-login # TODO: Write up an issue to convert this to actions/login-to-dockerhub + uses: ./_shared-workflows-docker-build-push-image/actions/dockerhub-login - name: Extract metadata (tags, labels) for Docker id: meta diff --git a/actions/docker-export-digest/README.md b/actions/docker-export-digest/README.md index c7a3739ba..769feac27 100644 --- a/actions/docker-export-digest/README.md +++ b/actions/docker-export-digest/README.md @@ -3,13 +3,12 @@ This is a composite GitHub Action used to export a docker digest as a workflow artifact, so it can be merged and pushed as part of a manifest. -[//]: # (TODO: Link to docs for the following) -This is meant to work in conjuction with `docker-build-push-image` and `docker-import-digests-push-manifest`. +This is meant to work in conjunction with [docker-build-push-image] and [docker-import-digests-push-manifest]. -It builds registry URLs for Grafana's registries, authenticates to them, and then -uses [docker/build-push-action](https://github.com/docker/build-push-action) to build and push the image(s). - -# TODO: do we need QEMU? +[docker/build-push-action]: https://github.com/docker/build-push-action +[docker-build-push-image]: ../docker-build-push-image/README.md +[docker-export-digest]: ../docker-export-digest/README.md +[docker-import-digests-push-manifest]: ../docker-import-digests-push-manifest/README.md diff --git a/actions/docker-import-digests-push-manifest/README.md b/actions/docker-import-digests-push-manifest/README.md index c185c0212..7719f397f 100644 --- a/actions/docker-import-digests-push-manifest/README.md +++ b/actions/docker-import-digests-push-manifest/README.md @@ -1,15 +1,14 @@ # docker-import-digest-push-manifest -This is a composite GitHub Action used to export a docker digest as a workflow artifact, so it can be merged and pushed -as part of a manifest. +This is a composite GitHub Action used to import Docker digests from a shared workflow artifact and merge them into a +tagged manifest. -[//]: # (TODO: Link to docs for the following) -This is meant to work in conjuction with `docker-build-push-image` and `docker-import-digests-push-manifest`. +This is meant to work in conjunction with [docker-build-push-image] and [docker-export-digest]. -It builds registry URLs for Grafana's registries, authenticates to them, and then -uses [docker/build-push-action](https://github.com/docker/build-push-action) to build and push the image(s). - -# TODO: do we need QEMU? +[docker/build-push-action]: https://github.com/docker/build-push-action +[docker-build-push-image]: ../docker-build-push-image/README.md +[docker-export-digest]: ../docker-export-digest/README.md +[docker-import-digests-push-manifest]: ../docker-import-digests-push-manifest/README.md From 8162046f5b9e408967385ba9edda998defffdc1d Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Tue, 23 Sep 2025 15:26:06 -0500 Subject: [PATCH 19/28] prettier --- actions/docker-build-push-image/README.md | 17 +++++++++-------- actions/docker-build-push-image/TEST.md | 3 +-- actions/docker-export-digest/README.md | 4 ++-- .../README.md | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/actions/docker-build-push-image/README.md b/actions/docker-build-push-image/README.md index c2a9f15a7..01ee4ceff 100644 --- a/actions/docker-build-push-image/README.md +++ b/actions/docker-build-push-image/README.md @@ -11,6 +11,7 @@ This action can work 1 of 2 ways: images whose digests are later exported and merged into a tagged docker manifest. For true multi-arch builds. This can push to the following registries: + 1. Google Artifact Registry 2. DockerHub @@ -50,7 +51,7 @@ jobs: ## Inputs | Name | Type | Description | -|-------------------------------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ----------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `build-args` | String | List of arguments necessary for the Docker image to be built. Passed to `docker/build-push-action`. | | `build-contexts` | String | List of additional build contexts (e.g., name=path). Passed to `docker/build-push-action`. | | `buildkitd-config` | String | The buildkitd config file to use. Defaults to `/etc/buildkitd.toml` if you're using Grafana's self-hosted runners. Passed to `docker/setup-buildx-action`. | @@ -81,7 +82,7 @@ jobs: ## Outputs | Name | Type | Description | -|----------------|--------|--------------------------------------------------------------| +| -------------- | ------ | ------------------------------------------------------------ | | `annotations` | String | Generated annotations (from docker/metadata-action) | | `digest` | String | Image digest (from docker/build-push-action) | | `imageid` | String | Image ID (from docker/build-push-action) | @@ -126,13 +127,13 @@ Each registry is setup as follows: So then the full checklist of work to do to implement a new registry is: - [ ] Add (and document) any inputs that you need to capture. Use the same prefix for all inputs, and all inputs must - _not_ be required. + _not_ be required. - [ ] Add a step before `setup-vars` that takes those input values and constructs a valid untagged image name for the - registry you'll be pushing to. Then set that as an output. - Ex: `echo "image=${DOCKERHUB_REGISTRY}/${DOCKERHUB_IMAGE}" | tee -a "${GITHUB_OUTPUT}"` + registry you'll be pushing to. Then set that as an output. + Ex: `echo "image=${DOCKERHUB_REGISTRY}/${DOCKERHUB_IMAGE}" | tee -a "${GITHUB_OUTPUT}"` - [ ] Add your image into the `setup-vars` step by passing the output image into an env variable, and adding it to the - list of images to be parsed. Use the existing repos as examples. + list of images to be parsed. Use the existing repos as examples. - [ ] Add a login step that depends - on `${{ inputs.push == 'true' && steps.registries.outputs.include- == 'true' }}`, where yourRegistry is - the value that will be passed into the `registries` input. Again, use existing repos as examples. + on `${{ inputs.push == 'true' && steps.registries.outputs.include- == 'true' }}`, where yourRegistry is + the value that will be passed into the `registries` input. Again, use existing repos as examples. - [ ] Celebrate diff --git a/actions/docker-build-push-image/TEST.md b/actions/docker-build-push-image/TEST.md index 823318707..98dc1c117 100644 --- a/actions/docker-build-push-image/TEST.md +++ b/actions/docker-build-push-image/TEST.md @@ -60,8 +60,7 @@ Design decisions ## Potential Follow-up Items 1. Should we convert naming to match new standards? - - `actions/dockerhub-login` -> `actions/login-to-dockerhub` + - `actions/dockerhub-login` -> `actions/login-to-dockerhub` 2. Deprecate `actions/push-to-gar-docker` and `actions/build-push-to-dockerhub` in favor of `actions/docker-build-push-image` 3. Integrated GHCR registry into `docker-build-push-image`. - diff --git a/actions/docker-export-digest/README.md b/actions/docker-export-digest/README.md index 769feac27..9cb1a206e 100644 --- a/actions/docker-export-digest/README.md +++ b/actions/docker-export-digest/README.md @@ -57,7 +57,7 @@ jobs: uses: grafana/shared-workflows/actions/docker-import-digests-push-manifest@main # TODO: Pin sha with: images: ${{ needs.build-push-image.outputs.images }} - gar-environment: 'dev' + gar-environment: "dev" registries: "gar,dockerhub" docker-metadata-json: ${{ needs.build-and-push.outputs.metadatajson }} ``` @@ -67,6 +67,6 @@ jobs: ## Inputs | Name | Type | Description | -|------------|--------|------------------------------------------------------------------------------------------------------------| +| ---------- | ------ | ---------------------------------------------------------------------------------------------------------- | | `digest` | String | Docker digest. This is included as an output for `docker-build-push-image` and `docker/build-push-action`. | | `platform` | String | Docker platform, ex: linux/arm64. | diff --git a/actions/docker-import-digests-push-manifest/README.md b/actions/docker-import-digests-push-manifest/README.md index 7719f397f..aae72ec09 100644 --- a/actions/docker-import-digests-push-manifest/README.md +++ b/actions/docker-import-digests-push-manifest/README.md @@ -57,7 +57,7 @@ jobs: uses: grafana/shared-workflows/actions/docker-import-digests-push-manifest@main # TODO: Pin sha with: images: ${{ needs.build-push-image.outputs.images }} - gar-environment: 'dev' + gar-environment: "dev" registries: "gar,dockerhub" docker-metadata-json: ${{ needs.build-and-push.outputs.metadatajson }} ``` @@ -67,7 +67,7 @@ jobs: ## Inputs | Name | Type | Description | -|------------------------|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ---------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `docker-metadata-json` | String | Docker metadata JSON, from `docker-build-push-image` or `docker/build-push-action`. | | `gar-environment` | String | Environment for pushing artifacts (can be either dev or prod). This sets the GAR Project to either `grafanalabs-dev` or `grafanalabs-global`. | | `images` | String | CSV of Docker images to push. These images should not include tags. Ex: us-docker.pkg.dev/grafanalabs-dev/gar-registry/image-name,docker.io/grafana/dockerhub-image | From d875e6f0197e62fcb897c58c37604002effc1da6 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Tue, 23 Sep 2025 15:30:57 -0500 Subject: [PATCH 20/28] Updated inputs in README --- actions/docker-build-push-image/README.md | 3 ++- actions/docker-import-digests-push-manifest/README.md | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/actions/docker-build-push-image/README.md b/actions/docker-build-push-image/README.md index 01ee4ceff..b0432c7b7 100644 --- a/actions/docker-build-push-image/README.md +++ b/actions/docker-build-push-image/README.md @@ -60,7 +60,8 @@ jobs: | `cache-to` | String | Where cache should be stored to. Passed to `docker/build-push-action`. | | `context` | String | Path to the Docker build context. Passed to `docker/build-push-action`. | | `docker-buildx-driver` | String | The driver to use for Docker Buildx. Passed to `docker/setup-buildx-action`. | -| `dockerhub-repository` | String | Ipsum dockerhubium | +| `dockerhub-registry` | String | DockerHub Registry to store docker images in. | +| `dockerhub-repository` | String | DockerHub Repository to store docker images in. Default: github.repository | | `file` | String | The dockerfile to use. Passed to `docker/build-push-action`. | | `gar-delete-credentials-file` | Boolean | Delete the Google credentials file after the action is finished. If you want to keep the credentials file for a later step, set this to false. | | `gar-environment` | String | Environment for pushing artifacts (can be either dev or prod). This sets the GAR Project (gar-project) to either `grafanalabs-dev` or `grafanalabs-global`. | diff --git a/actions/docker-import-digests-push-manifest/README.md b/actions/docker-import-digests-push-manifest/README.md index aae72ec09..7f22045a5 100644 --- a/actions/docker-import-digests-push-manifest/README.md +++ b/actions/docker-import-digests-push-manifest/README.md @@ -66,8 +66,9 @@ jobs: ## Inputs -| Name | Type | Description | -| ---------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `docker-metadata-json` | String | Docker metadata JSON, from `docker-build-push-image` or `docker/build-push-action`. | -| `gar-environment` | String | Environment for pushing artifacts (can be either dev or prod). This sets the GAR Project to either `grafanalabs-dev` or `grafanalabs-global`. | -| `images` | String | CSV of Docker images to push. These images should not include tags. Ex: us-docker.pkg.dev/grafanalabs-dev/gar-registry/image-name,docker.io/grafana/dockerhub-image | +| Name | Type | Description | +| ---------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `docker-metadata-json` | String | Docker metadata JSON, from `docker-build-push-image` or `docker/build-push-action`. | +| `gar-environment` | String | Environment for pushing artifacts (can be either dev or prod). This sets the GAR Project to either `grafanalabs-dev` or `grafanalabs-global`. | +| `images` | String | CSV of Docker images to push. These images should not include tags. Ex: us-docker.pkg.dev/grafanalabs-dev/gar-registry/image-name,docker.io/grafana/dockerhub-image | +| `push` | Boolean | Whether to push the manifest to the configured registries. | From 5089c2d940f5d45fcad4c6735b00adc8f9a59813 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Wed, 24 Sep 2025 12:59:02 -0500 Subject: [PATCH 21/28] remove test notes --- actions/docker-build-push-image/TEST.md | 66 ------------------------- 1 file changed, 66 deletions(-) delete mode 100644 actions/docker-build-push-image/TEST.md diff --git a/actions/docker-build-push-image/TEST.md b/actions/docker-build-push-image/TEST.md deleted file mode 100644 index 98dc1c117..000000000 --- a/actions/docker-build-push-image/TEST.md +++ /dev/null @@ -1,66 +0,0 @@ -Design decisions - -## Whole workflow - -1. The guide I used to build multi-arch images is - here: https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners -1. The names of all the new actions are prefixed with `docker-`, to improve discoverability. -1. I've tried to use docker's official actions ecosystem as much as possible. Most inputs are directly passed - through to the underlying docker actions, unless some sort of custom logic is necessary. -1. I've avoided repeating actions in places where I can. An example of this is one area where I differentiated from the - official guide (linked in item #1). Docker shows the metadata action being used twice (once when building the - image and once when pushing the manifest). Instead of setting up the metadata action a second time, I capture the - output from the original build, and use it as an input when pushing the manifests. -1. I tried to make this as reusable as possible. An example of this is GAR images: previously we made some assumptions - about the naming convention for GAR repos. Now you can now configure the entire image URL/path, so if your GAR bucket - doesn't match our naming convention this will (probably) still work. - -# docker-build-push-image - -1. This handles both gar and dockerhub logins, with a framework in place that makes it easy to add other - registries in the future. How to add new registries is documented in the README. -2. This handles both pushing tags and digests (for multiarch builds). The idea is that we can use this as a direct - replacement for both `push-to-gar-docker` and `build-push-to-dockerhub`, while also making it flexible enough that we - can use it for true multiarch builds, and good ole-fashioned emulation builds. -3. To configure which registries we build and push to, an input is provided that takes a CSV of - registries (`gar,dockerhub`). -4. For each registry that we configure, a script is included that takes the inputs for that registry and outputs an - image to the GITHUB_OUTPUT. Each image (without tags) is then appended to the `images` output. Inputs for each - registry begin with the same prefix (ex: `gar-config-value`) -5. For each registry that we configure, we only login to that registry if it's in the `registries` input list. -6. This action outputs both a tagged image list and an image list without tags. This is because when you're pushing a - manifest, tags cannot be included; however when you're pushing tags, you have to include them. So we offer the - ability to do either. -7. If `include-tags-in-push=true`, then we push the tags list, if `include-tags-in-push=false` then we push the untagged - images. When untagged images are pushed you must collect the docker digests and merge them into a manifest in a - following step/job. -8. To provide access to Grafana's Docker mirror, if a workflow is running on a self-hosted runner, and no - buildkitd-config is provided, then we set the default to `/etc/buildkitd.toml` if it exists. -9. In the event `/etc/buildkitd.toml` does not exist then we will not set a default buildkitd-config. This allows the - action to work by default when run inside a container. -10. Most inputs are passed directly to the underlying `docker/build-push-action`, with the exception of tags. The tags - are built depending on the input values of `include-tags-in-push` and `registries`. -11. If this action is run with `push=false` then we still successfully build images. However a warning is logged saying - that we're not pushing anything. -12. If this action is run without any registries configured then we still successfully build an image. We construct a - fake image name and log a warning that we're not pushing anything. - -## docker-export-digest - -1. The `digest` input can be collected from `docker-build-push-image` or `docker/build-push-image`; both of which - output a docker digest. - -## docker-import-digests-push-manifest - -1. The `docker-metadata-json` and `images` inputs can be captured from `docker-build-push-image`. -2. The `gar-environment` and `push` match inputs that are also used by `docker-build-push-image`. -3. For simplicity, I did not replicate the `setup-vars` logic from `docker-build-push-image`, and instead we log in to - both GAR and DockerHub when `push=true`. - -## Potential Follow-up Items - -1. Should we convert naming to match new standards? - - `actions/dockerhub-login` -> `actions/login-to-dockerhub` -2. Deprecate `actions/push-to-gar-docker` and `actions/build-push-to-dockerhub` in favor - of `actions/docker-build-push-image` -3. Integrated GHCR registry into `docker-build-push-image`. From c0c6bba89f5296d090f1cd2278b21500604235fa Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Wed, 1 Oct 2025 16:51:37 -0500 Subject: [PATCH 22/28] Add some help text to docker-build-push-image --- actions/docker-build-push-image/action.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/actions/docker-build-push-image/action.yaml b/actions/docker-build-push-image/action.yaml index 24e730ac4..22caa0fd8 100644 --- a/actions/docker-build-push-image/action.yaml +++ b/actions/docker-build-push-image/action.yaml @@ -175,6 +175,11 @@ runs: env: REGISTRIES: ${{ inputs.registries }} run: | + ############################################################# + # This splits REGISTRIES on commas, and then exports + # include-=true for each registry in the list + ############################################################# + for r in ${REGISTRIES//,/ }; do echo "include-${r}=true" | tee -a "${GITHUB_OUTPUT}" done @@ -212,6 +217,16 @@ runs: DOCKERHUB_IMAGE: ${{ steps.setup-dockerhub-vars.outputs.image }} PUSH: ${{ inputs.push }} run: | + ############################################################# + # This constructs as CSV list of images from the previous + # setup steps, and outputs that list + # + # If there are no images then we set images="dry-run-image" + # and log a warning that no images were configured + # + # If push is not true, then we log a warning that no images will be pushed + ############################################################# + images="" for image in "${GAR_IMAGE}" "${DOCKERHUB_IMAGE}" do @@ -261,10 +276,12 @@ runs: DEFAULT_BUILDKITD_CONFIG: /etc/buildkitd.toml RUNNER_ENVIRONMENT: ${{ runner.environment }} run: | + ############################################################# # This step does the following: # if buildkitd-config-inline != "", use that # elif buildkitd-config != "", use that # else, use buildkitd-config default config if on self hosted runners + ############################################################# buildkitd_config="" buildkitd_config_inline="" From 01752f777542cccac9722de478fb9eaf9fd80278 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Wed, 1 Oct 2025 22:00:45 -0500 Subject: [PATCH 23/28] Switch to cloned shared workflows for docker-import-digests-push-manifest Add TODOs --- actions/docker-export-digest/README.md | 2 +- .../docker-import-digests-push-manifest/README.md | 2 +- .../action.yaml | 15 +++++++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/actions/docker-export-digest/README.md b/actions/docker-export-digest/README.md index 9cb1a206e..37d6a68b1 100644 --- a/actions/docker-export-digest/README.md +++ b/actions/docker-export-digest/README.md @@ -41,7 +41,7 @@ jobs: include-tags-in-push: false outputs: "type=image,push-by-digest=true,name-canonical=true,push=true" - name: Export and upload digest - uses: grafana/shared-workflows/actions/docker-export-digest@rwhitaker/multi-arch-builds + uses: grafana/shared-workflows/actions/docker-export-digest@rwhitaker/multi-arch-builds # TODO: Fix version once released with: digest: ${{ steps.build.outputs.digest }} platform: linux/arm64 diff --git a/actions/docker-import-digests-push-manifest/README.md b/actions/docker-import-digests-push-manifest/README.md index 7f22045a5..edc3c6a39 100644 --- a/actions/docker-import-digests-push-manifest/README.md +++ b/actions/docker-import-digests-push-manifest/README.md @@ -41,7 +41,7 @@ jobs: include-tags-in-push: false outputs: "type=image,push-by-digest=true,name-canonical=true,push=true" - name: Export and upload digest - uses: grafana/shared-workflows/actions/docker-export-digest@rwhitaker/multi-arch-builds + uses: grafana/shared-workflows/actions/docker-export-digest@rwhitaker/multi-arch-builds # TODO: Fix version once released with: digest: ${{ steps.build.outputs.digest }} platform: linux/arm64 diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index 43ac68b9b..21f282f8d 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -24,6 +24,17 @@ inputs: runs: using: composite steps: + - name: Checkout shared-workflows + env: + action_repo: ${{ github.action_repository }} + action_ref: ${{ github.action_ref }} + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + repository: ${{ env.action_repo }} + ref: ${{ env.action_ref }} + path: _shared-workflows-docker-import-digests-push-manifest + persist-credentials: false + - name: Download digests uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: @@ -39,14 +50,14 @@ runs: - name: Login to GAR if: ${{ inputs.push == 'true' }} - uses: grafana/shared-workflows/actions/login-to-gar@rwhitaker/multi-arch-builds + uses: ./_shared-workflows-docker-import-digests-push-manifest/actions/login-to-gar with: environment: ${{ inputs.gar-environment }} delete_credentials_file: false - name: Login to DockerHub if: ${{ inputs.push == 'true' }} - uses: grafana/shared-workflows/actions/dockerhub-login@rwhitaker/multi-arch-builds + uses: ./_shared-workflows-docker-import-digests-push-manifest/actions/dockerhub-login - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests From 6009ebe6fa2ea1ca141d728afc0bd14334c58a64 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Thu, 2 Oct 2025 07:47:20 -0500 Subject: [PATCH 24/28] Fix login to gar action in docker-import-digests-push-manifest --- actions/docker-import-digests-push-manifest/action.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index 21f282f8d..b5fd4c9d0 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -51,9 +51,6 @@ runs: - name: Login to GAR if: ${{ inputs.push == 'true' }} uses: ./_shared-workflows-docker-import-digests-push-manifest/actions/login-to-gar - with: - environment: ${{ inputs.gar-environment }} - delete_credentials_file: false - name: Login to DockerHub if: ${{ inputs.push == 'true' }} From 839537168e1db18249a184b5b4f31e12e73c73ae Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Thu, 2 Oct 2025 17:04:20 -0500 Subject: [PATCH 25/28] adding recommendations --- actions/docker-build-push-image/README.md | 4 +- actions/docker-build-push-image/action.yaml | 11 +---- actions/docker-export-digest/README.md | 41 +++-------------- .../README.md | 40 +++------------- .../action.yaml | 46 ++++++++++++++++++- 5 files changed, 60 insertions(+), 82 deletions(-) diff --git a/actions/docker-build-push-image/README.md b/actions/docker-build-push-image/README.md index b0432c7b7..f6b4b61bd 100644 --- a/actions/docker-build-push-image/README.md +++ b/actions/docker-build-push-image/README.md @@ -36,7 +36,7 @@ jobs: contents: read id-token: write steps: - - uses: grafana/shared-workflows/actions/docker-build-push-image@main # TODO: Fix version once released + - uses: grafana/shared-workflows/actions/docker-build-push-image@docker-build-push-image/v0.0.0 with: platforms: linux/arm64,linux/amd64 tags: | @@ -109,8 +109,6 @@ The full DockerHub image is constructed as follows: ## Adding New Registries -This is currently configured to push to: - Each registry is setup as follows: - All inputs for a registry share the same prefix (ex: `gar-image`, `gar-repository`). diff --git a/actions/docker-build-push-image/action.yaml b/actions/docker-build-push-image/action.yaml index 22caa0fd8..7acb71c3c 100644 --- a/actions/docker-build-push-image/action.yaml +++ b/actions/docker-build-push-image/action.yaml @@ -218,7 +218,7 @@ runs: PUSH: ${{ inputs.push }} run: | ############################################################# - # This constructs as CSV list of images from the previous + # This constructs a CSV list of images from the previous # setup steps, and outputs that list # # If there are no images then we set images="dry-run-image" @@ -276,13 +276,6 @@ runs: DEFAULT_BUILDKITD_CONFIG: /etc/buildkitd.toml RUNNER_ENVIRONMENT: ${{ runner.environment }} run: | - ############################################################# - # This step does the following: - # if buildkitd-config-inline != "", use that - # elif buildkitd-config != "", use that - # else, use buildkitd-config default config if on self hosted runners - ############################################################# - buildkitd_config="" buildkitd_config_inline="" @@ -361,7 +354,7 @@ runs: rm -rf _shared-workflows-docker-build-push-image - name: Delete Google Application Credentials file - if: ${{ inputs.gar-delete-credentials-file == 'true' && env.GOOGLE_APPLICATION_CREDENTIALS != '' }} + if: ${{ always() && inputs.gar-delete-credentials-file == 'true' && env.GOOGLE_APPLICATION_CREDENTIALS != '' }} shell: sh run: | if [ -f "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then diff --git a/actions/docker-export-digest/README.md b/actions/docker-export-digest/README.md index 37d6a68b1..c49910873 100644 --- a/actions/docker-export-digest/README.md +++ b/actions/docker-export-digest/README.md @@ -1,9 +1,9 @@ # docker-export-digest -This is a composite GitHub Action used to export a docker digest as a workflow artifact, so it can be merged and pushed -as part of a manifest. +This is a composite GitHub Action used to export a docker digest as a workflow artifact. -This is meant to work in conjunction with [docker-build-push-image] and [docker-import-digests-push-manifest]. +This can be used in conjunction with [docker-build-push-image] and [docker-import-digests-push-manifest] to build +native multi-arch Docker images. [docker/build-push-action]: https://github.com/docker/build-push-action [docker-build-push-image]: ../docker-build-push-image/README.md @@ -21,45 +21,16 @@ on: - main jobs: - build-push-image: - outputs: - images: ${{ steps.build.outputs.images }} + upload-digest-as-artifact: permissions: contents: read id-token: write steps: - - name: Build Docker Image - id: build - uses: grafana/shared-workflows/actions/docker-build-push-image@main # TODO: Fix version once released - with: - platforms: linux/arm64 - tags: | - ${{ github.sha }} - main - push: true - registries: "gar,dockerhub" - include-tags-in-push: false - outputs: "type=image,push-by-digest=true,name-canonical=true,push=true" - name: Export and upload digest - uses: grafana/shared-workflows/actions/docker-export-digest@rwhitaker/multi-arch-builds # TODO: Fix version once released + uses: grafana/shared-workflows/actions/docker-export-digest@docker-export-digest/v0.0.0 with: - digest: ${{ steps.build.outputs.digest }} + digest: ${{ steps.docker-build-push-image.outputs.digest }} platform: linux/arm64 - merge-digest: - if: ${{ inputs.push == 'true' }} - runs-on: ubuntu-arm64-small - needs: build-and-push - permissions: - contents: read - id-token: write - steps: - - name: Download Multi-Arch Digests, Construct and Upload Manifest - uses: grafana/shared-workflows/actions/docker-import-digests-push-manifest@main # TODO: Pin sha - with: - images: ${{ needs.build-push-image.outputs.images }} - gar-environment: "dev" - registries: "gar,dockerhub" - docker-metadata-json: ${{ needs.build-and-push.outputs.metadatajson }} ``` diff --git a/actions/docker-import-digests-push-manifest/README.md b/actions/docker-import-digests-push-manifest/README.md index edc3c6a39..54e64edc0 100644 --- a/actions/docker-import-digests-push-manifest/README.md +++ b/actions/docker-import-digests-push-manifest/README.md @@ -3,7 +3,8 @@ This is a composite GitHub Action used to import Docker digests from a shared workflow artifact and merge them into a tagged manifest. -This is meant to work in conjunction with [docker-build-push-image] and [docker-export-digest]. +This can be used in conjunction with [docker-build-push-image] and [docker-export-digest] to build +native multi-arch Docker images. [docker/build-push-action]: https://github.com/docker/build-push-action [docker-build-push-image]: ../docker-build-push-image/README.md @@ -21,45 +22,18 @@ on: - main jobs: - build-push-image: - outputs: - images: ${{ steps.build.outputs.images }} - permissions: - contents: read - id-token: write - steps: - - name: Build Docker Image - id: build - uses: grafana/shared-workflows/actions/docker-build-push-image@main # TODO: Fix version once released - with: - platforms: linux/arm64 - tags: | - ${{ github.sha }} - main - push: true - registries: "gar,dockerhub" - include-tags-in-push: false - outputs: "type=image,push-by-digest=true,name-canonical=true,push=true" - - name: Export and upload digest - uses: grafana/shared-workflows/actions/docker-export-digest@rwhitaker/multi-arch-builds # TODO: Fix version once released - with: - digest: ${{ steps.build.outputs.digest }} - platform: linux/arm64 - merge-digest: - if: ${{ inputs.push == 'true' }} - runs-on: ubuntu-arm64-small - needs: build-and-push + import-and-merge-digest: permissions: contents: read id-token: write steps: - name: Download Multi-Arch Digests, Construct and Upload Manifest - uses: grafana/shared-workflows/actions/docker-import-digests-push-manifest@main # TODO: Pin sha + uses: grafana/shared-workflows/actions/docker-import-digests-push-manifest@docker-import-digests-push-manifest/v0.0.0 with: - images: ${{ needs.build-push-image.outputs.images }} + docker-metadata-json: ${{ needs.docker-build-push-image.outputs.metadatajson }} gar-environment: "dev" - registries: "gar,dockerhub" - docker-metadata-json: ${{ needs.build-and-push.outputs.metadatajson }} + images: ${{ needs.docker-build-push-image.outputs.images }} + push: true ``` diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index b5fd4c9d0..1833ca357 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -48,12 +48,54 @@ runs: driver: docker-container version: latest # see https://github.com/docker/build-push-action/issues/1345#issuecomment-2770572479 - - name: Login to GAR + - name: Prepare vars + id: prepare-vars if: ${{ inputs.push == 'true' }} + env: + IMAGES: ${{ inputs.images }} + shell: bash + run: | + set -euo pipefail + + DOCKERHUB_IMAGE=false + GAR_IMAGE=false + + IFS=',' read -ra IMAGE_LIST <<< "${IMAGES}" + + for image in "${IMAGE_LIST[@]}"; do + image="$(echo "$image" | xargs)" # trim spaces + registry="${image%%/*}" # everything before first slash + echo "Verifying image: $image" + echo "Verifying registry: $registry" + + # Default if there's no dot or colon (Docker Hub shorthand) + if [[ "$registry" != *.* && "$registry" != *:* ]]; then + DOCKERHUB_IMAGE=true + fi + + if [[ "$registry" == *".pkg.dev" ]] || [[ "$registry" == *"gcr.io" ]]; then + echo "$image → Google Artifact Registry" + GAR_IMAGE=true + elif [[ "$registry" == "docker.io" ]] || [[ "$registry" == "index.docker.io" ]]; then + echo "$image → DockerHub" + DOCKERHUB_IMAGE=true + else + echo "$image → Other registry ($registry)" + fi + done + + if [[ "$DOCKERHUB_IMAGE" == "true" ]]; then + echo "include-dockerhub=true" | tee -a "${GITHUB_OUTPUT}" + elif [[ "$GAR_IMAGE" == "true" ]]; then + echo "include-gar=true" | tee -a "${GITHUB_OUTPUT}" + fi + + - name: Login to GAR + if: ${{ steps.prepare-vars.outputs.include-gar == 'true' }} uses: ./_shared-workflows-docker-import-digests-push-manifest/actions/login-to-gar - name: Login to DockerHub - if: ${{ inputs.push == 'true' }} + if: ${{ steps.prepare-vars.outputs.include-dockerhub == 'true' }} uses: ./_shared-workflows-docker-import-digests-push-manifest/actions/dockerhub-login - name: Create manifest list and push From 7edbaf4d616e0c0e349626b933bb9e8e75018fa8 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Wed, 8 Oct 2025 13:21:31 -0500 Subject: [PATCH 26/28] Add login logic to docker-import-digests-push-manifest --- actions/docker-import-digests-push-manifest/action.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index 1833ca357..e18d2da2b 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -86,7 +86,8 @@ runs: if [[ "$DOCKERHUB_IMAGE" == "true" ]]; then echo "include-dockerhub=true" | tee -a "${GITHUB_OUTPUT}" - elif [[ "$GAR_IMAGE" == "true" ]]; then + fi + if [[ "$GAR_IMAGE" == "true" ]]; then echo "include-gar=true" | tee -a "${GITHUB_OUTPUT}" fi From 53b79068e87037c5eebc68ef3f4a9fc824029d13 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Wed, 8 Oct 2025 13:38:12 -0500 Subject: [PATCH 27/28] Fix missing metadata in docker-import-digests-push-manifest --- .../action.yaml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index e18d2da2b..d540362e0 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -20,6 +20,10 @@ inputs: description: | Whether to push the manifest to the configured registries. default: "false" + tags: + description: | # TODO fill this in and update README + tagsum ipsum + required: true runs: using: composite @@ -99,12 +103,18 @@ runs: if: ${{ steps.prepare-vars.outputs.include-dockerhub == 'true' }} uses: ./_shared-workflows-docker-import-digests-push-manifest/actions/dockerhub-login + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: ${{ inputs.images }} + tags: ${{ inputs.tags }} + - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests shell: bash env: IMAGES: ${{ inputs.images }} - DOCKER_METADATA_OUTPUT_JSON: ${{ inputs.docker-metadata-json }} PUSH: ${{ inputs.push }} run: | set -euo pipefail @@ -136,8 +146,6 @@ runs: - name: Inspect image shell: bash if: ${{ inputs.push == 'true' }} - env: - DOCKER_METADATA_OUTPUT_JSON: ${{ inputs.docker-metadata-json }} run: | for tag in $(jq -r '.tags[]' <<< "$DOCKER_METADATA_OUTPUT_JSON"); do echo "" From acd325c96447fc80edddb05a666833e9c875b880 Mon Sep 17 00:00:00 2001 From: Ricky Whitaker Date: Wed, 8 Oct 2025 16:04:19 -0500 Subject: [PATCH 28/28] update README for docker-import-digests-push-manifest --- actions/docker-import-digests-push-manifest/README.md | 1 + .../docker-import-digests-push-manifest/action.yaml | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/actions/docker-import-digests-push-manifest/README.md b/actions/docker-import-digests-push-manifest/README.md index 54e64edc0..196e6c5ff 100644 --- a/actions/docker-import-digests-push-manifest/README.md +++ b/actions/docker-import-digests-push-manifest/README.md @@ -46,3 +46,4 @@ jobs: | `gar-environment` | String | Environment for pushing artifacts (can be either dev or prod). This sets the GAR Project to either `grafanalabs-dev` or `grafanalabs-global`. | | `images` | String | CSV of Docker images to push. These images should not include tags. Ex: us-docker.pkg.dev/grafanalabs-dev/gar-registry/image-name,docker.io/grafana/dockerhub-image | | `push` | Boolean | Whether to push the manifest to the configured registries. | +| `tags` | String | List of Docker tags to be pushed. | diff --git a/actions/docker-import-digests-push-manifest/action.yaml b/actions/docker-import-digests-push-manifest/action.yaml index d540362e0..b441ca52b 100644 --- a/actions/docker-import-digests-push-manifest/action.yaml +++ b/actions/docker-import-digests-push-manifest/action.yaml @@ -21,8 +21,8 @@ inputs: Whether to push the manifest to the configured registries. default: "false" tags: - description: | # TODO fill this in and update README - tagsum ipsum + description: | + List of Docker tags to be pushed. required: true runs: @@ -59,6 +59,12 @@ runs: IMAGES: ${{ inputs.images }} shell: bash run: | + ################################################################## + # This step parses the input image list to determine if + # docker images or gar images have been passed in... so we + # can determine which systems to login to. + ################################################################## + set -euo pipefail DOCKERHUB_IMAGE=false