master-build.yml

# Signs windows and macOS binaries and installers
name: Master branch build/release (signed)
defaults:
  run:
    shell: bash
on:
  push:
    branches: [ master, main ]
  release:
    types: [ created ]
  workflow_dispatch:
    inputs:
      debug_enabled:
        description: 'Run the build with tmate set "debug_enabled"'
        type: boolean
        required: false
        default: false

# We do not cancel in-progress builds here because in general we
# want to see success on each commit, and we usually use [skip ci]
# to prevent builds on commits that don't matter.
#concurrency:
#  group: ${{ github.workflow }}-${{ github.ref }}
#  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}

env:
  AmplitudeAPIKey: ${{ secrets.AMPLITUDE_API_KEY }}
  BUILDKIT_PROGRESS: plain
  DOCKER_CLI_EXPERIMENTAL: enabled
  DDEV_DEBUG: true
  FURY_ACCOUNT: ${{ vars.FURY_ACCOUNT }}
  HOMEBREW_EDGE_REPOSITORY: ${{ vars.HOMEBREW_EDGE_REPOSITORY }}
  HOMEBREW_STABLE_REPOSITORY: ${{ vars.HOMEBREW_STABLE_REPOSITORY }}
  REPOSITORY_OWNER: ${{ github.repository_owner }}
  DOCKER_ORG: ${{ vars.DOCKER_ORG }}

permissions:
  packages: write

jobs:
  build-most:
    name: Build DDEV executables except Windows
    runs-on: ubuntu-latest
    steps:

      - uses: actions/checkout@v4
        with:
          # We need to get all branches and tags for git describe to work properly
          fetch-depth: 0

      - name: Set up Homebrew
        id: set-up-homebrew
        uses: Homebrew/actions/setup-homebrew@master

      - name: Install Docker and deps
        run: ./.github/workflows/linux-setup.sh

      - name: Install build tools
        run: ./.github/workflows/linux-build-setup.sh

      - uses: actions/setup-go@v5
        with:
          go-version: '>=1.21'

      - name: Build DDEV executables
        run: make linux_amd64 linux_arm64 darwin_amd64 darwin_arm64 completions mkcert

      - name: "Verify that instrumentation is working (Linux amd64)"
        run: |
          export PATH=".gotmp/bin/linux_amd64:$PATH"
          echo "DDEV_NO_INSTRUMENTATION=${DDEV_NO_INSTRUMENTATION}"
          if [ -z "${AmplitudeAPIKey}" ]; then echo "AmplitudeAPIKey is not set"; exit 1; fi
          ddev config global --instrumentation-opt-in=true
          ddev config global | grep instrumentation-opt-in=true
          ddev version | grep -v "AmplitudeAPIKey is not available."
          ddev config global --instrumentation-opt-in=false


      - name: save build results to cache
        uses: actions/cache@v4
        with:
          path: .gotmp/bin
          key: ${{ github.sha }}-${{ github.ref }}-build-most

  # This Windows self-hosted runner has to be set up with gnu tar and zstd.exe, or
  # this step will fail to properly create the cache.
  # Make sure gnu tar is the tar used here. System PATH should have C:\program files\gnu\usr\bin near top
  # Get zstd.exe from https://github.com/facebook/zstd/releases - I put it into C:\program files\gnu\usr\bin
  # so it would be in PATH
  # See https://github.com/actions/cache/issues/580
  # Run the Windows action with debug enabled to be able to see which tar is in use, etc.
  sign-windows:
    name: Build and Sign Windows binaries
    runs-on: [ self-hosted, windows-signer ]
    env:
      DDEV_WINDOWS_SIGN: ${{ vars.DDEV_WINDOWS_SIGN }}
    steps:

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Build and sign windows amd64/arm64 binaries and installers
        shell: bash
        run: |
          if [ "${DDEV_WINDOWS_SIGN}" != "true" ]; then echo "Warning: DDEV_WINDOWS_SIGN=${DDEV_WINDOWS_SIGN} is not true"; fi
          make windows_amd64_install windows_arm64_install
      - name: Show github.ref
        run: echo ${{ github.ref }}
      - name: Build chocolatey on release
        if: startsWith( github.ref, 'refs/tags/v1')
        run: make chocolatey

      - name: "Verify that instrumentation is working (Windows)"
        run: |
          export PATH=".gotmp/bin/windows_amd64:$PATH"
          echo "DDEV_NO_INSTRUMENTATION=${DDEV_NO_INSTRUMENTATION}"
          if [ -z "${AmplitudeAPIKey}" ]; then echo "AmplitudeAPIKey is not set"; exit 1; fi
          ddev config global --instrumentation-opt-in=true
          ddev config global | grep instrumentation-opt-in=true
          ddev version | grep -v "AmplitudeAPIKey is not available."
          ddev config global --instrumentation-opt-in=false

      - name: Verify that binaries and installers were signed
        run: |
          signtool verify -pa .gotmp/bin/windows_amd64/ddev_windows_amd64_installer.exe
          signtool verify -pa .gotmp/bin/windows_amd64/ddev.exe
          signtool verify -pa .gotmp/bin/windows_arm64/ddev_windows_arm64_installer.exe
          signtool verify -pa .gotmp/bin/windows_arm64/ddev.exe

      - name: Cache signed binaries
        # After 3.0.5 they were using zstdmt which is not available on Windows
        # See https://github.com/actions/cache/issues/891
        # But trying again with @v3, 2023-02-18
        uses: actions/cache@v4
        with:
          path: .gotmp/bin/windows*
          key: ${{ github.sha }}-${{ github.ref }}-signed-windows-binaries
          enableCrossOsArchive: true

  notarize-macos:
    name: Sign and Notarize ddev on macOS
    runs-on: macos-latest
    needs: build-most
    steps:
      - name: Load 1password secret(s)
        uses: 1password/load-secrets-action@v2
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}"
          DDEV_MACOS_APP_PASSWORD: "op://push-secrets/DDEV_MACOS_APP_PASSWORD/credential"
          DDEV_MACOS_SIGNING_PASSWORD: "op://push-secrets/DDEV_MACOS_SIGNING_PASSWORD/credential"

      - name: "setup macOS"
        run: |
          brew install coreutils gnu-getopt jq
      - uses: actions/checkout@v4
        with:
          # We need to get all branches and tags for git describe to work properly
          fetch-depth: 0
      - uses: actions/setup-go@v5
        with:
          go-version: '>=1.21'

      - name: restore build-most results from cache
        uses: actions/cache@v4
        id: buildmost
        with:
          path: .gotmp/bin
          key: ${{ github.sha }}-${{ github.ref }}-build-most
          fail-on-cache-miss: true
      - name: test that buildmost cache was loaded
        if: steps.buildmost.outputs.cache-hit != 'true'
        run: exit 1

      - name: Sign and notarize binaries (amd64 and arm64)
        env:
          TEAM_ID: "9HQ298V2BW"
        run: |
          set -o errexit -o pipefail
          if [ -z "${DDEV_MACOS_SIGNING_PASSWORD}" ] ; then echo "DDEV_MACOS_SIGNING_PASSWORD is empty"; exit 1; fi
          if [ -z "${DDEV_MACOS_APP_PASSWORD}" ] ; then echo "DDEV_MACOS_APP_PASSWORD is empty"; exit 1; fi
          for item in .gotmp/bin/darwin*/ddev; do
            echo "Signing and notarizing ${item} ..."
            codesign --remove-signature "${item}" || true
            curl -s https://raw.githubusercontent.com/ddev/signing_tools/master/macos_sign.sh | bash -s -  --signing-password="${DDEV_MACOS_SIGNING_PASSWORD}" --cert-file=certfiles/ddev_developer_id_cert.p12 --cert-name="Developer ID Application: Localdev Foundation (9HQ298V2BW)" --target-binary="${item}"
            curl -sSL -f https://raw.githubusercontent.com/ddev/signing_tools/master/macos_notarize.sh | bash -s -  --app-specific-password=${DDEV_MACOS_APP_PASSWORD} --apple-id=notarizer@localdev.foundation --primary-bundle-id=com.ddev.ddev --target-binary="${item}"
          done
      - name: Save notarized binaries to cache
        uses: actions/cache@v4
        with:
          path: .gotmp/bin/darwin*
          key: ${{ github.sha }}-${{ github.ref }}-notarize-macos

  artifacts:
    env:
      AUR_PACKAGE_NAME: ${{ vars.AUR_PACKAGE_NAME }}
      AUR_EDGE_GIT_URL: ${{ vars.AUR_EDGE_GIT_URL }}
      AUR_STABLE_GIT_URL: ${{ vars.AUR_STABLE_GIT_URL }}
      DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }}
    name: Upload artifacts
    runs-on: ubuntu-latest
    needs: [build-most, sign-windows, notarize-macos]
    steps:
      - name: Load 1password secret(s)
        uses: 1password/load-secrets-action@v2
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}"
          AUR_SSH_PRIVATE_KEY: "op://push-secrets/AUR_SSH_PRIVATE_KEY/private key?ssh-format=openssh"
          CHOCOLATEY_API_KEY: "op://push-secrets/CHOCOLATEY_API_KEY/credential"
          GITHUB_TOKEN: "op://push-secrets/DDEV_GITHUB_TOKEN/credential"
          DDEV_MACOS_APP_PASSWORD: "op://push-secrets/DDEV_MACOS_APP_PASSWORD/credential"
          DDEV_MACOS_SIGNING_PASSWORD: "op://push-secrets/DDEV_MACOS_SIGNING_PASSWORD/credential"
          DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential"
          FURY_TOKEN: "op://push-secrets/FURY_TOKEN/credential_${{ github.repository_owner }}"
          GORELEASER_KEY: "op://push-secrets/GORELEASER_KEY/credential"

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Restore build-most builds
        uses: actions/cache@v4
        id: buildmost
        with:
          path: .gotmp/bin
          key: ${{ github.sha }}-${{ github.ref }}-build-most
          enableCrossOsArchive: true
          fail-on-cache-miss: true
      - name: test that build-most was loaded
        if: steps.buildmost.outputs.cache-hit != 'true'
        run: exit 1

      - name: Restore Signed Windows artifacts
        uses: actions/cache@v4
        id: signedwindows
        with:
          path: .gotmp/bin/windows*
          key: ${{ github.sha }}-${{ github.ref }}-signed-windows-binaries
          enableCrossOsArchive: true
          fail-on-cache-miss: true
      - name: test that signed-windows was loaded
        if: steps.signedwindows.outputs.cache-hit != 'true'
        run: exit 1

      - name: Restore Signed Mac artifacts
        uses: actions/cache@v4
        id: notarizedmac
        with:
          path: .gotmp/bin/darwin*
          key: ${{ github.sha }}-${{ github.ref }}-notarize-macos
          enableCrossOsArchive: true
          fail-on-cache-miss: true
      - name: test that notarizedmac was loaded
        if: steps.notarizedmac.outputs.cache-hit != 'true'
        run: exit 1

      - name: "Verify that cgo_enabled is 0 in Linux binary"
        run: |
          .gotmp/bin/$(go env GOOS)_$(go env GOARCH)/ddev version
          cgo=$(".gotmp/bin/$(go env GOOS)_$(go env GOARCH)/ddev" version 2>/dev/null | awk '/cgo_enabled/ {print $2}')
          if [ "${cgo}" != "0" ]; then echo "CGO_ENABLED=${cgo} but it must be 0 in released binary" && exit 10; fi

      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        with:
          limit-access-to-actor: true
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}

      # Goreleaser does GitHub release artifacts, homebrew, AUR, deb/rpm
      - name: goreleaser
        uses: goreleaser/goreleaser-action@v6
        if: startsWith( github.ref, 'refs/tags/v1')
        with:
          distribution: goreleaser-pro
          version: "~> v2"
          args: release --clean
        env:
          CGO_ENABLED: 0

      # Do artifacts for upload to workflow URL
      - name: Generate artifacts
        run: ./.ci-scripts/generate_artifacts.sh ${{ github.workspace }}/artifacts

      - name: Upload all artifacts
        uses: actions/upload-artifact@v4
        with:
          name: all-ddev-executables
          path: ${{ github.workspace }}/artifacts/*
      - name: Upload macos-amd64 binary
        uses: actions/upload-artifact@v4
        with:
          name: ddev-macos-amd64
          path: .gotmp/bin/darwin_amd64/ddev
      - name: Upload macos-arm64 binary
        uses: actions/upload-artifact@v4
        with:
          name: ddev-macos-arm64
          path: .gotmp/bin/darwin_arm64/ddev
      - name: Upload linux-arm64 binary
        uses: actions/upload-artifact@v4
        with:
          name: ddev-linux-arm64
          path: .gotmp/bin/linux_arm64/ddev
      - name: Upload linux-amd64 binary
        uses: actions/upload-artifact@v4
        with:
          name: ddev-linux-amd64
          path: .gotmp/bin/linux_amd64/ddev
      - name: Upload windows-amd64 binary
        uses: actions/upload-artifact@v4
        with:
          name: ddev-windows-amd64
          path: .gotmp/bin/windows_amd64/ddev.exe

      - name: Upload windows_amd64 installer
        uses: actions/upload-artifact@v4
        with:
          name: ddev-windows-amd64-installer
          path: .gotmp/bin/windows_amd64/ddev_windows_amd64_installer*.exe
      - name: Upload windows-arm64 binary
        uses: actions/upload-artifact@v4
        with:
          name: ddev-windows-arm64
          path: .gotmp/bin/windows_arm64/ddev.exe
      - name: Upload windows_arm64 installer
        uses: actions/upload-artifact@v4
        with:
          name: ddev-windows-arm64-installer
          path: .gotmp/bin/windows_arm64/ddev_windows_arm64_installer*.exe

      - name: Show github.ref
        run: echo ${{ github.ref }}

      # TODO: This can be done by goreleaser these days
      - name: Chocolatey windows release
        if: github.repository_owner == 'ddev' && startsWith(github.ref, 'refs/tags/v1')
        run: |
          pushd .gotmp/bin/windows_amd64/chocolatey
          docker run --rm -v $PWD:/tmp/chocolatey -w /tmp/chocolatey linuturk/mono-choco push -s https://push.chocolatey.org/ --api-key "${CHOCOLATEY_API_KEY}"
          popd

  push-devcontainers:
    name: Push devcontainer feature
    runs-on: ubuntu-latest
    needs: artifacts
    steps:
      - name: Load 1password secret(s)
        uses: 1password/load-secrets-action@v2
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}"
          GITHUB_TOKEN: "op://push-secrets/DDEV_GITHUB_TOKEN/credential"

      - uses: actions/checkout@v4
        with:
          # We need to get all branches and tags for git describe to work properly
          fetch-depth: 0

      - name: Replace version in devcontainer-feature.json
        run: |
          sed -i 's/\${DDEV_VERSION}/'"${GITHUB_REF##*/}"'/g' ./containers/devcontainers/install-ddev/devcontainer-feature.json

      - name: Push DDEV devcontainer feature
        if: startsWith(github.ref, 'refs/tags/v1')
        uses: devcontainers/action@v1
        with:
          publish-features: "true"
          base-path-to-features: "./containers/devcontainers"
          devcontainer-cli-version: "0.53.0"
          disable-repo-tagging: "true"

  push-gitpod-base:
    name: Push ddev-gitpod-base docker image
    runs-on: ubuntu-latest
    needs: artifacts

    steps:
      - name: Load 1password secret(s)
        uses: 1password/load-secrets-action@v2
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}"
          DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential"
      - uses: actions/checkout@v4
        with:
          # We need to get all branches and tags for git describe to work properly
          fetch-depth: 0
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ vars.DOCKERHUB_USERNAME }}
          password: ${{ env.DOCKERHUB_TOKEN }}
      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        with:
          limit-access-to-actor: true
          github-token: ${{ secrets.GITHUB_TOKEN }}
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}

      - name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ vars.DOCKERHUB_USERNAME }}
          password: ${{ env.DOCKERHUB_TOKEN }}

      - name: Push ddev-gitpod-base image
        if: startsWith(github.ref, 'refs/tags/v1')
        run: |
          pushd containers/ddev-gitpod-base >/dev/null
          # For testing, push to ddevhq docker org
          set -x
          make push VERSION="${GITHUB_REF##*/}" DOCKER_ORG=${DOCKER_ORG}
          popd >/dev/null