ci.yml

name: CI

on:
  pull_request:
    branches:
      - main
      - release-*
  merge_group:
  workflow_dispatch:
    inputs:
      force:
        type: boolean
        description: "Force rebuild"
        required: false
        default: false

defaults:
  run:
    shell: bash

concurrency:
  group: ${{ github.ref_name }}-ci
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  checks:
    name: Checks and variables
    runs-on: ubuntu-22.04
    permissions:
      contents: read
      id-token: write
    outputs:
      docs_only: ${{ github.event.pull_request && steps.docs.outputs.docs_only == 'true' }}
      k8s_latest: ${{ steps.vars.outputs.k8s_latest }}
      go_path: ${{ steps.vars.outputs.go_path }}
      go_code_md5: ${{ steps.vars.outputs.go_code_md5 }}
      binary_cache_hit: ${{ steps.binary-cache.outputs.cache-hit }}
      chart_version: ${{ steps.vars.outputs.chart_version }}
      ic_version: ${{ steps.vars.outputs.ic_version }}
      docker_md5: ${{ steps.vars.outputs.docker_md5 }}
      build_tag: ${{ steps.vars.outputs.build_tag }}
      stable_tag: ${{ steps.vars.outputs.stable_tag }}
      forked_workflow: ${{ steps.vars.outputs.forked_workflow }}
      stable_image_exists: ${{ steps.stable_exists.outputs.exists }}
      additional_tag: ${{ steps.vars.outputs.additional_tag }}
      image_matrix_oss: ${{ steps.vars.outputs.image_matrix_oss }}
      image_matrix_plus: ${{ steps.vars.outputs.image_matrix_plus }}
      image_matrix_nap: ${{ steps.vars.outputs.image_matrix_nap }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          fetch-depth: 0

      - name: Filter only docs changes
        id: docs
        run: |
          files=$(git diff --name-only HEAD^ | egrep -v "^docs/" | egrep -v "^examples/" | egrep -v "^README.md")
          if [ -z "$files" ]; then
            echo "docs_only=true" >> $GITHUB_OUTPUT
          else
            echo "docs_only=false" >> $GITHUB_OUTPUT
          fi
          echo $files
          cat $GITHUB_OUTPUT
        shell: bash --noprofile --norc -o pipefail {0}

      - name: Setup Golang Environment
        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
        with:
          go-version-file: go.mod

      - name: Output Variables
        id: vars
        run: |
          kindest_latest=$(curl -s "https://hub.docker.com/v2/repositories/kindest/node/tags" \
            | grep -o '"name": *"[^"]*' \
            | grep -o '[^"]*$' \
            | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \
            | sort -rV \
            | head -n 1 \
            | sed 's/^.\{1\}//' \
            | tr -d '\n')
          echo "k8s_latest=$kindest_latest" >> $GITHUB_OUTPUT
          echo "go_path=$(go env GOPATH)" >> $GITHUB_OUTPUT
          source .github/data/version.txt
          echo "ic_version=${IC_VERSION}" >> $GITHUB_OUTPUT
          echo "chart_version=${HELM_CHART_VERSION}" >> $GITHUB_OUTPUT
          echo "forked_workflow=${{ (github.event.pull_request && github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) || github.repository != 'nginxinc/kubernetes-ingress' }}" >> $GITHUB_OUTPUT
          ./.github/scripts/variables.sh go_code_md5 >> $GITHUB_OUTPUT
          ./.github/scripts/variables.sh docker_md5 >> $GITHUB_OUTPUT
          ./.github/scripts/variables.sh build_tag >> $GITHUB_OUTPUT
          ./.github/scripts/variables.sh stable_tag >> $GITHUB_OUTPUT
          ref=${{ github.ref_name }}
          if [[ $ref =~ merge ]]; then
            additional_tag="pr-${ref%*/merge}"
          else
            additional_tag="${ref//\//-}"
          fi
          echo "additional_tag=${additional_tag}" >> $GITHUB_OUTPUT
          echo "image_matrix_oss=$(cat .github/data/matrix-images-oss.json | jq -c)" >> $GITHUB_OUTPUT
          echo "image_matrix_plus=$(cat .github/data/matrix-images-plus.json | jq -c)" >> $GITHUB_OUTPUT
          echo "image_matrix_nap=$(cat .github/data/matrix-images-nap.json | jq -c)" >> $GITHUB_OUTPUT
          cat $GITHUB_OUTPUT

      - name: Fetch Cached Binary Artifacts
        id: binary-cache
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: ${{ github.workspace }}/dist
          key: nginx-ingress-${{ steps.vars.outputs.go_code_md5 }}
          lookup-only: true

      - name: Authenticate to Google Cloud
        id: auth
        uses: google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa # v2.1.3
        with:
          token_format: access_token
          workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
          service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}
        if: ${{ steps.vars.outputs.forked_workflow == 'false' }}

      - name: Login to GCR
        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
        with:
          registry: gcr.io
          username: oauth2accesstoken
          password: ${{ steps.auth.outputs.access_token }}
        if: ${{ steps.vars.outputs.forked_workflow == 'false' }}

      - name: Check if stable image exists
        id: stable_exists
        run: |
          if docker pull gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress:${{ steps.vars.outputs.stable_tag }}; then
            echo "exists=true" >> $GITHUB_OUTPUT
          fi
        if: ${{ steps.vars.outputs.forked_workflow == 'false' }}

      - name: Output variables
        run: |
          echo docs_only: ${{ github.event.pull_request && steps.docs.outputs.docs_only == 'true' }}
          echo k8s_latest: ${{ steps.vars.outputs.k8s_latest }}
          echo go_path: ${{ steps.vars.outputs.go_path }}
          echo go_code_md5: ${{ steps.vars.outputs.go_code_md5 }}
          echo binary_cache_hit: ${{ steps.binary-cache.outputs.cache-hit }}
          echo chart_version: ${{ steps.vars.outputs.chart_version }}
          echo ic_version: ${{ steps.vars.outputs.ic_version }}
          echo docker_md5: ${{ steps.vars.outputs.docker_md5 }}
          echo build_tag: ${{ steps.vars.outputs.build_tag }}
          echo stable_tag: ${{ steps.vars.outputs.stable_tag }}
          echo forked_workflow: ${{ steps.vars.outputs.forked_workflow }}
          echo stable_image_exists: ${{ steps.stable_exists.outputs.exists }}
          echo additional_tag: ${{ steps.vars.outputs.additional_tag }}
          echo 'image_matrix_oss: ${{ steps.vars.outputs.image_matrix_oss }}'
          echo 'image_matrix_plus: ${{ steps.vars.outputs.image_matrix_plus }}'
          echo 'image_matrix_nap: ${{ steps.vars.outputs.image_matrix_nap }}'

  verify-codegen:
    name: Verify generated code
    runs-on: ubuntu-22.04
    permissions:
      contents: read
    steps:
      - name: Checkout Repository
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

      - name: Check if go.mod and go.sum are up to date
        run: go mod tidy && git diff --exit-code -- go.mod go.sum

      - name: Check if CRDs changed
        run: make update-crds && git diff --name-only --exit-code config/crd/bases

      - name: Check if Codegen changed
        run: |
          cd ../.. && mkdir -p github.com/nginxinc && mv kubernetes-ingress/kubernetes-ingress github.com/nginxinc/ && cd github.com/nginxinc/kubernetes-ingress
          make update-codegen && git diff --name-only --exit-code pkg/**
          cd ../../.. && mv github.com/nginxinc/kubernetes-ingress kubernetes-ingress/kubernetes-ingress

  unit-tests:
    name: Unit Tests
    runs-on: ubuntu-22.04
    needs: checks
    steps:
      - name: Checkout Repository
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

      - name: Setup Golang Environment
        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
        with:
          go-version-file: go.mod
        if: ${{ needs.checks.outputs.binary_cache_hit != 'true' }}

      - name: Run Tests
        run: make cover
        if: ${{ needs.checks.outputs.binary_cache_hit != 'true' }}

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0
        with:
          files: ./coverage.txt
          token: ${{ secrets.CODECOV_TOKEN }} # required
        if: ${{ needs.checks.outputs.binary_cache_hit != 'true' }}

  release-notes:
    name: Release Notes
    runs-on: ubuntu-22.04
    needs: [checks, unit-tests, verify-codegen]
    outputs:
      release-url: ${{ steps.release-notes.outputs.release-url }}
    permissions:
      contents: write # for lucacome/draft-release
    steps:
      - name: Checkout Repository
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          fetch-depth: 0

      - name: Create/Update Draft
        uses: lucacome/draft-release@8a63d32c79a171ae6048e614a8988f0ac3ed56d4 # v1.1.0
        id: release-notes
        with:
          minor-label: "enhancement"
          major-label: "change"
          publish: ${{ github.ref_type == 'tag' && vars.OLD_RELEASE_FLOW == 'true' }}
          collapse-after: 50
          variables: |
            helm-chart=${{ needs.checks.outputs.chart_version }}
          notes-footer: |
            ## Upgrade
            - For NGINX, use the {{version}} images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name={{version-number}}), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress).
            - For NGINX Plus, use the {{version}} images from the F5 Container registry, the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE), the [GCP Marketplace](https://console.cloud.google.com/marketplace/browse?filter=partner:F5,%20Inc.&filter=solution-type:k8s&filter=category:networking) or build your own image using the {{version}} source code.
            - For Helm, use version {{helm-chart}} of the chart.

            ## Resources
            - Documentation -- https://docs.nginx.com/nginx-ingress-controller/
            - Configuration examples -- https://github.com/nginxinc/kubernetes-ingress/tree/{{version}}/examples
            - Helm Chart -- https://github.com/nginxinc/kubernetes-ingress/tree/{{version}}/charts/nginx-ingress
            - Operator -- https://github.com/nginxinc/nginx-ingress-helm-operator
        if: ${{ github.event_name == 'push' && github.ref != 'refs/heads/main' }}

  binaries:
    name: Build Binaries
    runs-on: ubuntu-22.04
    needs: [checks, unit-tests, verify-codegen]
    permissions:
      contents: write # for goreleaser/goreleaser-action to manage releases
      id-token: write # for goreleaser/goreleaser-action to sign artifacts
      issues: write # for goreleaser/goreleaser-action to close milestone
    steps:
      - name: Checkout Repository
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          fetch-depth: 0

      - name: Setup Golang Environment
        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
        with:
          go-version-file: go.mod
        if: ${{ needs.checks.outputs.binary_cache_hit != 'true' }}

      - name: Build binaries
        uses: goreleaser/goreleaser-action@286f3b13b1b49da4ac219696163fb8c1c93e1200 # v6.0.0
        with:
          version: latest
          args: build --snapshot --clean
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GOPATH: ${{ needs.checks.outputs.go_path }}
          AWS_PRODUCT_CODE: ${{ secrets.AWS_PRODUCT_CODE }}
          AWS_PUB_KEY: ${{ secrets.AWS_PUB_KEY }}
          AWS_NAP_DOS_PRODUCT_CODE: ${{ secrets.AWS_NAP_DOS_PRODUCT_CODE }}
          AWS_NAP_DOS_PUB_KEY: ${{ secrets.AWS_NAP_DOS_PUB_KEY }}
          AWS_NAP_WAF_PRODUCT_CODE: ${{ secrets.AWS_NAP_WAF_PRODUCT_CODE }}
          AWS_NAP_WAF_PUB_KEY: ${{ secrets.AWS_NAP_WAF_PUB_KEY }}
          AWS_NAP_WAF_DOS_PRODUCT_CODE: ${{ secrets.AWS_NAP_WAF_DOS_PRODUCT_CODE }}
          AWS_NAP_WAF_DOS_PUB_KEY: ${{ secrets.AWS_NAP_WAF_DOS_PUB_KEY }}
          GORELEASER_CURRENT_TAG: "v${{ needs.checks.outputs.ic_version }}"
        if: ${{ needs.checks.outputs.binary_cache_hit != 'true' }}

      - name: Store Artifacts in Cache
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: ${{ github.workspace }}/dist
          key: nginx-ingress-${{ needs.checks.outputs.go_code_md5 }}
        if: ${{ needs.checks.outputs.binary_cache_hit != 'true' }}

  build-docker:
    name: Build Docker OSS
    needs: [binaries, checks]
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_oss ) }}
    uses: ./.github/workflows/build-oss.yml
    with:
      platforms: ${{ matrix.platforms }}
      image: ${{ matrix.image }}
      go-md5: ${{ needs.checks.outputs.go_code_md5 }}
      base-image-md5: ${{ needs.checks.outputs.docker_md5 }}
      authenticated: ${{ needs.checks.outputs.forked_workflow != 'true' }}
      full-build: ${{ inputs.force && inputs.force || false }}
      tag: ${{ needs.checks.outputs.build_tag }}
      branch: ${{ (github.head_ref && needs.checks.outputs.forked_workflow != 'true') && github.head_ref || github.ref }}
    permissions:
      contents: read
      actions: read
      security-events: write
      id-token: write
      packages: write
      pull-requests: write # for scout report
    secrets: inherit

  build-docker-plus:
    name: Build Docker Plus
    needs: [binaries, checks]
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_plus ) }}
    uses: ./.github/workflows/build-plus.yml
    with:
      platforms: ${{ matrix.platforms }}
      image: ${{ matrix.image }}
      target: ${{ matrix.target }}
      go-md5: ${{ needs.checks.outputs.go_code_md5 }}
      base-image-md5: ${{ needs.checks.outputs.docker_md5 }}
      branch: ${{ (github.head_ref && needs.checks.outputs.forked_workflow != 'true') && github.head_ref || github.ref }}
      tag: ${{ needs.checks.outputs.build_tag }}
      authenticated: ${{ needs.checks.outputs.forked_workflow != 'true' }}
      full-build: ${{ inputs.force && inputs.force || false }}
    permissions:
      contents: read
      security-events: write
      id-token: write
      pull-requests: write # for scout report
    secrets: inherit

  build-docker-nap:
    name: Build Docker NAP
    needs: [binaries, checks]
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON( needs.checks.outputs.image_matrix_nap ) }}
    uses: ./.github/workflows/build-plus.yml
    with:
      platforms: ${{ matrix.platforms }}
      image: ${{ matrix.image }}
      target: ${{ matrix.target }}
      go-md5: ${{ needs.checks.outputs.go_code_md5 }}
      base-image-md5: ${{ needs.checks.outputs.docker_md5 }}
      branch: ${{ (github.head_ref && needs.checks.outputs.forked_workflow != 'true') && github.head_ref || github.ref }}
      tag: ${{ needs.checks.outputs.build_tag }}
      nap-modules: ${{ matrix.nap_modules }}
      authenticated: ${{ needs.checks.outputs.forked_workflow != 'true' }}
      full-build: ${{ inputs.force && inputs.force || false }}
    permissions:
      contents: read
      security-events: write
      id-token: write
      pull-requests: write # for scout report
    secrets: inherit

  tag-target:
    name: Tag untested image with PR number
    needs: [checks, build-docker, build-docker-plus, build-docker-nap]
    permissions:
      contents: read # To checkout repository
      id-token: write # To sign into Google Container Registry
    uses: ./.github/workflows/retag-images.yml
    with:
      source_tag: ${{ needs.checks.outputs.build_tag }}
      target_tag: ${{ needs.checks.outputs.additional_tag }}
      dry_run: false
    secrets: inherit
    if: ${{ inputs.force || (needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.stable_image_exists != 'true' && needs.checks.outputs.docs_only == 'false') }}

  helm-tests:
    if: ${{ needs.checks.outputs.docs_only != 'true' }}
    name: Helm Tests ${{ matrix.base-os }}
    runs-on: ubuntu-22.04
    needs: [checks, binaries, build-docker, build-docker-plus]
    strategy:
      fail-fast: false
      matrix:
        include:
          - base-os: debian
            image: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress
            tag: ${{ needs.checks.outputs.build_tag }}
            type: oss
          - base-os: debian-plus
            image: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress
            tag: ${{ needs.checks.outputs.build_tag }}
            type: plus
    permissions:
      contents: read
      id-token: write
    steps:
      - name: Checkout Repository
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

      - name: Authenticate to Google Cloud
        id: auth
        uses: google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa # v2.1.3
        with:
          token_format: access_token
          workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
          service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}
        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}

      - name: Login to GCR
        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
        with:
          registry: gcr.io
          username: oauth2accesstoken
          password: ${{ steps.auth.outputs.access_token }}
        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}

      - name: Check if stable image exists
        id: stable_exists
        run: |
          if docker pull ${{ matrix.image }}:${{ needs.checks.outputs.stable_tag }}; then
            echo "exists=true" >> $GITHUB_OUTPUT
          fi
        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}

      - name: Pull build image
        run: |
          docker pull ${{ matrix.image }}:${{ needs.checks.outputs.build_tag }}
        if: ${{ ( needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' ) && steps.stable_exists.outputs.exists != 'true' }}

      - name: Fetch Cached Artifacts
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: ${{ github.workspace }}/dist
          key: nginx-ingress-${{ needs.checks.outputs.go_code_md5 }}
        if: ${{ needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false' }}

      - name: Docker Buildx
        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
        if: ${{ needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false' }}

      - name: Build Docker Image ${{ matrix.base-os }}
        uses: docker/build-push-action@31159d49c0d4756269a0940a750801a1ea5d7003 # v6.1.0
        with:
          file: build/Dockerfile
          context: "."
          cache-from: type=gha,scope=${{ matrix.base-os }}
          target: goreleaser
          tags: "${{ matrix.image }}:${{ matrix.tag }}"
          pull: true
          load: true
          build-args: |
            BUILD_OS=${{ matrix.base-os }}
            IC_VERSION=CI
          secrets: |
            ${{ matrix.type == 'plus' && format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) || '' }}
            ${{ matrix.type == 'plus' && format('"nginx-repo.key={0}"', secrets.NGINX_KEY) || '' }}
        if: ${{ needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false' }}

      - name: Deploy Kubernetes
        id: k8s
        run: |
          kind create cluster --name ${{ github.run_id }} --image=kindest/node:v${{ needs.checks.outputs.k8s_latest }} --wait 75s
          kind load docker-image "${{ matrix.image }}:${{ matrix.tag }}" --name ${{ github.run_id }}
        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}

      - name: Install Chart
        run: >
          helm install
          ${{ matrix.type }}
          .
          --set controller.image.repository=${{ matrix.image }}
          --set controller.image.tag=${{ matrix.tag }}
          --set controller.service.type=NodePort
          --set controller.nginxplus=${{ contains(matrix.type, 'plus') && 'true' || 'false' }}
          --set controller.telemetryReporting.enable=false
          --wait
        working-directory: ${{ github.workspace }}/charts/nginx-ingress
        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}

      - name: Expose Test Ingresses
        run: |
          kubectl port-forward service/${{ matrix.type }}-nginx-ingress-controller 8080:80 8443:443 &
        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}

      - name: Test HTTP
        run: |
          counter=0
          max_attempts=5
          until [ $(curl --write-out %{http_code} -s --output /dev/null http://localhost:8080) -eq 404 ]; do
            if [ ${counter} -eq ${max_attempts} ]; then
              exit 1
            fi
            printf '.'; counter=$(($counter+1)); sleep 5;
          done
        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}

      - name: Test HTTPS
        run: |
          counter=0
          max_attempts=5
          until [ $(curl --write-out %{http_code} -ks --output /dev/null https://localhost:8443) -eq 000 ]; do
            if [ ${counter} -eq ${max_attempts} ]; then
              exit 1
            fi
            printf '.'; counter=$(($counter+1)); sleep 5;
          done
        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}

  setup-matrix:
    if: ${{ inputs.force || needs.checks.outputs.docs_only != 'true' }}
    name: Setup Matrix for Smoke Tests
    runs-on: ubuntu-22.04
    needs: [binaries, checks]
    permissions:
      contents: read
      id-token: write
    outputs:
      matrix_oss: ${{ steps.set-matrix.outputs.matrix_oss }}
      matrix_plus: ${{ steps.set-matrix.outputs.matrix_plus }}
      matrix_nap: ${{ steps.set-matrix.outputs.matrix_nap }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

      - id: set-matrix
        run: |
          echo "matrix_oss=$(cat .github/data/matrix-smoke-oss.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT
          echo "matrix_plus=$(cat .github/data/matrix-smoke-plus.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT
          echo "matrix_nap=$(cat .github/data/matrix-smoke-nap.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT

      - name: Docker Buildx
        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0

      - name: Authenticate to Google Cloud
        id: auth
        uses: google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa # v2.1.3
        with:
          token_format: access_token
          workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
          service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}
        if: ${{ needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.docs_only == 'false' }}

      - name: Login to GCR
        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
        with:
          registry: gcr.io
          username: oauth2accesstoken
          password: ${{ steps.auth.outputs.access_token }}
        if: ${{ needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.docs_only == 'false' }}

      - name: Check if test image exists
        id: check-image
        run: |
          docker pull gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}
        shell: bash
        continue-on-error: true
        if: ${{ needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.docs_only == 'false' }}

      - name: Build Test-Runner Container
        uses: docker/build-push-action@31159d49c0d4756269a0940a750801a1ea5d7003 # v6.1.0
        with:
          file: tests/Dockerfile
          context: "."
          cache-from: type=gha,scope=test-runner
          tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
          pull: true
          push: ${{ needs.checks.outputs.forked_workflow == 'false' }}
          load: false
        if: ${{ steps.check-image.outcome == 'failure' && needs.checks.outputs.docs_only == 'false' }}

  smoke-tests-oss:
    if: ${{ inputs.force || needs.checks.outputs.docs_only != 'true' }}
    name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} smoke tests
    needs:
      - checks
      - setup-matrix
      - build-docker
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.setup-matrix.outputs.matrix_oss) }}
    permissions:
      contents: read
      id-token: write
    uses: ./.github/workflows/setup-smoke.yml
    secrets: inherit
    with:
      image: ${{ matrix.images.image }}
      target: ${{ matrix.images.target }}
      nap-modules: ${{ matrix.images.nap_modules }}
      marker: ${{ matrix.images.marker }}
      label: ${{ matrix.images.label }}
      go-md5: ${{ needs.checks.outputs.go_code_md5 }}
      build-tag: ${{ needs.checks.outputs.build_tag }}
      stable-tag: ${{ needs.checks.outputs.stable_tag }}
      authenticated: ${{ needs.checks.outputs.forked_workflow != 'true' }}
      k8s-version: ${{ matrix.k8s }}

  smoke-tests-plus:
    if: ${{ inputs.force || needs.checks.outputs.docs_only != 'true' }}
    name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} smoke tests
    needs:
      - checks
      - setup-matrix
      - build-docker-plus
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.setup-matrix.outputs.matrix_plus) }}
    permissions:
      contents: read
      id-token: write
    uses: ./.github/workflows/setup-smoke.yml
    secrets: inherit
    with:
      image: ${{ matrix.images.image }}
      target: ${{ matrix.images.target }}
      nap-modules: ${{ matrix.images.nap_modules }}
      marker: ${{ matrix.images.marker }}
      label: ${{ matrix.images.label }}
      go-md5: ${{ needs.checks.outputs.go_code_md5 }}
      build-tag: ${{ needs.checks.outputs.build_tag }}
      stable-tag: ${{ needs.checks.outputs.stable_tag }}
      authenticated: ${{ needs.checks.outputs.forked_workflow != 'true' }}
      k8s-version: ${{ matrix.k8s }}

  smoke-tests-nap:
    if: ${{ inputs.force || needs.checks.outputs.docs_only != 'true' }}
    name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} smoke tests
    needs:
      - checks
      - setup-matrix
      - build-docker-nap
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.setup-matrix.outputs.matrix_nap) }}
    permissions:
      contents: read
      id-token: write
    uses: ./.github/workflows/setup-smoke.yml
    secrets: inherit
    with:
      image: ${{ matrix.images.image }}
      target: ${{ matrix.images.target }}
      nap-modules: ${{ matrix.images.nap_modules }}
      marker: ${{ matrix.images.marker }}
      label: ${{ matrix.images.label }}
      go-md5: ${{ needs.checks.outputs.go_code_md5 }}
      build-tag: ${{ needs.checks.outputs.build_tag }}
      stable-tag: ${{ needs.checks.outputs.stable_tag }}
      authenticated: ${{ needs.checks.outputs.forked_workflow != 'true' }}
      k8s-version: ${{ matrix.k8s }}

  tag-stable:
    name: Tag tested image as stable
    needs: [checks, smoke-tests-oss, smoke-tests-plus, smoke-tests-nap]
    permissions:
      contents: read # To checkout repository
      id-token: write # To sign into Google Container Registry
    uses: ./.github/workflows/retag-images.yml
    with:
      source_tag: ${{ needs.checks.outputs.build_tag }}
      target_tag: ${{ needs.checks.outputs.stable_tag }}
      dry_run: false
    secrets: inherit
    if: ${{ inputs.force || (needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.stable_image_exists != 'true' && needs.checks.outputs.docs_only == 'false') }}

  tag-results:
    if: ${{ always() }}
    runs-on: ubuntu-22.04
    name: Final CI Results
    needs: [tag-stable, smoke-tests-oss, smoke-tests-plus, smoke-tests-nap]
    steps:
      - run: |
          tagResult="${{ needs.tag-stable.result }}"
          smokeOSSResult="${{ needs.smoke-tests-oss.result }}"
          smokePlusResult="${{ needs.smoke-tests-plus.result }}"
          smokeNAPResult="${{ needs.smoke-tests-nap.result }}"
          if [[ $tagResult != "success" && $tagResult != "skipped" ]]; then
            exit 1
          fi
          if [[ $smokeOSSResult != "success" && $smokeOSSResult != "skipped" ]]; then
            exit 1
          fi
          if [[ $smokePlusResult != "success" && $smokePlusResult != "skipped" ]]; then
            exit 1
          fi
          if [[ $smokeNAPResult != "success" && $smokeNAPResult != "skipped" ]]; then
            exit 1
          fi

  trigger-image-promotion:
    name: Promote images on Force Run
    needs:
      - build-docker
      - build-docker-plus
      - build-docker-nap
      - tag-results
    permissions:
      contents: write # for pushing to Helm Charts repository
      id-token: write # To sign into Google Container Registry
      actions: read
      packages: write # for helm to push to GHCR
      security-events: write
      pull-requests: write # for scout report
    uses: ./.github/workflows/image-promotion.yml
    if: ${{ inputs.force && inputs.force || false }}