ansible-test.yml

name: CI
on:
  # Run CI against all pushes (direct commits, also merged PRs), Pull Requests
  push:
    branches: [main]
    paths-ignore:
      - 'docs/**'
      - '.github/workflows/_shared-*'
      - '.github/workflows/docs*.yml'
      - '.github/actions/docs/**'
  pull_request:
    paths-ignore:
      - 'docs/**'
      - '.github/workflows/_shared-*'
      - '.github/workflows/docs*.yml'
      - '.github/actions/docs/**'
  schedule:
    - cron: '0 14 * * *'
env:
  NAMESPACE: community
  COLLECTION_NAME: hashi_vault
  ANSIBLE_FORCE_COLOR: true
  ANSIBLE_COLLECTIONS_PATHS: ${{ github.workspace }}

jobs:

###
# Sanity tests (REQUIRED)
# https://docs.ansible.com/ansible/latest/dev_guide/testing_sanity.html

  sanity:
    name: Sanity (Ⓐ${{ matrix.ansible }})
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        runner:
          - ubuntu-latest
        test_container:
          - default
        ansible:
          - stable-2.9
          - stable-2.10
          - stable-2.11
          - stable-2.12
          - stable-2.13
          - stable-2.14
          - stable-2.15
          - devel
    steps:

      # ansible-test requires the collection to be in a directory in the form
      # .../ansible_collections/${{env.NAMESPACE}}/${{env.COLLECTION_NAME}}/
      - name: Initialize env vars
        uses: briantist/ezenv@v1
        with:
          env: |
            COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME}
            TEST_INVOCATION="sanity --docker ${{ matrix.test_container }} -v --color ${{ github.event_name != 'schedule' && '--coverage' || '' }}"

      - name: Check out code
        uses: actions/checkout@v3
        with:
          path: ${{ env.COLLECTION_PATH }}

      - name: Link to .github  # easier access to local actions
        run: ln -s "${COLLECTION_PATH}/.github" .github

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          # it is just required to run that once as "ansible-test sanity" in the docker image
          # will run on all python versions it supports.
          python-version: 3.9

      # Install the head of the given branch (devel, stable-2.14)
      - name: Install ansible-base (${{ matrix.ansible }})
        run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check

      - name: Pull Ansible test images
        timeout-minutes: 5
        continue-on-error: true
        uses: ./.github/actions/pull-ansible-test-images
        with:
          working-directory: ${{ env.COLLECTION_PATH }}
          ansible-test-invocation: ${{ env.TEST_INVOCATION }}

      # run ansible-test sanity inside of Docker.
      # The docker container has all the pinned dependencies that are required
      # and all python versions ansible supports.
      - name: Run sanity tests
        run: ansible-test ${{ env.TEST_INVOCATION }}
        working-directory: ${{ env.COLLECTION_PATH }}

      - name: Generate coverage report
        if: ${{ github.event_name != 'schedule' }}
        run: ansible-test coverage xml -v --requirements --group-by command --group-by environment --group-by target
        working-directory: ${{ env.COLLECTION_PATH }}

      - name: Upload ${{ github.job }} coverage reports
        if: ${{ github.event_name != 'schedule' }}
        uses: actions/upload-artifact@v3
        with:
          name: coverage=${{ github.job }}=ansible_${{ matrix.ansible }}=data
          path: ${{ env.COLLECTION_PATH }}/tests/output/reports/
          if-no-files-found: error
          retention-days: 1


  units:
    runs-on: ${{ matrix.runner }}
    name: Units (Ⓐ${{ matrix.ansible }}-py${{ matrix.python && matrix.python || 'ANY' }})
    strategy:
      # As soon as the first unit test fails, cancel the others to free up the CI queue
      fail-fast: false
      matrix:
        runner:
          - ubuntu-latest
        test_container:
          - default
        python:
          # only used for pre-ansible 2.12
          - '3.6'
          - '3.7'
          - '3.8'
        ansible:
          - stable-2.9
          - stable-2.10
          - stable-2.11
        include:
          - ansible: stable-2.12
            python: ''
            runner: ubuntu-latest
            test_container: default
          - ansible: stable-2.13
            python: ''
            runner: ubuntu-latest
            test_container: default
          - ansible: stable-2.14
            python: ''
            runner: ubuntu-latest
            test_container: default
          - ansible: stable-2.15
            python: ''
            runner: ubuntu-latest
            test_container: default
          - ansible: devel
            python: ''
            runner: ubuntu-latest
            test_container: default

    steps:
      - name: Initialize env vars
        uses: briantist/ezenv@v1
        with:
          env: |
            COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME}
            TEST_INVOCATION="units --color --docker ${{ matrix.test_container }} ${{ github.event_name != 'schedule' && '--coverage' || '' }} ${{ matrix.python && format('--python {0}', matrix.python) || '' }}"

      - name: Check out code
        uses: actions/checkout@v3
        with:
          path: ${{ env.COLLECTION_PATH }}

      - name: Link to .github  # easier access to local actions
        run: ln -s "${COLLECTION_PATH}/.github" .github

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          # it is just required to run that once as "ansible-test units" in the docker image
          # will run on all python versions it supports.
          python-version: 3.9

      - name: Install ansible-base (${{ matrix.ansible }})
        run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check

      - name: Pull Ansible test images
        timeout-minutes: 5
        continue-on-error: true
        uses: ./.github/actions/pull-ansible-test-images
        with:
          working-directory: ${{ env.COLLECTION_PATH }}
          ansible-test-invocation: ${{ env.TEST_INVOCATION }}

      # Run the unit tests
      - name: Run unit test
        run: ansible-test ${{ env.TEST_INVOCATION }}
        working-directory: ${{ env.COLLECTION_PATH }}

      - name: Generate coverage report
        if: ${{ github.event_name != 'schedule' }}
        run: ansible-test coverage xml -v --requirements --group-by command --group-by environment --group-by target
        working-directory: ${{ env.COLLECTION_PATH }}

      - name: Upload ${{ github.job }} coverage reports
        if: ${{ github.event_name != 'schedule' }}
        uses: actions/upload-artifact@v3
        with:
          name: coverage=${{ github.job }}=ansible_${{ matrix.ansible }}=data
          path: ${{ env.COLLECTION_PATH }}/tests/output/reports/
          if-no-files-found: error
          retention-days: 1

###
# Integration tests (RECOMMENDED)
#
# https://docs.ansible.com/ansible/latest/dev_guide/testing_integration.html

  integration:
    runs-on: ${{ matrix.runner }}
    name: I (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }})
    strategy:
      fail-fast: false
      matrix:
        runner:
          - ubuntu-latest
        test_container:
          - default
        ansible:
          - stable-2.9
          - stable-2.10
          - stable-2.11
          - stable-2.12
          - stable-2.13
          - stable-2.14
          - stable-2.15
          - devel
        python:
          - '3.6'
          - '3.7'
          - '3.8'
          - '3.9'
          - '3.10'
          - '3.11'
        exclude:
          # https://docs.ansible.com/ansible/devel/installation_guide/intro_installation.html#control-node-requirements
          # https://docs.ansible.com/ansible/devel/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix
          - ansible: 'stable-2.9'
            python: '3.10'
          - ansible: 'stable-2.9'
            python: '3.9'
          - ansible: 'stable-2.9'
            python: '3.11'
          - ansible: 'stable-2.10'
            python: '3.10'
          - ansible: 'stable-2.10'
            python: '3.11'
          - ansible: 'stable-2.11'
            python: '3.10'
          - ansible: 'stable-2.11'
            python: '3.11'
          - ansible: 'stable-2.12'
            python: '3.11'
          - ansible: 'stable-2.13'
            python: '3.11'
          - ansible: 'stable-2.15'
            python: '3.6'
          - ansible: 'stable-2.15'
            python: '3.7'
          - ansible: 'stable-2.15'
            python: '3.8'
          - ansible: 'devel'
            python: '3.6'
          - ansible: 'devel'
            python: '3.7'
          - ansible: 'devel'
            python: '3.8'

    steps:
      - name: Initialize env vars
        uses: briantist/ezenv@v1
        with:
          env: |
            COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME}
            COLLECTION_INTEGRATION_PATH=${COLLECTION_PATH}/tests/integration
            COLLECTION_INTEGRATION_TARGETS=${COLLECTION_INTEGRATION_PATH}/targets
            TEST_INVOCATION="integration -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --docker ${{ matrix.test_container }} ${{ github.event_name != 'schedule' && '--coverage' || '' }} --docker-network hashi_vault_default"

      - name: Check out code
        uses: actions/checkout@v3
        with:
          path: ${{ env.COLLECTION_PATH }}

      - name: Link to .github  # easier access to local actions
        run: ln -s "${COLLECTION_PATH}/.github" .github

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.9

      - name: Get Vault versions
        id: vault_versions
        uses: ./.github/actions/docker-image-versions
        with:
          num_major_versions: 1
          num_minor_versions: 2
          num_micro_versions: 1

      - name: Install ansible-base (${{ matrix.ansible }})
        run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check

      - name: Pull Ansible test images
        timeout-minutes: 5
        continue-on-error: true
        uses: ./.github/actions/pull-ansible-test-images
        with:
          working-directory: ${{ env.COLLECTION_PATH }}
          ansible-test-invocation: ${{ env.TEST_INVOCATION }}

      - name: Set Vault Version (older)
        uses: briantist/ezenv@v1
        with:
          env: VAULT_VERSION=${{ fromJSON(steps.vault_versions.outputs.versions)[1] }}

      - name: Prepare docker dependencies (Vault ${{ env.VAULT_VERSION }})
        run: ./setup.sh -e vault_version=${VAULT_VERSION}
        working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_gha

      - name: Run integration test (Vault ${{ env.VAULT_VERSION }})
        run: ansible-test ${{ env.TEST_INVOCATION }}
        working-directory: ${{ env.COLLECTION_PATH }}

      - name: Set Vault Version (newer)
        uses: briantist/ezenv@v1
        with:
          env: VAULT_VERSION=${{ fromJSON(steps.vault_versions.outputs.versions)[0] }}

      - name: Prepare docker dependencies (Vault ${{ env.VAULT_VERSION }})
        run: ./setup.sh -e vault_version=${VAULT_VERSION}
        working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_gha

      - name: Run integration test (Vault ${{ env.VAULT_VERSION }})
        run: ansible-test ${{ env.TEST_INVOCATION }}
        working-directory: ${{ env.COLLECTION_PATH }}

        # ansible-test support producing code coverage data
      - name: Generate coverage report
        if: ${{ github.event_name != 'schedule' }}
        run: ansible-test coverage xml -v --requirements --group-by command --group-by environment --group-by target
        working-directory: ${{ env.COLLECTION_PATH }}

      - name: Upload ${{ github.job }} coverage reports
        if: ${{ github.event_name != 'schedule' }}
        uses: actions/upload-artifact@v3
        with:
          name: coverage=${{ github.job }}=ansible_${{ matrix.ansible }}=${{ matrix.python }}=data
          path: ${{ env.COLLECTION_PATH }}/tests/output/reports/
          if-no-files-found: error
          retention-days: 1

  local_test_invocation:
    if: false
    runs-on: ${{ matrix.runner }}
    name: LI - ${{ matrix.runner }} (Ⓐ${{ matrix.ansible }}+py${{ matrix.python }})
    strategy:
      fail-fast: false
      matrix:
        ansible:
          - stable-2.15
          - devel
        python:
          - 3.9
        runner:
          - ubuntu-latest
        test_container:
          - default

    steps:
      - name: Initialize env vars
        uses: briantist/ezenv@v1
        with:
          env: |
            COLLECTION_PATH=ansible_collections/${NAMESPACE}/${COLLECTION_NAME}
            COLLECTION_INTEGRATION_PATH=${COLLECTION_PATH}/tests/integration
            COLLECTION_INTEGRATION_TARGETS=${COLLECTION_INTEGRATION_PATH}/targets
            DOCKER_TEST_INVOCATION="integration -v --color --retry-on-error --continue-on-error --controller docker:${{ matrix.test_container }},python=${{ matrix.python }} ${{ github.event_name != 'schedule' && '--coverage' || '' }}"

      - name: Check out code
        uses: actions/checkout@v3
        with:
          path: ${{ env.COLLECTION_PATH }}

      - name: Link to .github  # easier access to local actions
        run: ln -s "${COLLECTION_PATH}/.github" .github

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}

      - name: Install ansible-base (${{ matrix.ansible }})
        run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check

      - name: Install community.crypto
        uses: ./.github/actions/collection-via-git
        with:
          collection: community.crypto

      - name: Install community.docker
        uses: ./.github/actions/collection-via-git
        with:
          collection: community.docker

      - name: Pull Ansible test images
        timeout-minutes: 5
        continue-on-error: true
        uses: ./.github/actions/pull-ansible-test-images
        with:
          working-directory: ${{ env.COLLECTION_PATH }}
          ansible-test-invocation: ${{ env.DOCKER_TEST_INVOCATION }}

      - name: localenv_docker - setup
        run: |
          pwd
          pip install -r files/requirements/requirements.txt -c files/requirements/constraints.txt
          ./setup.sh
        working-directory: ${{ env.COLLECTION_INTEGRATION_TARGETS }}/setup_localenv_docker

      - name: localenv_docker - Run integration test (in docker)
        run: |
          ansible-test ${{ env.DOCKER_TEST_INVOCATION }} --docker-network hashi_vault_default
        working-directory: ${{ env.COLLECTION_PATH }}

      - name: Run integration again (ensure tests do not break against still-running containers)
        run: |
          ansible-test ${{ env.DOCKER_TEST_INVOCATION }} --docker-network hashi_vault_default
        working-directory: ${{ env.COLLECTION_PATH }}

      #TODO add capability in the Ansible side once vault_list and vault_delete exist
      - name: Run a third time, but delete Vault's cubbyhole contents first
        working-directory: ${{ env.COLLECTION_PATH }}
        env:
          VAULT_TOKEN: 47542cbc-6bf8-4fba-8eda-02e0a0d29a0a
          VAULT_ADDR: http://vault:8200
        run: |
          echo 'vault list cubbyhole \
          | tail -n +3 \
          | xargs -I{} -n 1 vault delete cubbyhole/{}' \
          | docker run --rm --network hashi_vault_default -e VAULT_TOKEN -e VAULT_ADDR -i hashicorp/vault sh

          ansible-test ${{ env.DOCKER_TEST_INVOCATION }} --docker-network hashi_vault_default

        # ansible-test support producing code coverage data
      - name: Generate coverage report
        if: ${{ github.event_name != 'schedule' }}
        run: ansible-test coverage xml -v --requirements --group-by command --group-by environment --group-by target
        working-directory: ${{ env.COLLECTION_PATH }}

      - name: Upload ${{ github.job }} coverage reports
        if: ${{ github.event_name != 'schedule' }}
        uses: actions/upload-artifact@v3
        with:
          name: coverage=${{ github.job }}=${{ matrix.runner }}=ansible_${{ matrix.ansible }}=${{ matrix.python }}=data
          path: ${{ env.COLLECTION_PATH }}/tests/output/reports/
          if-no-files-found: error
          retention-days: 1

  upload-coverage:
    needs:
      - sanity
      - units
      - integration
      - local_test_invocation
    # don't upload coverage on scheduled runs
    # https://github.com/ansible-collections/community.hashi_vault/issues/180
    if: false
    name: Upload Codecov reports
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v3

      - name: Download artifacts
        uses: actions/download-artifact@v3
        with:
          path: ./cov

      # See the reports at https://codecov.io/gh/ansible-collections/community.hashi_vault
      - name: Upload coverage reports to Codecov
        uses: ./.github/actions/ansible-codecov
        with:
          directory: ./cov
          directory-flag-pattern: =ansible_{ansible-%}=