Skip to content

Commit

Permalink
ci: Add tft plan and workflow
Browse files Browse the repository at this point in the history
This change is for running tests in Testing Farm CI. This is a replacement for
BaseOS CI that we are currently using. Running it Testing Farm gives us more
control.

It adds a workflow for running tests, and a plans directory containing a test
plan and a README-plans.md with some info.

Note that this workflow runs from the main branch. This means that changes to
the workflow must be merged to main, then pull requests will be able to run it.
This is because the workflow uses on: issue_comment context, this is a security
measure recommended by GitHub. It saves us from leaking organization secrets.

The functionality is WIP, so await future fixes and updates.

Signed-off-by: Sergei Petrosian <[email protected]>
  • Loading branch information
spetrosi committed Jul 25, 2024
1 parent ff70929 commit 7c2b887
Show file tree
Hide file tree
Showing 5 changed files with 277 additions and 1 deletion.
1 change: 1 addition & 0 deletions .fmf/version
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1
178 changes: 178 additions & 0 deletions .github/workflows/tft.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
name: Run integration tests in Testing Farm
on:
issue_comment:
types:
- created
permissions:
contents: read
# This is required for the ability to create/update the Pull request status
statuses: write
# The concurrency key is used to prevent multiple workflows from running at the same time
concurrency:
# group name contains reponame-pr_num to allow simualteneous runs in different PRs
group: testing-farm-${{ github.event.repository.name }}-${{ github.event.issue.number }}
cancel-in-progress: true
jobs:
prepare_vars:
name: Get info from role and PR to determine if and how to test
# Let's schedule tests only on user request. NOT automatically.
# Only repository owner or member can schedule tests
if: |
github.event.issue.pull_request
&& (contains(github.event.comment.body, '[citest]') || contains(github.event.comment.body, '[citest-all]'))
&& (contains(fromJson('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.comment.author_association)
|| contains('systemroller', github.event.comment.user.login))
runs-on: ubuntu-latest
outputs:
supported_platforms: ${{ steps.supported_platforms.outputs.supported_platforms }}
head_sha: ${{ steps.head_sha.outputs.head_sha }}
datetime: ${{ steps.datetime.outputs.datetime }}
memory: ${{ steps.memory.outputs.memory }}
steps:

- name: Checkout repo
uses: actions/checkout@v4

- name: Get head sha of the PR
id: head_sha
run: |
head_sha=$(gh api "repos/$REPO/pulls/$PR_NO" --jq '.head.sha')
echo "head_sha=$head_sha" >> $GITHUB_OUTPUT
env:
REPO: ${{ github.repository }}
PR_NO: ${{ github.event.issue.number }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Get cuurent datetime
id: datetime
run: |
printf -v datetime '%(%Y%m%d-%H%M%S)T' -1
echo "datetime=$datetime" >> $GITHUB_OUTPUT
- name: Get memory
id: memory
run: |
if [ -d tests/provision.fmf ]; then
memory=$(grep -rPo ' m: \K(.*)' tests/provision.fmf)
fi
if [ -n "$memory" ]; then
echo "memory=$memory" >> $GITHUB_OUTPUT
else
echo "memory=2048" >> $GITHUB_OUTPUT
fi
- name: Get supported platforms
id: supported_platforms
run: |
supported_platforms=""
meta_main=meta/main.yml
# All Fedora are supported, add latest Fedora versions to supported_platforms
if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qi fedora$; then
supported_platforms+=" Fedora-39"
supported_platforms+=" Fedora-40"
fi
# Specific Fedora versions supported
if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qiP 'fedora\d+$'; then
for fedora_ver in $(yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -iPo 'fedora\K(\d+$)'); do
supported_platforms+=" Fedora-$fedora_ver"
done
fi
if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qi el7; then
supported_platforms+=" CentOS-7-latest"
fi
for ver in 8 9 10; do
if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qi el"$ver"; then
supported_platforms+=" CentOS-Stream-$ver"
fi
done
echo "supported_platforms=$supported_platforms" >> $GITHUB_OUTPUT
testing-farm:
name: ${{ matrix.platform }}/ansible-${{ matrix.ansible_version }}
needs: prepare_vars
strategy:
fail-fast: false
matrix:
include:
- platform: Fedora-39
ansible_version: 2.17
- platform: Fedora-40
ansible_version: 2.17
- platform: CentOS-7-latest
ansible_version: 2.9
- platform: CentOS-Stream-8
ansible_version: 2.9
# On CentOS-Stream-8, latest supported Ansible is 2.16
- platform: CentOS-Stream-8
ansible_version: 2.16
- platform: CentOS-Stream-9
ansible_version: 2.17
- platform: CentOS-Stream-10
ansible_version: 2.17
runs-on: ubuntu-latest
env:
ARTIFACTS_DIR_NAME: "tf_${{ github.event.repository.name }}-${{ github.event.issue.number }}_\
${{ matrix.platform }}-${{ matrix.ansible_version }}_\
${{ needs.prepare_vars.outputs.datetime }}/artifacts"
ARTIFACT_TARGET_DIR: /srv/pub/alt/linuxsystemroles/logs
steps:
- name: Set commit status as pending
if: contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)
uses: myrotvorets/set-commit-status-action@master
with:
sha: ${{ needs.prepare_vars.outputs.head_sha }}
status: pending
context: ${{ matrix.platform }}|ansible-${{ matrix.ansible_version }}
description: Test started
targetUrl: ""

- name: Set commit status as success with a description that platform is skipped
if: "!contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)"
uses: myrotvorets/set-commit-status-action@master
with:
sha: ${{ needs.prepare_vars.outputs.head_sha }}
status: success
context: ${{ matrix.platform }}|ansible-${{ matrix.ansible_version }}
description: The role does not support this platform. Skipping.
targetUrl: ""

- name: Run test in testing farm
uses: sclorg/testing-farm-as-github-action@v2
if: contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)
env:
ARTIFACTS_DIR: ${{ env.ARTIFACT_TARGET_DIR }}/${{ env.ARTIFACTS_DIR_NAME }}
ARTIFACTS_URL: https://dl.fedoraproject.org/pub/alt/linuxsystemroles/logs/${{ env.ARTIFACTS_DIR_NAME }}
with:
git_url: ${{ github.server_url }}/${{ github.repository }}
git_ref: ${{ needs.prepare_vars.outputs.head_sha }}
pipeline_settings: '{ "type": "tmt-multihost" }'
variables: "ANSIBLE_VER=${{ matrix.ansible_version }};\
REPO_NAME=${{ github.event.repository.name }};\
GITHUB_ORG=linux-system-roles;\
PR_NUM=${{ github.event.issue.number }};\
ARTIFACTS_DIR=${{ env.ARTIFACTS_DIR }};\
ARTIFACTS_URL=${{ env.ARTIFACTS_URL }}"
# Note that LINUXSYSTEMROLES_SSH_KEY must be single-line, TF doesn't read multi-line variables fine.
secrets: "LINUXSYSTEMROLES_USER=${{ secrets.LINUXSYSTEMROLES_USER }};\
LINUXSYSTEMROLES_DOMAIN=${{ secrets.LINUXSYSTEMROLES_DOMAIN }};\
LINUXSYSTEMROLES_SSH_KEY=${{ secrets.LINUXSYSTEMROLES_SSH_KEY }}"
compose: ${{ matrix.platform }}
# There are two blockers for using public ranch:
# 1. multihost is not supported in public https://github.com/teemtee/tmt/issues/2620
# 2. Security issue that leaks long secrets - Jira TFT-2698
tf_scope: private
api_key: ${{ secrets.TF_API_KEY_RH }}
update_pull_request_status: false
tmt_hardware: '{ "memory": ">= ${{ needs.prepare_vars.outputs.memory }} MB" }'

- name: Set final commit status
uses: myrotvorets/set-commit-status-action@master
if: always() && contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)
env:
ARTIFACTS_URL: https://dl.fedoraproject.org/pub/alt/linuxsystemroles/logs/${{ env.ARTIFACTS_DIR_NAME }}
with:
sha: ${{ needs.prepare_vars.outputs.head_sha }}
status: ${{ job.status }}
context: ${{ matrix.platform }}|ansible-${{ matrix.ansible_version }}
description: Test finished
targetUrl: ${{ env.ARTIFACTS_URL }}
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# SELinux

[![ansible-lint.yml](https://github.com/linux-system-roles/selinux/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/ansible-lint.yml) [![ansible-test.yml](https://github.com/linux-system-roles/selinux/actions/workflows/ansible-test.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/ansible-test.yml) [![codeql.yml](https://github.com/linux-system-roles/selinux/actions/workflows/codeql.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/codeql.yml) [![markdownlint.yml](https://github.com/linux-system-roles/selinux/actions/workflows/markdownlint.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/markdownlint.yml) [![python-unit-test.yml](https://github.com/linux-system-roles/selinux/actions/workflows/python-unit-test.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/python-unit-test.yml) [![woke.yml](https://github.com/linux-system-roles/selinux/actions/workflows/woke.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/woke.yml)
[![ansible-lint.yml](https://github.com/linux-system-roles/selinux/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/ansible-lint.yml) [![ansible-test.yml](https://github.com/linux-system-roles/selinux/actions/workflows/ansible-test.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/ansible-test.yml) [![codeql.yml](https://github.com/linux-system-roles/selinux/actions/workflows/codeql.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/codeql.yml) [![markdownlint.yml](https://github.com/linux-system-roles/selinux/actions/workflows/markdownlint.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/markdownlint.yml) [![python-unit-test.yml](https://github.com/linux-system-roles/selinux/actions/workflows/python-unit-test.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/python-unit-test.yml) [![tft.yml](https://github.com/linux-system-roles/selinux/actions/workflows/tft.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/tft.yml) [![woke.yml](https://github.com/linux-system-roles/selinux/actions/workflows/woke.yml/badge.svg)](https://github.com/linux-system-roles/selinux/actions/workflows/woke.yml)

## Expected functionality

Expand Down
35 changes: 35 additions & 0 deletions plans/README-plans.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# Introduction CI Testing Plans

Linux System Roles CI runs [tmt](https://tmt.readthedocs.io/en/stable/index.html) test plans in [Testing farm](https://docs.testing-farm.io/Testing%20Farm/0.1/index.html) with the [tmt.yml](https://github.com/linux-system-roles/selinux/blob/main/.github/workflows/tmt.yml) GitHub workflow.

The plans/general.fmf plan is a test plan that is general for all roles. It does the following steps:

1. Provisions two machines, one used as an Ansible control node, and second used as a managed node.
2. Does the required preparation on machines.
3. For the given role and the given PR, runs the general test from [test.sh](https://github.com/linux-system-roles/tft-tests/blob/main/tests/general/test.sh).

The [tmt.yml](https://github.com/linux-system-roles/selinux/blob/main/.github/workflows/tmt.yml) workflow runs the above plan and uploads the results to our Fedora storage for public access.
This workflow uses Testing Farm's Github Action [Schedule tests on Testing Farm](https://github.com/marketplace/actions/schedule-tests-on-testing-farm).

## Running Tests

You can run tests locally with the `tmt try` cli.

### Prerequisites

* Install `tmt` as described in [Installation](https://tmt.readthedocs.io/en/stable/stories/install.html).

### Running Tests Locally

For now, this functionality requires you to push changes to a PR because the plan only runs from the main branch, or from the PR branch.
So this is WIP.

To run tests locally, in the role repository, enter `tmt run plans --name plans/general <platform>`.
Where `<platform>` is the name of the platform you want to run tests against.

For example, `tmt run plans --name plans/general Fedora-40`.

This command identifies the plans/general plan and provisions two machines, one used as an Ansible control node, and second used as a managed node.

You can also use `tmt try` to get to an interreactive prompt and be able to ssh into test machines.
You must run `tmt try -p plans/general Fedora-40`, and the in the promt type `p` to prepare the machines, then `t` to run the tests.
62 changes: 62 additions & 0 deletions plans/general.fmf
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
summary: A general test for a system role
provision:
- name: control_node
role: control_node
# TF uses `how: artemis`, tmt try uses `how: virtual`. No need to define `how`
# `connection: system` is for `how: virtual` to make VMs get a real IP to configure ssh easily
# This setting is ignored on artemis so we can keep it
connection: system
- name: managed_node1
role: managed_node
connection: system
environment:
ANSIBLE_VER: 2.17
REPO_NAME: selinux
PYTHON_VERSION: 3.12
SYSTEM_ROLES_ONLY_TESTS: ""
PR_NUM: ""
prepare:
- name: Use vault.centos.org repos (CS 7, 8 EOL workaround)
script: |
if grep -q -e 'CentOS Stream release 8' -e 'CentOS Linux release 7.9' /etc/redhat-release; then
sed -i '/^mirror/d;s/#\(baseurl=http:\/\/\)mirror/\1vault/' /etc/yum.repos.d/*.repo
fi

- name: Enable epel to install beakerlib on all platforms except CS10 and Fedora, there it's not available and not needed
script: |
if ! grep -q -e 'CentOS Stream release 10' -e 'Fedora release' /etc/redhat-release; then
yum install epel-release -y
fi
where: control_node

- name: Additional steps to enable EPEL on EL 7
script: |
if grep -q 'CentOS Linux release 7.9' /etc/redhat-release; then
yum install yum-utils -y
yum-config-manager --enable epel epel-debuginfo epel-source
fi
where: control_node

- name: Install python on managed node when running CS8 with ansible!=2.9
script: |
if [ "$ANSIBLE_VER" != "2.9" ] && grep -q 'CentOS Stream release 8' /etc/redhat-release; then
dnf install -y python"$PYTHON_VERSION"
fi
where: managed_node

- name: Distribute SSH keys when provisioned with how=virtual
script: |
if [ -f ${TMT_TREE%/*}/provision/control_node/id_ecdsa.pub ]; then
cat ${TMT_TREE%/*}/provision/control_node/id_ecdsa.pub >> ~/.ssh/authorized_keys
fi
where: managed_node

discover:
- name: Run test playbooks from control_node
how: fmf
url: https://github.com/linux-system-roles/tft-tests
ref: main
where: control_node
filter: tag:test_playbooks
execute:
how: tmt

0 comments on commit 7c2b887

Please sign in to comment.