Skip to content

Commit

Permalink
build(linux): add Linux packaging and e2e tests (#1071)
Browse files Browse the repository at this point in the history
build(linux): add Linux packaging and e2e tests (#1071)

- Add packaging and e2e runners which use the packages to run tests
- Before merging this, we need to:
  - add linux runners to this repository (by merging runfinch/infrastructure#683)
  - remove the change in this PR which makes it so CI runs on all branches, not just main
  - uncomment the disabling of the macOS/Windows runners

Signed-off-by: Justin Alvarez <[email protected]>
  • Loading branch information
pendo324 authored Sep 18, 2024
1 parent c382e22 commit 985570e
Show file tree
Hide file tree
Showing 29 changed files with 920 additions and 169 deletions.
1 change: 1 addition & 0 deletions .github/workflows/ci-docs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ on:
paths:
- '**.md'
- 'contrib/**'
- '!contrib/packaging/**'
- '.github/CODEOWNERS'

jobs:
Expand Down
192 changes: 49 additions & 143 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,18 @@ on:
branches:
- main
paths-ignore:
- '**.md'
- 'contrib/**'
- '.github/CODEOWNERS'
- "**.md"
- "contrib/**"
- ".github/CODEOWNERS"
pull_request:
branches:
- main
paths-ignore:
- '**.md'
- 'contrib/**'
- '.github/CODEOWNERS'
paths:
- "**.go"
- "contrib/packaging/**"
- "!contrib/hello-finch/**"
- "!.github/CODEOWNERS"
workflow_dispatch:
permissions:
id-token: write
contents: write
Expand Down Expand Up @@ -141,154 +143,58 @@ jobs:
go-version-file: go.mod
cache: true
- run: make check-licenses
e2e-tests:
macos-e2e-tests:
strategy:
fail-fast: false
matrix:
os:
[
[self-hosted, macos, amd64, 13, test],
[self-hosted, macos, amd64, 14, test],
[self-hosted, macos, arm64, 13, test],
[self-hosted, macos, arm64, 14, test],
]
test-command: ['test-e2e-vm-serial', 'test-e2e-container']
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
# We need to get all the git tags to make version injection work. See VERSION in Makefile for more detail.
fetch-depth: 0
persist-credentials: false
submodules: recursive
- uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
with:
go-version-file: go.mod
cache: true
- name: Set output variables
id: vars
run: |
has_creds=${{ (github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name) && github.actor != 'dependabot[bot]' }}
echo "has_creds=$has_creds" >> $GITHUB_OUTPUT
- name: configure aws credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
if: ${{ steps.vars.outputs.has_creds == true }}
with:
role-to-assume: ${{ secrets.ROLE }}
role-session-name: credhelper-test
aws-region: ${{ secrets.REGION }}
- name: Clean up previous files
run: |
sudo rm -rf /opt/finch
sudo rm -rf ~/.finch
sudo rm -rf ./_output
if pgrep '^qemu-system'; then
sudo pkill '^qemu-system'
fi
if pgrep '^socket_vmnet'; then
sudo pkill '^socket_vmnet'
fi
- name: Install Rosetta 2
run: echo "A" | softwareupdate --install-rosetta || true
- run: brew install lz4 automake autoconf libtool yq
shell: zsh {0}
- name: Build project
run: |
export PATH="/opt/homebrew/opt/libtool/libexec/gnubin:$PATH"
make
shell: zsh {0}
- run: |
git status
git clean -f -d
REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} make ${{ matrix.test-command }}
shell: zsh {0}
version: ["13", "14"]
test-command: ["test-e2e-vm-serial", "test-e2e-container"]
arch: ["X64", "arm64"]
runner-type: ["test"]
uses: ./.github/workflows/e2e-macos.yaml
secrets: inherit
with:
arch: ${{ matrix.arch }}
version: ${{ matrix.version }}
runner-type: ${{ matrix.runner-type }}
test-command: ${{ matrix.test-command }}
windows-e2e-tests:
strategy:
fail-fast: false
matrix:
os: [[self-hosted, windows, amd64, test]]
test-command: ['test-e2e-vm-serial', 'test-e2e-container']
runs-on: ${{ matrix.os }}
timeout-minutes: 180
steps:
- name: Configure git CRLF settings
run: |
git config --global core.autocrlf false
git config --global core.eol lf
- name: Cleanup previous checkouts
run: |
takeown /F C:\actions-runner\_work\finch /R
Remove-Item C:\actions-runner\_work\finch\finch -Recurse -Force -ErrorAction Ignore
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
# We need to get all the git tags to make version injection work. See VERSION in Makefile for more detail.
fetch-depth: 0
persist-credentials: false
submodules: recursive
- name: Set output variables
id: vars
run: |
$has_creds="${{ (github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name) && github.actor != 'dependabot[bot]'}}"
echo "has_creds=$has_creds" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
exit 0 # if $has_creds is false, powershell will exit with code 1 and this step will fail
- name: configure aws credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
if: env.has_creds == 'true'
with:
role-to-assume: ${{ secrets.ROLE }}
role-session-name: credhelper-test
aws-region: ${{ secrets.REGION }}
- name: Remove Finch VM
run: |
# We want these cleanup commands to always run, ignore errors so the step completes.
$ErrorActionPreference = 'Ignore'
taskkill /f /im wslservice.exe 2> nul || cmd /c "exit /b 0"
wsl --list --verbose
wsl --shutdown
wsl --unregister lima-finch
wsl --list --verbose
- name: Clean up previous files
run: |
Remove-Item C:\Users\Administrator\.finch -Recurse -ErrorAction Ignore
Remove-Item C:\Users\Administrator\AppData\Local\.finch -Recurse -ErrorAction Ignore
make clean
cd deps/finch-core && make clean
- name: Build project
run: |
git status
make
- name: Run e2e tests
run: |
# set path to use newer ssh version
$newPath = (";C:\Program Files\Git\bin\;" + "C:\Program Files\Git\usr\bin\;" + "$env:Path")
$env:Path = $newPath
# set networking config option to allow for VM/container -> host communication
echo "[experimental]`nnetworkingMode=mirrored`nhostAddressLoopback=true" > C:\Users\Administrator\.wslconfig
test-command: ["test-e2e-vm-serial", "test-e2e-container"]
arch: ["amd64"]
runner-type: ["test"]
uses: ./.github/workflows/e2e-windows.yaml
secrets: inherit
with:
arch: ${{ matrix.arch }}
runner-type: ${{ matrix.runner-type }}
test-command: ${{ matrix.test-command }}
linux-e2e-tests:
strategy:
fail-fast: false
matrix:
os: ["amazonlinux"]
arch: ["X64", "arm64"]
version: ["2023", "2"]
test-command: ["test-e2e-container"]
runner-type: ["test"]
uses: ./.github/workflows/e2e-linux.yaml
secrets: inherit
with:
os: ${{ matrix.os }}
arch: ${{ matrix.arch }}
version: ${{ matrix.version }}
runner-type: ${{ matrix.runner-type }}
test-command: ${{ matrix.test-command }}

git status
git clean -f -d
make ${{ matrix.test-command }}
- name: Remove Finch VM and Clean Up Previous Environment
if: ${{ always() }}
run: |
# We want these cleanup commands to always run, ignore errors so the step completes.
$ErrorActionPreference = 'Ignore'
taskkill /f /im wslservice.exe 2> nul || cmd /c "exit /b 0"
wsl --list --verbose
wsl --shutdown
wsl --unregister lima-finch
wsl --list --verbose
Remove-Item C:\Users\Administrator\AppData\Local\.finch -Recurse
make clean
cd deps/finch-core && make clean
exit 0 # Cleanup may set the exit code e.g. if a file doesn't exist; just ignore
mdlint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: avto-dev/markdown-lint@04d43ee9191307b50935a753da3b775ab695eceb # v1.5.0
with:
args: '**/*.md'
args: "**/*.md"
# CHANGELOG.md is only updated by release-please bot.
ignore: 'CHANGELOG.md'
ignore: "CHANGELOG.md"
103 changes: 103 additions & 0 deletions .github/workflows/e2e-linux.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
name: e2e-linux
on:
workflow_call:
inputs:
os:
type: string
required: true
arch:
type: string
required: true
version:
type: string
required: true
runner-type:
type: string
required: true
test-command:
type: string
required: true

permissions:
# This is required for configure-aws-credentials to request an OIDC JWT ID token to access AWS resources later on.
# More info: https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings
id-token: write
# This is required for actions/checkout
contents: read

jobs:
test:
runs-on:
[
"self-hosted",
"${{ inputs.os }}",
"${{ inputs.arch }}",
"${{ inputs.version }}",
"${{ inputs.runner-type }}",
]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
# checkout uses node version 20, which doesn't run on AL2
if: ${{ ! (startsWith(inputs.os, 'amazon') && inputs.version == '2' ) }}
with:
# We need to get all the git tags to make version injection work. See VERSION in Makefile for more detail.
fetch-depth: 0
persist-credentials: false
submodules: recursive
- name: Check repo out manually
if: ${{ (startsWith(inputs.os, 'amazon') && inputs.version == '2' ) }}
run: |
git clone https://github.com/${GITHUB_REPOSITORY}.git .
git config --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/prh/*"
git config --add remote.origin.fetch "+refs/pull/*/merge:refs/remotes/origin/prm/*"
git fetch origin
git checkout ${GITHUB_SHA}
- name: Set output variables
id: vars
run: |
has_creds=${{ (github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name) && github.actor != 'dependabot[bot]' }}
echo "has_creds=$has_creds" >> $GITHUB_OUTPUT
- name: configure aws credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
# this action requires node20, skip on AL2
if: ${{ steps.vars.outputs.has_creds == true && (!(startsWith(inputs.os, 'amazon') && inputs.version == '2' ))}}
with:
role-to-assume: ${{ secrets.ROLE }}
role-session-name: credhelper-test
aws-region: ${{ secrets.REGION }}
- name: Clean up previous files
if: always()
run: |
sudo systemctl stop finch.service || true
sudo systemctl stop finch-buildkit.service || true
sudo systemctl stop finch-soci.service || true
sudo sudo rpm -e runfinch-finch || true
sudo systemctl stop containerd.service
sudo systemctl daemon-reload
sudo rm -rf /etc/finch
sudo rm -rf /var/lib/finch
sudo rm -rf /var/lib/containerd
sudo rm -rf /var/soci-snapshotter
sudo rm -rf ./_output
- name: Build project
run: |
./contrib/packaging/rpm/build.sh --local
- name: Install Finch
run: |
sudo rpm -i ./_output/packages/$(ls -t ./_output/packages/ | grep runfinch-finch | head -1)
sudo systemctl daemon-reload
sudo systemctl start containerd.service
sudo systemctl start finch.service
sudo systemctl start finch-buildkit.service
sudo systemctl start finch-soci.service
- name: Run e2e tests
run: |
git status
git clean -f -d
# required by one of the tests which uses SSH_AUTH_SOCK
eval "$(ssh-agent -s)"
INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} sudo -E make ${{ inputs.test-command }}
- name: Clean up repo AL2
if: ${{ (startsWith(inputs.os, 'amazon') && inputs.version == '2' && always() ) }}
run: |
rm -rf "${GITHUB_WORKSPACE}"
Loading

0 comments on commit 985570e

Please sign in to comment.