-
Notifications
You must be signed in to change notification settings - Fork 193
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[CI] Add a VM-based Jenkins pipeline
New Jenkins pipeline runs a Docker container that creates a minimal VM based on https://github.com/gramineproject/device-testing-tools repo and runs a subset of Gramine tests, in particular, the device IOCTL tests. The pipeline uses Ubuntu 22.04 with modern Linux kernel (to have an upstream SGX driver and support for SGX in KVM) and QEMU/KVM to run the VM. A LibOS regression test `device_ioctl` is added to test the CI; currently it is minimal but will be expanded when ioctl passthrough functionality is added to Gramine. Signed-off-by: Dmitrii Kuvaiskii <[email protected]>
- Loading branch information
Showing
13 changed files
with
341 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,92 @@ | ||
stage('build') { | ||
sh ''' | ||
# we add `/sbin` to PATH to find the `modprobe` program | ||
export PATH="/sbin:$PATH" | ||
|
||
git clone https://github.com/gramineproject/device-testing-tools.git | ||
cd device-testing-tools | ||
|
||
cd initramfs_builder | ||
{ | ||
echo '#!/bin/sh' | ||
echo 'if test -n $SGX; then GRAMINE=gramine-sgx; else GRAMINE=gramine-direct; fi' | ||
echo 'cd $PWD_FOR_VM' | ||
echo '( cd device-testing-tools/gramine-device-testing-module; insmod gramine-testing-dev.ko )' | ||
|
||
# only couple tests -- executing in a VM with virtio-9p-pci FS passthrough is very slow | ||
echo 'cd libos/test/regression' | ||
echo 'gramine-test build helloworld; $GRAMINE helloworld' | ||
echo 'gramine-test build device_ioctl; $GRAMINE device_ioctl' | ||
echo 'echo "TESTS OK"' | ||
echo 'poweroff -n -f' | ||
} > new_init | ||
make ${MAKEOPTS} | ||
|
||
cd ../gramine-device-testing-module | ||
make ${MAKEOPTS} | ||
''' | ||
|
||
env.MESON_OPTIONS = '' | ||
if (env.UBSAN == '1') { | ||
env.MESON_OPTIONS += ' -Dubsan=enabled' | ||
} | ||
if (env.ASAN == '1') { | ||
env.MESON_OPTIONS += ' -Dasan=enabled' | ||
} | ||
if (env.CC == 'clang') { | ||
env.MESON_OPTIONS += ' -Dmusl=disabled' | ||
} | ||
|
||
try { | ||
// copy gramine_test_dev_ioctl.h device header for `device_ioctl` LibOS test | ||
sh ''' | ||
cp -f device-testing-tools/gramine-device-testing-module/gramine_test_dev_ioctl.h \ | ||
libos/test/regression/ | ||
''' | ||
|
||
sh ''' | ||
meson setup build/ \ | ||
--werror \ | ||
--prefix="$PREFIX" \ | ||
--buildtype="$BUILDTYPE" \ | ||
-Ddirect=disabled \ | ||
-Dsgx=enabled \ | ||
-Dtests=enabled \ | ||
-Dsgx_driver=upstream \ | ||
$MESON_OPTIONS | ||
ninja -vC build/ | ||
''' | ||
|
||
// install | ||
sh ''' | ||
ninja -vC build/ install | ||
gramine-sgx-gen-private-key | ||
''' | ||
} finally { | ||
archiveArtifacts 'build/meson-logs/**/*' | ||
archiveArtifacts 'build/subprojects/glibc-*/glibc-build.log' | ||
} | ||
|
||
// archive all installed files | ||
// NOTE we can't use ${env.PREFIX} here, because path needs to be relative to workdir | ||
archiveArtifacts "usr/**/*" | ||
|
||
// Absolute path to libdir, as configured by Meson. | ||
// For our current builds this should be "$WORKSPACE/usr/lib/x86_64-linux-gnu": | ||
// --prefix is set from $PREFIX above (see config-docker.jenkinsfile) and should be "$WORKSPACE/usr"; | ||
// --libdir is distro-dependent, but on Debian and derivatives it's "lib/x86_64-linux-gnu" | ||
libdir = sh(returnStdout: true, script: ''' | ||
meson introspect build/ --buildoptions \ | ||
| jq -r '(map(select(.name == "prefix")) + map(select(.name == "libdir"))) | map(.value) | join("/")' | ||
''').trim() | ||
|
||
env.GRAMINE_PKGLIBDIR = libdir + '/gramine' | ||
|
||
// In CI we install to non-standard --prefix (see above). This makes sure the libraries are | ||
// available anyway (e.g. gramine-sgx-pf-crypt needs libsgx_util.so). | ||
env.PKG_CONFIG_PATH = libdir + '/pkgconfig' | ||
|
||
// prevent cheating and testing from repo | ||
sh 'rm -rf build' | ||
sh 'git clean -Xf subprojects' | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
stage('clean-vm') { | ||
sh ''' | ||
rm -rf device-testing-tools driver | ||
rm -rf libos/test/regression/gramine_test_dev_ioctl.h | ||
''' | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
stage('test') { | ||
timeout(time: 15, unit: 'MINUTES') { | ||
sh ''' | ||
export PWD_FOR_VM=$PWD | ||
|
||
cd device-testing-tools/initramfs_builder | ||
|
||
# we add `/sbin` to PATH to find insmod and poweroff programs | ||
./run.sh PWD_FOR_VM=$PWD_FOR_VM SGX=$SGX IS_VM=$IS_VM PATH=/sbin:$PATH \ | ||
PKG_CONFIG_PATH=$PKG_CONFIG_PATH PYTHONPATH=$PYTHONPATH \ | ||
XDG_CONFIG_HOME=$XDG_CONFIG_HOME GRAMINE_PKGLIBDIR=$GRAMINE_PKGLIBDIR | tee OUTPUT | ||
grep "TESTS OK" OUTPUT | ||
''' | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
node('whatnots') { | ||
checkout scm | ||
|
||
env.SGX = '1' | ||
env.IS_VM = '1' | ||
|
||
load '.ci/lib/config-docker.jenkinsfile' | ||
|
||
env.DOCKER_ARGS_SGX += ''' | ||
--volume=/usr/include/x86_64-linux-gnu/asm/sgx.h:/usr/include/asm/sgx.h:ro | ||
''' | ||
|
||
// Overwrite Gramine-specific seccomp policy because it conflicts with KVM requirements, see | ||
// https://github.com/moby/moby/issues/42963 for details. | ||
env.DOCKER_ARGS_COMMON += | ||
" --security-opt seccomp=${env.WORKSPACE}/scripts/docker_seccomp_aug_2022.json" | ||
|
||
// Required by QEMU to run the same Linux kernel in VM (because we use host kernel as guest | ||
// kernel for simplicity) | ||
env.DOCKER_ARGS_COMMON += ' --volume=/boot:/boot:ro' | ||
|
||
// only root and `kvm` group can access /dev/kvm, so add `kvm` GID to the in-Docker user | ||
kvm_gid = sh(returnStdout: true, script: 'getent group kvm | cut -d: -f3').trim() | ||
env.DOCKER_ARGS_COMMON += ' --group-add ' + kvm_gid | ||
|
||
env.DOCKER_ARGS_COMMON += ' --device=/dev/kvm:/dev/kvm' | ||
|
||
// only root and `sgx` group can access /dev/sgx_vepc, so add `sgx` GID to the in-Docker user | ||
sgx_gid = sh(returnStdout: true, script: 'getent group sgx | cut -d: -f3').trim() | ||
env.DOCKER_ARGS_SGX += ' --group-add ' + sgx_gid | ||
|
||
env.DOCKER_ARGS_SGX += ' --device=/dev/sgx_vepc:/dev/sgx_vepc' | ||
|
||
docker.build( | ||
"local:${env.BUILD_TAG}", | ||
'-f .ci/ubuntu22.04.dockerfile .' | ||
).inside("${env.DOCKER_ARGS_COMMON} ${env.DOCKER_ARGS_SGX}") { | ||
load '.ci/lib/config.jenkinsfile' | ||
load '.ci/lib/config-release.jenkinsfile' | ||
|
||
load '.ci/lib/stage-lint.jenkinsfile' | ||
load '.ci/lib/stage-clean-check-prepare.jenkinsfile' | ||
load '.ci/lib/stage-build-sgx-vm.jenkinsfile' | ||
load '.ci/lib/stage-test-vm.jenkinsfile' | ||
load '.ci/lib/stage-clean-vm.jenkinsfile' | ||
load '.ci/lib/stage-clean-check.jenkinsfile' | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,99 @@ | ||
FROM ubuntu:22.04 | ||
|
||
RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ | ||
autoconf \ | ||
bc \ | ||
bison \ | ||
build-essential \ | ||
cargo \ | ||
clang \ | ||
curl \ | ||
flex \ | ||
gawk \ | ||
gdb \ | ||
gettext \ | ||
git \ | ||
jq \ | ||
libapr1-dev \ | ||
libaprutil1-dev \ | ||
libcjson-dev \ | ||
libelf-dev \ | ||
libevent-dev \ | ||
libexpat1 \ | ||
libexpat1-dev \ | ||
libmemcached-tools \ | ||
libnss-mdns \ | ||
libnuma1 \ | ||
libomp-dev \ | ||
libpcre2-dev \ | ||
libpcre3-dev \ | ||
libprotobuf-c-dev \ | ||
libssl-dev \ | ||
libunwind8 \ | ||
libxfixes3 \ | ||
libxi6 \ | ||
libxml2-dev \ | ||
libxrender1 \ | ||
libxxf86vm1 \ | ||
linux-headers-generic \ | ||
musl \ | ||
musl-tools \ | ||
nasm \ | ||
net-tools \ | ||
netcat-openbsd \ | ||
ninja-build \ | ||
pkg-config \ | ||
protobuf-c-compiler \ | ||
protobuf-compiler \ | ||
pylint \ | ||
python3 \ | ||
python3-apport \ | ||
python3-apt \ | ||
python3-breathe \ | ||
python3-click \ | ||
python3-cryptography \ | ||
python3-jinja2 \ | ||
python3-lxml \ | ||
python3-numpy \ | ||
python3-pip \ | ||
python3-protobuf \ | ||
python3-pyelftools \ | ||
python3-pytest \ | ||
python3-pytest-xdist \ | ||
python3-scipy \ | ||
python3-sphinx-rtd-theme \ | ||
shellcheck \ | ||
sphinx-doc \ | ||
sqlite3 \ | ||
texinfo \ | ||
uthash-dev \ | ||
wget \ | ||
zlib1g \ | ||
zlib1g-dev | ||
|
||
# NOTE about meson version: we support "0.56 or newer", so in CI we pin to latest patch version of | ||
# the earliest supported minor version (pip implicitly installs latest version satisfying the | ||
# specification) | ||
RUN python3 -m pip install -U \ | ||
'tomli>=1.1.0' \ | ||
'tomli-w>=0.4.0' \ | ||
'meson>=0.56,<0.57' \ | ||
'recommonmark>=0.5.0,<=0.7.1' \ | ||
'docutils>=0.17,<0.18' | ||
|
||
# Dependencies required for building kernel modules and running VMs | ||
RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ | ||
cpio \ | ||
dwarves \ | ||
g++-10 \ | ||
gcc-10 \ | ||
kmod \ | ||
qemu-kvm | ||
|
||
# Kernel on the host machine is built with GCC-10, so we need to set it as default in Docker | ||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 && \ | ||
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10 && \ | ||
update-alternatives --set gcc /usr/bin/gcc-10 && \ | ||
update-alternatives --set g++ /usr/bin/g++-10 | ||
|
||
CMD ["bash"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
#define _GNU_SOURCE /* for loff_t */ | ||
#include <err.h> | ||
#include <fcntl.h> | ||
#include <stdint.h> | ||
#include <stdio.h> | ||
#include <string.h> | ||
#include <unistd.h> | ||
|
||
#include "common.h" | ||
#include "rw_file.h" | ||
|
||
#include "gramine_test_dev_ioctl.h" /* currently unused */ | ||
|
||
#define STRING_READWRITE "Hello world via read/write\n" | ||
|
||
int main(void) { | ||
int devfd = CHECK(open("/dev/gramine_test_dev", O_RDWR)); | ||
|
||
ssize_t bytes = posix_fd_write(devfd, STRING_READWRITE, sizeof(STRING_READWRITE)); | ||
if (bytes != sizeof(STRING_READWRITE)) | ||
CHECK(-1); | ||
|
||
CHECK(close(devfd)); | ||
puts("TEST OK"); | ||
return 0; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
loader.entrypoint = "file:{{ gramine.libos }}" | ||
libos.entrypoint = "{{ entrypoint }}" | ||
|
||
loader.env.LD_LIBRARY_PATH = "/lib" | ||
|
||
fs.mounts = [ | ||
{ path = "/lib", uri = "file:{{ gramine.runtimedir(libc) }}" }, | ||
{ path = "/{{ entrypoint }}", uri = "file:{{ binary_dir }}/{{ entrypoint }}" }, | ||
{ path = "/dev/gramine_test_dev", uri = "dev:/dev/gramine_test_dev" }, | ||
] | ||
|
||
sgx.debug = true | ||
|
||
sgx.trusted_files = [ | ||
"file:{{ gramine.libos }}", | ||
"file:{{ gramine.runtimedir(libc) }}/", | ||
"file:{{ binary_dir }}/{{ entrypoint }}", | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -138,3 +138,10 @@ manifests = [ | |
manifests = [ | ||
"attestation", | ||
] | ||
|
||
|
||
[vm] | ||
|
||
manifests = [ | ||
"device_ioctl", | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -131,3 +131,9 @@ manifests = [ | |
"rdtsc", | ||
"sighandler_divbyzero", | ||
] | ||
|
||
[vm] | ||
|
||
manifests = [ | ||
"device_ioctl", | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.