From e5e06e840472447f6245c1861cde5a94baf561b6 Mon Sep 17 00:00:00 2001 From: Dmitrii Kuvaiskii Date: Wed, 9 Nov 2022 17:45:11 +0000 Subject: [PATCH] [CI] Add a VM-based Jenkins pipeline New Jenkins pipeline runs a Docker container that creates a minimal VM based on https://github.com/gramineproject/device-testing-tools repo and runs a subset of Gramine tests, in particular, the device IOCTL tests. The pipeline uses Ubuntu 22.04 with modern Linux kernel (to have an upstream SGX driver and support for SGX in KVM) and QEMU/KVM to run the VM. A LibOS regression test `device_ioctl` is added to test the CI; currently it is minimal but will be expanded when ioctl passthrough functionality is added to Gramine. Signed-off-by: Dmitrii Kuvaiskii --- .ci/lib/stage-build-sgx-vm.jenkinsfile | 92 +++++++++++++++++ .ci/lib/stage-clean-vm.jenkinsfile | 6 ++ .ci/lib/stage-test-vm.jenkinsfile | 15 +++ .ci/linux-sgx-vm-gcc-release.jenkinsfile | 48 +++++++++ .ci/ubuntu22.04.dockerfile | 99 +++++++++++++++++++ libos/test/regression/device_ioctl.c | 26 +++++ .../regression/device_ioctl.manifest.template | 18 ++++ libos/test/regression/meson.build | 7 ++ libos/test/regression/test_libos.py | 6 ++ libos/test/regression/tests.toml | 7 ++ libos/test/regression/tests_musl.toml | 6 ++ python/graminelibos/regression.py | 1 + python/graminelibos/util_tests.py | 15 ++- 13 files changed, 341 insertions(+), 5 deletions(-) create mode 100644 .ci/lib/stage-build-sgx-vm.jenkinsfile create mode 100644 .ci/lib/stage-clean-vm.jenkinsfile create mode 100644 .ci/lib/stage-test-vm.jenkinsfile create mode 100644 .ci/linux-sgx-vm-gcc-release.jenkinsfile create mode 100644 .ci/ubuntu22.04.dockerfile create mode 100644 libos/test/regression/device_ioctl.c create mode 100644 libos/test/regression/device_ioctl.manifest.template diff --git a/.ci/lib/stage-build-sgx-vm.jenkinsfile b/.ci/lib/stage-build-sgx-vm.jenkinsfile new file mode 100644 index 0000000000..90c2bbe239 --- /dev/null +++ b/.ci/lib/stage-build-sgx-vm.jenkinsfile @@ -0,0 +1,92 @@ +stage('build') { + sh ''' + # we add `/sbin` to PATH to find the `modprobe` program + export PATH="/sbin:$PATH" + + git clone https://github.com/gramineproject/device-testing-tools.git + cd device-testing-tools + + cd initramfs_builder + { + echo '#!/bin/sh' + echo 'if test -n $SGX; then GRAMINE=gramine-sgx; else GRAMINE=gramine-direct; fi' + echo 'cd $PWD_FOR_VM' + echo '( cd device-testing-tools/gramine-device-testing-module; insmod gramine-testing-dev.ko )' + + # only couple tests -- executing in a VM with virtio-9p-pci FS passthrough is very slow + echo 'cd libos/test/regression' + echo 'gramine-test build helloworld; $GRAMINE helloworld' + echo 'gramine-test build device_ioctl; $GRAMINE device_ioctl' + echo 'echo "TESTS OK"' + echo 'poweroff -n -f' + } > new_init + make ${MAKEOPTS} + + cd ../gramine-device-testing-module + make ${MAKEOPTS} + ''' + + env.MESON_OPTIONS = '' + if (env.UBSAN == '1') { + env.MESON_OPTIONS += ' -Dubsan=enabled' + } + if (env.ASAN == '1') { + env.MESON_OPTIONS += ' -Dasan=enabled' + } + if (env.CC == 'clang') { + env.MESON_OPTIONS += ' -Dmusl=disabled' + } + + try { + // copy gramine_test_dev_ioctl.h device header for `device_ioctl` LibOS test + sh ''' + cp -f device-testing-tools/gramine-device-testing-module/gramine_test_dev_ioctl.h \ + libos/test/regression/ + ''' + + sh ''' + meson setup build/ \ + --werror \ + --prefix="$PREFIX" \ + --buildtype="$BUILDTYPE" \ + -Ddirect=disabled \ + -Dsgx=enabled \ + -Dtests=enabled \ + -Dsgx_driver=upstream \ + $MESON_OPTIONS + ninja -vC build/ + ''' + + // install + sh ''' + ninja -vC build/ install + gramine-sgx-gen-private-key + ''' + } finally { + archiveArtifacts 'build/meson-logs/**/*' + archiveArtifacts 'build/subprojects/glibc-*/glibc-build.log' + } + + // archive all installed files + // NOTE we can't use ${env.PREFIX} here, because path needs to be relative to workdir + archiveArtifacts "usr/**/*" + + // Absolute path to libdir, as configured by Meson. + // For our current builds this should be "$WORKSPACE/usr/lib/x86_64-linux-gnu": + // --prefix is set from $PREFIX above (see config-docker.jenkinsfile) and should be "$WORKSPACE/usr"; + // --libdir is distro-dependent, but on Debian and derivatives it's "lib/x86_64-linux-gnu" + libdir = sh(returnStdout: true, script: ''' + meson introspect build/ --buildoptions \ + | jq -r '(map(select(.name == "prefix")) + map(select(.name == "libdir"))) | map(.value) | join("/")' + ''').trim() + + env.GRAMINE_PKGLIBDIR = libdir + '/gramine' + + // In CI we install to non-standard --prefix (see above). This makes sure the libraries are + // available anyway (e.g. gramine-sgx-pf-crypt needs libsgx_util.so). + env.PKG_CONFIG_PATH = libdir + '/pkgconfig' + + // prevent cheating and testing from repo + sh 'rm -rf build' + sh 'git clean -Xf subprojects' +} diff --git a/.ci/lib/stage-clean-vm.jenkinsfile b/.ci/lib/stage-clean-vm.jenkinsfile new file mode 100644 index 0000000000..a3cf826640 --- /dev/null +++ b/.ci/lib/stage-clean-vm.jenkinsfile @@ -0,0 +1,6 @@ +stage('clean-vm') { + sh ''' + rm -rf device-testing-tools driver + rm -rf libos/test/regression/gramine_test_dev_ioctl.h + ''' +} diff --git a/.ci/lib/stage-test-vm.jenkinsfile b/.ci/lib/stage-test-vm.jenkinsfile new file mode 100644 index 0000000000..543c4e343c --- /dev/null +++ b/.ci/lib/stage-test-vm.jenkinsfile @@ -0,0 +1,15 @@ +stage('test') { + timeout(time: 15, unit: 'MINUTES') { + sh ''' + export PWD_FOR_VM=$PWD + + cd device-testing-tools/initramfs_builder + + # we add `/sbin` to PATH to find insmod and poweroff programs + ./run.sh PWD_FOR_VM=$PWD_FOR_VM SGX=$SGX IS_VM=$IS_VM PATH=/sbin:$PATH \ + PKG_CONFIG_PATH=$PKG_CONFIG_PATH PYTHONPATH=$PYTHONPATH \ + XDG_CONFIG_HOME=$XDG_CONFIG_HOME GRAMINE_PKGLIBDIR=$GRAMINE_PKGLIBDIR | tee OUTPUT + grep "TESTS OK" OUTPUT + ''' + } +} diff --git a/.ci/linux-sgx-vm-gcc-release.jenkinsfile b/.ci/linux-sgx-vm-gcc-release.jenkinsfile new file mode 100644 index 0000000000..faf82993b7 --- /dev/null +++ b/.ci/linux-sgx-vm-gcc-release.jenkinsfile @@ -0,0 +1,48 @@ +node('whatnots') { + checkout scm + + env.SGX = '1' + env.IS_VM = '1' + + load '.ci/lib/config-docker.jenkinsfile' + + env.DOCKER_ARGS_SGX += ''' + --volume=/usr/include/x86_64-linux-gnu/asm/sgx.h:/usr/include/asm/sgx.h:ro + ''' + + // Overwrite Gramine-specific seccomp policy because it conflicts with KVM requirements, see + // https://github.com/moby/moby/issues/42963 for details. + env.DOCKER_ARGS_COMMON += + " --security-opt seccomp=${env.WORKSPACE}/scripts/docker_seccomp_aug_2022.json" + + // Required by QEMU to run the same Linux kernel in VM (because we use host kernel as guest + // kernel for simplicity) + env.DOCKER_ARGS_COMMON += ' --volume=/boot:/boot:ro' + + // only root and `kvm` group can access /dev/kvm, so add `kvm` GID to the in-Docker user + kvm_gid = sh(returnStdout: true, script: 'getent group kvm | cut -d: -f3').trim() + env.DOCKER_ARGS_COMMON += ' --group-add ' + kvm_gid + + env.DOCKER_ARGS_COMMON += ' --device=/dev/kvm:/dev/kvm' + + // only root and `sgx` group can access /dev/sgx_vepc, so add `sgx` GID to the in-Docker user + sgx_gid = sh(returnStdout: true, script: 'getent group sgx | cut -d: -f3').trim() + env.DOCKER_ARGS_SGX += ' --group-add ' + sgx_gid + + env.DOCKER_ARGS_SGX += ' --device=/dev/sgx_vepc:/dev/sgx_vepc' + + docker.build( + "local:${env.BUILD_TAG}", + '-f .ci/ubuntu22.04.dockerfile .' + ).inside("${env.DOCKER_ARGS_COMMON} ${env.DOCKER_ARGS_SGX}") { + load '.ci/lib/config.jenkinsfile' + load '.ci/lib/config-release.jenkinsfile' + + load '.ci/lib/stage-lint.jenkinsfile' + load '.ci/lib/stage-clean-check-prepare.jenkinsfile' + load '.ci/lib/stage-build-sgx-vm.jenkinsfile' + load '.ci/lib/stage-test-vm.jenkinsfile' + load '.ci/lib/stage-clean-vm.jenkinsfile' + load '.ci/lib/stage-clean-check.jenkinsfile' + } +} diff --git a/.ci/ubuntu22.04.dockerfile b/.ci/ubuntu22.04.dockerfile new file mode 100644 index 0000000000..8e09a0f3ab --- /dev/null +++ b/.ci/ubuntu22.04.dockerfile @@ -0,0 +1,99 @@ +FROM ubuntu:22.04 + +RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ + autoconf \ + bc \ + bison \ + build-essential \ + cargo \ + clang \ + curl \ + flex \ + gawk \ + gdb \ + gettext \ + git \ + jq \ + libapr1-dev \ + libaprutil1-dev \ + libcjson-dev \ + libelf-dev \ + libevent-dev \ + libexpat1 \ + libexpat1-dev \ + libmemcached-tools \ + libnss-mdns \ + libnuma1 \ + libomp-dev \ + libpcre2-dev \ + libpcre3-dev \ + libprotobuf-c-dev \ + libssl-dev \ + libunwind8 \ + libxfixes3 \ + libxi6 \ + libxml2-dev \ + libxrender1 \ + libxxf86vm1 \ + linux-headers-generic \ + musl \ + musl-tools \ + nasm \ + net-tools \ + netcat-openbsd \ + ninja-build \ + pkg-config \ + protobuf-c-compiler \ + protobuf-compiler \ + pylint \ + python3 \ + python3-apport \ + python3-apt \ + python3-breathe \ + python3-click \ + python3-cryptography \ + python3-jinja2 \ + python3-lxml \ + python3-numpy \ + python3-pip \ + python3-protobuf \ + python3-pyelftools \ + python3-pytest \ + python3-pytest-xdist \ + python3-scipy \ + python3-sphinx-rtd-theme \ + shellcheck \ + sphinx-doc \ + sqlite3 \ + texinfo \ + uthash-dev \ + wget \ + zlib1g \ + zlib1g-dev + +# NOTE about meson version: we support "0.56 or newer", so in CI we pin to latest patch version of +# the earliest supported minor version (pip implicitly installs latest version satisfying the +# specification) +RUN python3 -m pip install -U \ + 'tomli>=1.1.0' \ + 'tomli-w>=0.4.0' \ + 'meson>=0.56,<0.57' \ + 'recommonmark>=0.5.0,<=0.7.1' \ + 'docutils>=0.17,<0.18' + +# Dependencies required for building kernel modules and running VMs +RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ + cpio \ + dwarves \ + g++-10 \ + gcc-10 \ + kmod \ + qemu-kvm + +# Kernel on the host machine is built with GCC-10, so we need to set it as default in Docker +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10 && \ + update-alternatives --set gcc /usr/bin/gcc-10 && \ + update-alternatives --set g++ /usr/bin/g++-10 + +CMD ["bash"] diff --git a/libos/test/regression/device_ioctl.c b/libos/test/regression/device_ioctl.c new file mode 100644 index 0000000000..0725347311 --- /dev/null +++ b/libos/test/regression/device_ioctl.c @@ -0,0 +1,26 @@ +#define _GNU_SOURCE /* for loff_t */ +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "rw_file.h" + +#include "gramine_test_dev_ioctl.h" /* currently unused */ + +#define STRING_READWRITE "Hello world via read/write\n" + +int main(void) { + int devfd = CHECK(open("/dev/gramine_test_dev", O_RDWR)); + + ssize_t bytes = posix_fd_write(devfd, STRING_READWRITE, sizeof(STRING_READWRITE)); + if (bytes != sizeof(STRING_READWRITE)) + CHECK(-1); + + CHECK(close(devfd)); + puts("TEST OK"); + return 0; +} diff --git a/libos/test/regression/device_ioctl.manifest.template b/libos/test/regression/device_ioctl.manifest.template new file mode 100644 index 0000000000..0916505e9a --- /dev/null +++ b/libos/test/regression/device_ioctl.manifest.template @@ -0,0 +1,18 @@ +loader.entrypoint = "file:{{ gramine.libos }}" +libos.entrypoint = "{{ entrypoint }}" + +loader.env.LD_LIBRARY_PATH = "/lib" + +fs.mounts = [ + { path = "/lib", uri = "file:{{ gramine.runtimedir(libc) }}" }, + { path = "/{{ entrypoint }}", uri = "file:{{ binary_dir }}/{{ entrypoint }}" }, + { path = "/dev/gramine_test_dev", uri = "dev:/dev/gramine_test_dev" }, +] + +sgx.debug = true + +sgx.trusted_files = [ + "file:{{ gramine.libos }}", + "file:{{ gramine.runtimedir(libc) }}/", + "file:{{ binary_dir }}/{{ entrypoint }}", +] diff --git a/libos/test/regression/meson.build b/libos/test/regression/meson.build index 882fdc8b8d..4d6d53f963 100644 --- a/libos/test/regression/meson.build +++ b/libos/test/regression/meson.build @@ -1,3 +1,5 @@ +fs = import('fs') + tests = { 'abort': {}, 'abort_multithread': {}, @@ -149,6 +151,11 @@ if host_machine.cpu_family() == 'x86_64' } endif +# device_ioctl test may only be executed in a VM environment that prepares the below header file +if fs.exists('gramine_test_dev_ioctl.h') + tests += { 'device_ioctl': {} } +endif + tests_musl = tests if host_machine.cpu_family() == 'x86_64' diff --git a/libos/test/regression/test_libos.py b/libos/test/regression/test_libos.py index a61d281cd9..820a27b3f6 100644 --- a/libos/test/regression/test_libos.py +++ b/libos/test/regression/test_libos.py @@ -12,6 +12,7 @@ from graminelibos.regression import ( HAS_AVX, HAS_SGX, + IS_VM, ON_X86, USES_MUSL, RegressionTestCase, @@ -1032,6 +1033,11 @@ def test_002_device_passthrough(self): stdout, _ = self.run_binary(['device_passthrough']) self.assertIn('TEST OK', stdout) + @unittest.skipUnless(IS_VM, '/dev/gramine_test_dev is available only on some Jenkins machines') + def test_003_device_ioctl(self): + stdout, _ = self.run_binary(['device_ioctl']) + self.assertIn('TEST OK', stdout) + def test_010_path(self): stdout, _ = self.run_binary(['proc_path']) self.assertIn('proc path test success', stdout) diff --git a/libos/test/regression/tests.toml b/libos/test/regression/tests.toml index 6224531cc6..ec9c19f6d0 100644 --- a/libos/test/regression/tests.toml +++ b/libos/test/regression/tests.toml @@ -138,3 +138,10 @@ manifests = [ manifests = [ "attestation", ] + + +[vm] + +manifests = [ + "device_ioctl", +] diff --git a/libos/test/regression/tests_musl.toml b/libos/test/regression/tests_musl.toml index 8fe72b7300..5daea3f49f 100644 --- a/libos/test/regression/tests_musl.toml +++ b/libos/test/regression/tests_musl.toml @@ -131,3 +131,9 @@ manifests = [ "rdtsc", "sighandler_divbyzero", ] + +[vm] + +manifests = [ + "device_ioctl", +] diff --git a/python/graminelibos/regression.py b/python/graminelibos/regression.py index 1b12d12e40..5a966dbd6b 100644 --- a/python/graminelibos/regression.py +++ b/python/graminelibos/regression.py @@ -19,6 +19,7 @@ HAS_AVX = os.environ.get('AVX') == '1' HAS_EDMM = os.environ.get('EDMM') == '1' HAS_SGX = os.environ.get('SGX') == '1' +IS_VM = os.environ.get('IS_VM') == '1' ON_X86 = os.uname().machine in ['x86_64'] USES_MUSL = os.environ.get('GRAMINE_MUSL') == '1' diff --git a/python/graminelibos/util_tests.py b/python/graminelibos/util_tests.py index c108fc944a..4fd0ca0183 100644 --- a/python/graminelibos/util_tests.py +++ b/python/graminelibos/util_tests.py @@ -26,11 +26,12 @@ class TestConfig: `tests.toml` can have the following keys: - - `manifests`, `sgx.manifests`, `arch.[ARCH].manifests`: list of manifests to build - (for all hosts, SGX-only, [ARCH]-only) + - `manifests`, `sgx.manifests`, `vm.manifests`, `arch.[ARCH].manifests`: list of manifests to + build (for all hosts, SGX-only, VM-only (for device testing), [ARCH]-only) - - `manifests_cmd` (and same with `sgx.` and `arch.[ARCH].`): a shell command that prints out - manifests to build, in separate lines (used by LTP, where the list depends on enabled tests) + - `manifests_cmd` (and same with `sgx.`, `vm` and `arch.[ARCH].`): a shell command that prints + out manifests to build, in separate lines (used by LTP, where the list depends on enabled + tests) - `binary_dir`: path to test binaries, passed as `binary_dir` to manifest templates; expands @GRAMINE_PKGLIBDIR@ to library directory of Gramine's installation @@ -57,6 +58,10 @@ def __init__(self, path): self.sgx_manifests = self.get_manifests(data.get('sgx', {})) + self.vm_manifests = [] + if os.environ.get('IS_VM') == '1': + self.vm_manifests = self.get_manifests(data.get('vm', {})) + self.binary_dir = data.get('binary_dir', '.').replace( '@GRAMINE_PKGLIBDIR@', _CONFIG_PKGLIBDIR) @@ -79,7 +84,7 @@ def __init__(self, path): if not self.key: self.key = os.fspath(_SGX_RSA_KEY_PATH) - self.all_manifests = self.manifests + self.sgx_manifests + self.all_manifests = self.manifests + self.sgx_manifests + self.vm_manifests @staticmethod def get_manifests(data):