diff --git a/.ci/lib/stage-build-sgx-vm.jenkinsfile b/.ci/lib/stage-build-sgx-vm.jenkinsfile new file mode 100644 index 0000000000..17c281dd9a --- /dev/null +++ b/.ci/lib/stage-build-sgx-vm.jenkinsfile @@ -0,0 +1,93 @@ +stage('build') { + sh ''' + # we add `/sbin` to PATH to find the `modprobe` program + export PATH="/sbin:$PATH" + + git clone https://github.com/gramineproject/device-testing-tools.git + cd device-testing-tools + git checkout 096436b20eb5c314b6180e63eb6e6d4e7f31d1c8 # TODO: use `master` after PR merged + + cd initramfs_builder + { + echo '#!/bin/sh' + echo 'if test -n $SGX; then GRAMINE=gramine-sgx; else GRAMINE=gramine-direct; fi' + echo 'cd $PWD_FOR_VM' + echo '( cd device-testing-tools/gramine-device-testing-module; insmod gramine-testing-dev.ko )' + + # only couple tests -- executing in a VM with virtio-9p-pci FS passthrough is very slow + echo 'cd libos/test/regression' + echo 'gramine-test build helloworld; $GRAMINE helloworld' + echo 'gramine-test build device_ioctl; $GRAMINE device_ioctl' + echo 'echo "TESTS OK"' + echo 'poweroff -n -f' + } > new_init + make ${MAKEOPTS} + + cd ../gramine-device-testing-module + make ${MAKEOPTS} + ''' + + env.MESON_OPTIONS = '' + if (env.UBSAN == '1') { + env.MESON_OPTIONS += ' -Dubsan=enabled' + } + if (env.ASAN == '1') { + env.MESON_OPTIONS += ' -Dasan=enabled' + } + if (env.CC == 'clang') { + env.MESON_OPTIONS += ' -Dmusl=disabled' + } + + try { + // copy gramine_test_dev_ioctl.h device header for `device_ioctl` LibOS test + sh ''' + cp -f device-testing-tools/gramine-device-testing-module/gramine_test_dev_ioctl.h \ + libos/test/regression/ + ''' + + sh ''' + meson setup build/ \ + --werror \ + --prefix="$PREFIX" \ + --buildtype="$BUILDTYPE" \ + -Ddirect=disabled \ + -Dsgx=enabled \ + -Dtests=enabled \ + -Dsgx_driver=upstream \ + $MESON_OPTIONS + ninja -vC build/ + ''' + + // install + sh ''' + ninja -vC build/ install + gramine-sgx-gen-private-key + ''' + } finally { + archiveArtifacts 'build/meson-logs/**/*' + archiveArtifacts 'build/subprojects/glibc-*/glibc-build.log' + } + + // archive all installed files + // NOTE we can't use ${env.PREFIX} here, because path needs to be relative to workdir + archiveArtifacts "usr/**/*" + + // Absolute path to libdir, as configured by Meson. + // For our current builds this should be "$WORKSPACE/usr/lib/x86_64-linux-gnu": + // --prefix is set from $PREFIX above (see config-docker.jenkinsfile) and should be "$WORKSPACE/usr"; + // --libdir is distro-dependent, but on Debian and derivatives it's "lib/x86_64-linux-gnu" + libdir = sh(returnStdout: true, script: ''' + meson introspect build/ --buildoptions \ + | jq -r '(map(select(.name == "prefix")) + map(select(.name == "libdir"))) | map(.value) | join("/")' + ''').trim() + + env.GRAMINE_PKGLIBDIR = libdir + '/gramine' + + // In CI we install to non-standard --prefix (see above). This makes sure the libraries are + // available anyway (e.g. gramine-sgx-pf-crypt needs libsgx_util.so). + env.PKG_CONFIG_PATH = libdir + '/pkgconfig' + + // prevent cheating and testing from repo + sh 'rm -rf build' + sh 'git clean -Xf subprojects' +} diff --git a/.ci/lib/stage-clean-vm.jenkinsfile b/.ci/lib/stage-clean-vm.jenkinsfile new file mode 100644 index 0000000000..a3cf826640 --- /dev/null +++ b/.ci/lib/stage-clean-vm.jenkinsfile @@ -0,0 +1,6 @@ +stage('clean-vm') { + sh ''' + rm -rf device-testing-tools driver + rm -rf libos/test/regression/gramine_test_dev_ioctl.h + ''' +} diff --git a/.ci/lib/stage-test-vm.jenkinsfile b/.ci/lib/stage-test-vm.jenkinsfile new file mode 100644 index 0000000000..543c4e343c --- /dev/null +++ b/.ci/lib/stage-test-vm.jenkinsfile @@ -0,0 +1,15 @@ +stage('test') { + timeout(time: 15, unit: 'MINUTES') { + sh ''' + export PWD_FOR_VM=$PWD + + cd device-testing-tools/initramfs_builder + + # we add `/sbin` to PATH to find insmod and poweroff programs + ./run.sh PWD_FOR_VM=$PWD_FOR_VM SGX=$SGX IS_VM=$IS_VM PATH=/sbin:$PATH \ + PKG_CONFIG_PATH=$PKG_CONFIG_PATH PYTHONPATH=$PYTHONPATH \ + XDG_CONFIG_HOME=$XDG_CONFIG_HOME GRAMINE_PKGLIBDIR=$GRAMINE_PKGLIBDIR | tee OUTPUT + grep "TESTS OK" OUTPUT + ''' + } +} diff --git a/.ci/linux-sgx-vm-gcc-release.jenkinsfile b/.ci/linux-sgx-vm-gcc-release.jenkinsfile new file mode 100644 index 0000000000..faf82993b7 --- /dev/null +++ b/.ci/linux-sgx-vm-gcc-release.jenkinsfile @@ -0,0 +1,48 @@ +node('whatnots') { + checkout scm + + env.SGX = '1' + env.IS_VM = '1' + + load '.ci/lib/config-docker.jenkinsfile' + + env.DOCKER_ARGS_SGX += ''' + --volume=/usr/include/x86_64-linux-gnu/asm/sgx.h:/usr/include/asm/sgx.h:ro + ''' + + // Overwrite Gramine-specific seccomp policy because it conflicts with KVM requirements, see + // https://github.com/moby/moby/issues/42963 for details. + env.DOCKER_ARGS_COMMON += + " --security-opt seccomp=${env.WORKSPACE}/scripts/docker_seccomp_aug_2022.json" + + // Required by QEMU to run the same Linux kernel in VM (because we use host kernel as guest + // kernel for simplicity) + env.DOCKER_ARGS_COMMON += ' --volume=/boot:/boot:ro' + + // only root and `kvm` group can access /dev/kvm, so add `kvm` GID to the in-Docker user + kvm_gid = sh(returnStdout: true, script: 'getent group kvm | cut -d: -f3').trim() + env.DOCKER_ARGS_COMMON += ' --group-add ' + kvm_gid + + env.DOCKER_ARGS_COMMON += ' --device=/dev/kvm:/dev/kvm' + + // only root and `sgx` group can access /dev/sgx_vepc, so add `sgx` GID to the in-Docker user + sgx_gid = sh(returnStdout: true, script: 'getent group sgx | cut -d: -f3').trim() + env.DOCKER_ARGS_SGX += ' --group-add ' + sgx_gid + + env.DOCKER_ARGS_SGX += ' --device=/dev/sgx_vepc:/dev/sgx_vepc' + + docker.build( + "local:${env.BUILD_TAG}", + '-f .ci/ubuntu22.04.dockerfile .' + ).inside("${env.DOCKER_ARGS_COMMON} ${env.DOCKER_ARGS_SGX}") { + load '.ci/lib/config.jenkinsfile' + load '.ci/lib/config-release.jenkinsfile' + + load '.ci/lib/stage-lint.jenkinsfile' + load '.ci/lib/stage-clean-check-prepare.jenkinsfile' + load '.ci/lib/stage-build-sgx-vm.jenkinsfile' + load '.ci/lib/stage-test-vm.jenkinsfile' + load '.ci/lib/stage-clean-vm.jenkinsfile' + load '.ci/lib/stage-clean-check.jenkinsfile' + } +} diff --git a/.ci/ubuntu22.04.dockerfile b/.ci/ubuntu22.04.dockerfile new file mode 100644 index 0000000000..8e09a0f3ab --- /dev/null +++ b/.ci/ubuntu22.04.dockerfile @@ -0,0 +1,99 @@ +FROM ubuntu:22.04 + +RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ + autoconf \ + bc \ + bison \ + build-essential \ + cargo \ + clang \ + curl \ + flex \ + gawk \ + gdb \ + gettext \ + git \ + jq \ + libapr1-dev \ + libaprutil1-dev \ + libcjson-dev \ + libelf-dev \ + libevent-dev \ + libexpat1 \ + libexpat1-dev \ + libmemcached-tools \ + libnss-mdns \ + libnuma1 \ + libomp-dev \ + libpcre2-dev \ + libpcre3-dev \ + libprotobuf-c-dev \ + libssl-dev \ + libunwind8 \ + libxfixes3 \ + libxi6 \ + libxml2-dev \ + libxrender1 \ + libxxf86vm1 \ + linux-headers-generic \ + musl \ + musl-tools \ + nasm \ + net-tools \ + netcat-openbsd \ + ninja-build \ + pkg-config \ + protobuf-c-compiler \ + protobuf-compiler \ + pylint \ + python3 \ + python3-apport \ + python3-apt \ + python3-breathe \ + python3-click \ + python3-cryptography \ + python3-jinja2 \ + python3-lxml \ + python3-numpy \ + python3-pip \ + python3-protobuf \ + python3-pyelftools \ + python3-pytest \ + python3-pytest-xdist \ + python3-scipy \ + python3-sphinx-rtd-theme \ + shellcheck \ + sphinx-doc \ + sqlite3 \ + texinfo \ + uthash-dev \ + wget \ + zlib1g \ + zlib1g-dev + +# NOTE about meson version: we support "0.56 or newer", so in CI we pin to latest patch version of +# the earliest supported minor version (pip implicitly installs latest version satisfying the +# specification) +RUN python3 -m pip install -U \ + 'tomli>=1.1.0' \ + 'tomli-w>=0.4.0' \ + 'meson>=0.56,<0.57' \ + 'recommonmark>=0.5.0,<=0.7.1' \ + 'docutils>=0.17,<0.18' + +# Dependencies required for building kernel modules and running VMs +RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ + cpio \ + dwarves \ + g++-10 \ + gcc-10 \ + kmod \ + qemu-kvm + +# Kernel on the host machine is built with GCC-10, so we need to set it as default in Docker +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10 && \ + update-alternatives --set gcc /usr/bin/gcc-10 && \ + update-alternatives --set g++ /usr/bin/g++-10 + +CMD ["bash"] diff --git a/libos/test/regression/device_ioctl.c b/libos/test/regression/device_ioctl.c new file mode 100644 index 0000000000..8ce7582d19 --- /dev/null +++ b/libos/test/regression/device_ioctl.c @@ -0,0 +1,26 @@ +#define _GNU_SOURCE /* for loff_t */ +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "rw_file.h" + +#include "gramine_test_dev_ioctl.h" /* currently unused */ + +#define STRING_READWRITE "Hello world via read/write\n" + +int main(void) { + int devfd = CHECK(open("/dev/gramine_test_dev", O_RDWR)); + + ssize_t bytes = posix_fd_write(devfd, STRING_READWRITE, sizeof(STRING_READWRITE)); + if (bytes != sizeof(STRING_READWRITE)) + CHECK(-1); + + CHECK(close(devfd)); + puts("TEST OK"); + return 0; +} diff --git a/libos/test/regression/device_ioctl.manifest.template b/libos/test/regression/device_ioctl.manifest.template new file mode 100644 index 0000000000..0916505e9a --- /dev/null +++ b/libos/test/regression/device_ioctl.manifest.template @@ -0,0 +1,18 @@ +loader.entrypoint = "file:{{ gramine.libos }}" +libos.entrypoint = "{{ entrypoint }}" + +loader.env.LD_LIBRARY_PATH = "/lib" + +fs.mounts = [ + { path = "/lib", uri = "file:{{ gramine.runtimedir(libc) }}" }, + { path = "/{{ entrypoint }}", uri = "file:{{ binary_dir }}/{{ entrypoint }}" }, + { path = "/dev/gramine_test_dev", uri = "dev:/dev/gramine_test_dev" }, +] + +sgx.debug = true + +sgx.trusted_files = [ + "file:{{ gramine.libos }}", + "file:{{ gramine.runtimedir(libc) }}/", + "file:{{ binary_dir }}/{{ entrypoint }}", +] diff --git a/libos/test/regression/meson.build b/libos/test/regression/meson.build index 52e9848df6..d3229e8201 100644 --- a/libos/test/regression/meson.build +++ b/libos/test/regression/meson.build @@ -1,3 +1,5 @@ +fs = import('fs') + tests = { 'abort': {}, 'abort_multithread': {}, @@ -148,6 +150,11 @@ if host_machine.cpu_family() == 'x86_64' } endif +# device_ioctl test may only be executed in a VM environment that prepares the below header file +if fs.exists('gramine_test_dev_ioctl.h') + tests += { 'device_ioctl': {} } +endif + tests_musl = tests if host_machine.cpu_family() == 'x86_64' diff --git a/libos/test/regression/test_libos.py b/libos/test/regression/test_libos.py index c3cbe843f9..006f488e93 100644 --- a/libos/test/regression/test_libos.py +++ b/libos/test/regression/test_libos.py @@ -12,6 +12,7 @@ from graminelibos.regression import ( HAS_AVX, HAS_SGX, + IS_VM, ON_X86, USES_MUSL, RegressionTestCase, @@ -1024,6 +1025,11 @@ def test_002_device_passthrough(self): stdout, _ = self.run_binary(['device_passthrough']) self.assertIn('TEST OK', stdout) + @unittest.skipUnless(IS_VM, '/dev/gramine_test_dev is available only on some Jenkins machines') + def test_003_device_ioctl(self): + stdout, _ = self.run_binary(['device_ioctl']) + self.assertIn('TEST OK', stdout) + def test_010_path(self): stdout, _ = self.run_binary(['proc_path']) self.assertIn('proc path test success', stdout) diff --git a/libos/test/regression/tests.toml b/libos/test/regression/tests.toml index f5355288b5..1d026c7a38 100644 --- a/libos/test/regression/tests.toml +++ b/libos/test/regression/tests.toml @@ -137,3 +137,10 @@ manifests = [ manifests = [ "attestation", ] + + +[vm] + +manifests = [ + "device_ioctl", +] diff --git a/libos/test/regression/tests_musl.toml b/libos/test/regression/tests_musl.toml index b4705567e5..b7d5039bd5 100644 --- a/libos/test/regression/tests_musl.toml +++ b/libos/test/regression/tests_musl.toml @@ -130,3 +130,9 @@ manifests = [ "rdtsc", "sighandler_divbyzero", ] + +[vm] + +manifests = [ + "device_ioctl", +] diff --git a/python/graminelibos/regression.py b/python/graminelibos/regression.py index 1b12d12e40..5a966dbd6b 100644 --- a/python/graminelibos/regression.py +++ b/python/graminelibos/regression.py @@ -19,6 +19,7 @@ HAS_AVX = os.environ.get('AVX') == '1' HAS_EDMM = os.environ.get('EDMM') == '1' HAS_SGX = os.environ.get('SGX') == '1' +IS_VM = os.environ.get('IS_VM') == '1' ON_X86 = os.uname().machine in ['x86_64'] USES_MUSL = os.environ.get('GRAMINE_MUSL') == '1' diff --git a/python/graminelibos/util_tests.py b/python/graminelibos/util_tests.py index c108fc944a..4fd0ca0183 100644 --- a/python/graminelibos/util_tests.py +++ b/python/graminelibos/util_tests.py @@ -26,11 +26,12 @@ class TestConfig: `tests.toml` can have the following keys: - - `manifests`, `sgx.manifests`, `arch.[ARCH].manifests`: list of manifests to build - (for all hosts, SGX-only, [ARCH]-only) + - `manifests`, `sgx.manifests`, `vm.manifests`, `arch.[ARCH].manifests`: list of manifests to + build (for all hosts, SGX-only, VM-only (for device testing), [ARCH]-only) - - `manifests_cmd` (and same with `sgx.` and `arch.[ARCH].`): a shell command that prints out - manifests to build, in separate lines (used by LTP, where the list depends on enabled tests) + - `manifests_cmd` (and same with `sgx.`, `vm` and `arch.[ARCH].`): a shell command that prints + out manifests to build, in separate lines (used by LTP, where the list depends on enabled + tests) - `binary_dir`: path to test binaries, passed as `binary_dir` to manifest templates; expands @GRAMINE_PKGLIBDIR@ to library directory of Gramine's installation @@ -57,6 +58,10 @@ def __init__(self, path): self.sgx_manifests = self.get_manifests(data.get('sgx', {})) + self.vm_manifests = [] + if os.environ.get('IS_VM') == '1': + self.vm_manifests = self.get_manifests(data.get('vm', {})) + self.binary_dir = data.get('binary_dir', '.').replace( '@GRAMINE_PKGLIBDIR@', _CONFIG_PKGLIBDIR) @@ -79,7 +84,7 @@ def __init__(self, path): if not self.key: self.key = os.fspath(_SGX_RSA_KEY_PATH) - self.all_manifests = self.manifests + self.sgx_manifests + self.all_manifests = self.manifests + self.sgx_manifests + self.vm_manifests @staticmethod def get_manifests(data):