[CI] Add a VM-based Jenkins pipeline

New Jenkins pipeline runs a Docker container that creates a minimal VM
based on https://github.com/gramineproject/device-testing-tools repo and
runs a subset of Gramine tests, in particular, the device IOCTL tests.
The pipeline uses Ubuntu 22.04 with modern Linux kernel (to have an
upstream SGX driver and support for SGX in KVM) and QEMU/KVM to run the
VM.

A LibOS regression test `device_ioctl` is added to test the CI;
currently it is minimal but will be expanded when ioctl passthrough
functionality is added to Gramine.

Signed-off-by: Dmitrii Kuvaiskii <[email protected]>

.ci/lib/stage-build-sgx-vm.jenkinsfile

@@ -0,0 +1,93 @@
+stage('build') {
+    sh '''
+        # we add `/sbin` to PATH to find the `modprobe` program
+        export PATH="/sbin:$PATH"
+
+        git clone https://github.com/gramineproject/device-testing-tools.git
+        cd device-testing-tools
+        git checkout 096436b20eb5c314b6180e63eb6e6d4e7f31d1c8 # TODO: use `master` after PR merged
+
+        cd initramfs_builder
+        {
+           echo '#!/bin/sh'
+           echo 'if test -n $SGX; then GRAMINE=gramine-sgx; else GRAMINE=gramine-direct; fi'
+           echo 'cd $PWD_FOR_VM'
+           echo '( cd device-testing-tools/gramine-device-testing-module; insmod gramine-testing-dev.ko )'
+
+           # only couple tests -- executing in a VM with virtio-9p-pci FS passthrough is very slow
+           echo 'cd libos/test/regression'
+           echo 'gramine-test build helloworld; $GRAMINE helloworld'
+           echo 'gramine-test build device_ioctl; $GRAMINE device_ioctl'
+           echo 'echo "TESTS OK"'
+           echo 'poweroff -n -f'
+        } > new_init
+        make ${MAKEOPTS}
+
+        cd ../gramine-device-testing-module
+        make ${MAKEOPTS}
+    '''
+
+    env.MESON_OPTIONS = ''
+    if (env.UBSAN == '1') {
+        env.MESON_OPTIONS += ' -Dubsan=enabled'
+    }
+    if (env.ASAN == '1') {
+        env.MESON_OPTIONS += ' -Dasan=enabled'
+    }
+    if (env.CC == 'clang') {
+        env.MESON_OPTIONS += ' -Dmusl=disabled'
+    }
+
+    try {
+        // copy gramine_test_dev_ioctl.h device header for `device_ioctl` LibOS test
+        sh '''
+            cp -f device-testing-tools/gramine-device-testing-module/gramine_test_dev_ioctl.h \
+                libos/test/regression/
+        '''
+
+        sh '''
+            meson setup build/ \
+                --werror \
+                --prefix="$PREFIX" \
+                --buildtype="$BUILDTYPE" \
+                -Ddirect=disabled \
+                -Dsgx=enabled \
+                -Dtests=enabled \
+                -Dsgx_driver=upstream \
+                $MESON_OPTIONS
+            ninja -vC build/
+        '''
+
+        // install
+        sh '''
+            ninja -vC build/ install
+            gramine-sgx-gen-private-key
+        '''
+    } finally {
+        archiveArtifacts 'build/meson-logs/**/*'
+        archiveArtifacts 'build/subprojects/glibc-*/glibc-build.log'
+    }
+
+    // archive all installed files
+    // NOTE we can't use ${env.PREFIX} here, because path needs to be relative to workdir
+    archiveArtifacts "usr/**/*"
+
+    // Absolute path to libdir, as configured by Meson.
+    // For our current builds this should be "$WORKSPACE/usr/lib/x86_64-linux-gnu":
+    // --prefix is set from $PREFIX above (see config-docker.jenkinsfile) and should be "$WORKSPACE/usr";
+    // --libdir is distro-dependent, but on Debian and derivatives it's "lib/x86_64-linux-gnu"
+    libdir = sh(returnStdout: true, script: '''
+        meson introspect build/ --buildoptions \
+        | jq -r '(map(select(.name == "prefix")) + map(select(.name == "libdir"))) | map(.value) | join("/")'
+    ''').trim()
+
+    env.GRAMINE_PKGLIBDIR = libdir + '/gramine'
+
+    // In CI we install to non-standard --prefix (see above). This makes sure the libraries are
+    // available anyway (e.g. gramine-sgx-pf-crypt needs libsgx_util.so).
+    env.PKG_CONFIG_PATH = libdir + '/pkgconfig'
+
+    // prevent cheating and testing from repo
+    sh 'rm -rf build'
+    sh 'git clean -Xf subprojects'
+}

.ci/lib/stage-clean-vm.jenkinsfile

@@ -0,0 +1,6 @@
+stage('clean-vm') {
+    sh '''
+        rm -rf device-testing-tools driver
+        rm -rf libos/test/regression/gramine_test_dev_ioctl.h
+    '''
+}

.ci/lib/stage-test-vm.jenkinsfile

@@ -0,0 +1,15 @@
+stage('test') {
+    timeout(time: 15, unit: 'MINUTES') {
+        sh '''
+            export PWD_FOR_VM=$PWD
+
+            cd device-testing-tools/initramfs_builder
+
+            # we add `/sbin` to PATH to find insmod and poweroff programs
+            ./run.sh PWD_FOR_VM=$PWD_FOR_VM SGX=$SGX IS_VM=$IS_VM PATH=/sbin:$PATH \
+                PKG_CONFIG_PATH=$PKG_CONFIG_PATH PYTHONPATH=$PYTHONPATH \
+                XDG_CONFIG_HOME=$XDG_CONFIG_HOME GRAMINE_PKGLIBDIR=$GRAMINE_PKGLIBDIR | tee OUTPUT
+            grep "TESTS OK" OUTPUT
+        '''
+    }
+}

.ci/linux-sgx-vm-gcc-release.jenkinsfile

@@ -0,0 +1,48 @@
+node('whatnots') {
+    checkout scm
+
+    env.SGX = '1'
+    env.IS_VM = '1'
+
+    load '.ci/lib/config-docker.jenkinsfile'
+
+    env.DOCKER_ARGS_SGX += '''
+        --volume=/usr/include/x86_64-linux-gnu/asm/sgx.h:/usr/include/asm/sgx.h:ro
+    '''
+
+    // Overwrite Gramine-specific seccomp policy because it conflicts with KVM requirements, see
+    // https://github.com/moby/moby/issues/42963 for details.
+    env.DOCKER_ARGS_COMMON +=
+        " --security-opt seccomp=${env.WORKSPACE}/scripts/docker_seccomp_aug_2022.json"
+
+    // Required by QEMU to run the same Linux kernel in VM (because we use host kernel as guest
+    // kernel for simplicity)
+    env.DOCKER_ARGS_COMMON += ' --volume=/boot:/boot:ro'
+
+    // only root and `kvm` group can access /dev/kvm, so add `kvm` GID to the in-Docker user
+    kvm_gid = sh(returnStdout: true, script: 'getent group kvm | cut -d: -f3').trim()
+    env.DOCKER_ARGS_COMMON += ' --group-add ' + kvm_gid
+
+    env.DOCKER_ARGS_COMMON += ' --device=/dev/kvm:/dev/kvm'
+
+    // only root and `sgx` group can access /dev/sgx_vepc, so add `sgx` GID to the in-Docker user
+    sgx_gid = sh(returnStdout: true, script: 'getent group sgx | cut -d: -f3').trim()
+    env.DOCKER_ARGS_SGX += ' --group-add ' + sgx_gid
+
+    env.DOCKER_ARGS_SGX += ' --device=/dev/sgx_vepc:/dev/sgx_vepc'
+
+    docker.build(
+        "local:${env.BUILD_TAG}",
+        '-f .ci/ubuntu22.04.dockerfile .'
+    ).inside("${env.DOCKER_ARGS_COMMON} ${env.DOCKER_ARGS_SGX}") {
+        load '.ci/lib/config.jenkinsfile'
+        load '.ci/lib/config-release.jenkinsfile'
+
+        load '.ci/lib/stage-lint.jenkinsfile'
+        load '.ci/lib/stage-clean-check-prepare.jenkinsfile'
+        load '.ci/lib/stage-build-sgx-vm.jenkinsfile'
+        load '.ci/lib/stage-test-vm.jenkinsfile'
+        load '.ci/lib/stage-clean-vm.jenkinsfile'
+        load '.ci/lib/stage-clean-check.jenkinsfile'
+    }
+}

.ci/ubuntu22.04.dockerfile

@@ -0,0 +1,99 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    autoconf \
+    bc \
+    bison \
+    build-essential \
+    cargo \
+    clang \
+    curl \
+    flex \
+    gawk \
+    gdb \
+    gettext \
+    git \
+    jq \
+    libapr1-dev \
+    libaprutil1-dev \
+    libcjson-dev \
+    libelf-dev \
+    libevent-dev \
+    libexpat1 \
+    libexpat1-dev \
+    libmemcached-tools \
+    libnss-mdns \
+    libnuma1 \
+    libomp-dev \
+    libpcre2-dev \
+    libpcre3-dev \
+    libprotobuf-c-dev \
+    libssl-dev \
+    libunwind8 \
+    libxfixes3 \
+    libxi6 \
+    libxml2-dev \
+    libxrender1 \
+    libxxf86vm1 \
+    linux-headers-generic \
+    musl \
+    musl-tools \
+    nasm \
+    net-tools \
+    netcat-openbsd \
+    ninja-build \
+    pkg-config \
+    protobuf-c-compiler \
+    protobuf-compiler \
+    pylint \
+    python3 \
+    python3-apport \
+    python3-apt \
+    python3-breathe \
+    python3-click \
+    python3-cryptography \
+    python3-jinja2 \
+    python3-lxml \
+    python3-numpy \
+    python3-pip \
+    python3-protobuf \
+    python3-pyelftools \
+    python3-pytest \
+    python3-pytest-xdist \
+    python3-scipy \
+    python3-sphinx-rtd-theme \
+    shellcheck \
+    sphinx-doc \
+    sqlite3 \
+    texinfo \
+    uthash-dev \
+    wget \
+    zlib1g \
+    zlib1g-dev
+
+# NOTE about meson version: we support "0.56 or newer", so in CI we pin to latest patch version of
+# the earliest supported minor version (pip implicitly installs latest version satisfying the
+# specification)
+RUN python3 -m pip install -U \
+    'tomli>=1.1.0' \
+    'tomli-w>=0.4.0' \
+    'meson>=0.56,<0.57' \
+    'recommonmark>=0.5.0,<=0.7.1' \
+    'docutils>=0.17,<0.18'
+
+# Dependencies required for building kernel modules and running VMs
+RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    cpio \
+    dwarves \
+    g++-10 \
+    gcc-10 \
+    kmod \
+    qemu-kvm
+
+# Kernel on the host machine is built with GCC-10, so we need to set it as default in Docker
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 && \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10 && \
+    update-alternatives --set gcc /usr/bin/gcc-10 && \
+    update-alternatives --set g++ /usr/bin/g++-10
+
+CMD ["bash"]

libos/test/regression/device_ioctl.c

@@ -0,0 +1,26 @@
+#define _GNU_SOURCE /* for loff_t */
+#include <err.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "common.h"
+#include "rw_file.h"
+
+#include "gramine_test_dev_ioctl.h" /* currently unused */
+
+#define STRING_READWRITE      "Hello world via read/write\n"
+
+int main(void) {
+    int devfd = CHECK(open("/dev/gramine_test_dev", O_RDWR));
+
+    ssize_t bytes = posix_fd_write(devfd, STRING_READWRITE, sizeof(STRING_READWRITE));
+    if (bytes != sizeof(STRING_READWRITE))
+        CHECK(-1);
+
+    CHECK(close(devfd));
+    puts("TEST OK");
+    return 0;
+}

libos/test/regression/device_ioctl.manifest.template

@@ -0,0 +1,18 @@
+loader.entrypoint = "file:{{ gramine.libos }}"
+libos.entrypoint = "{{ entrypoint }}"
+
+loader.env.LD_LIBRARY_PATH = "/lib"
+
+fs.mounts = [
+  { path = "/lib", uri = "file:{{ gramine.runtimedir(libc) }}" },
+  { path = "/{{ entrypoint }}", uri = "file:{{ binary_dir }}/{{ entrypoint }}" },
+  { path = "/dev/gramine_test_dev", uri = "dev:/dev/gramine_test_dev" },
+]
+
+sgx.debug = true
+
+sgx.trusted_files = [
+  "file:{{ gramine.libos }}",
+  "file:{{ gramine.runtimedir(libc) }}/",
+  "file:{{ binary_dir }}/{{ entrypoint }}",
+]

libos/test/regression/meson.build

@@ -1,3 +1,5 @@
+fs = import('fs')
+
 tests = {
     'abort': {},
     'abort_multithread': {},
@@ -148,6 +150,11 @@ if host_machine.cpu_family() == 'x86_64'
     }
 endif
 
+# device_ioctl test may only be executed in a VM environment that prepares the below header file
+if fs.exists('gramine_test_dev_ioctl.h')
+    tests += { 'device_ioctl': {} }
+endif
+
 tests_musl = tests
 
 if host_machine.cpu_family() == 'x86_64'

libos/test/regression/test_libos.py

@@ -12,6 +12,7 @@
 from graminelibos.regression import (
     HAS_AVX,
     HAS_SGX,
+    IS_VM,
     ON_X86,
     USES_MUSL,
     RegressionTestCase,
@@ -1024,6 +1025,11 @@ def test_002_device_passthrough(self):
         stdout, _ = self.run_binary(['device_passthrough'])
         self.assertIn('TEST OK', stdout)
 
+    @unittest.skipUnless(IS_VM, '/dev/gramine_test_dev is available only on some Jenkins machines')
+    def test_003_device_ioctl(self):
+        stdout, _ = self.run_binary(['device_ioctl'])
+        self.assertIn('TEST OK', stdout)
+
     def test_010_path(self):
         stdout, _ = self.run_binary(['proc_path'])
         self.assertIn('proc path test success', stdout)

libos/test/regression/tests.toml

@@ -137,3 +137,10 @@ manifests = [
 manifests = [
   "attestation",
 ]
+
+
+[vm]
+
+manifests = [
+  "device_ioctl",
+]

libos/test/regression/tests_musl.toml

@@ -130,3 +130,9 @@ manifests = [
   "rdtsc",
   "sighandler_divbyzero",
 ]
+
+[vm]
+
+manifests = [
+  "device_ioctl",
+]

python/graminelibos/regression.py

@@ -19,6 +19,7 @@
 HAS_AVX = os.environ.get('AVX') == '1'
 HAS_EDMM = os.environ.get('EDMM') == '1'
 HAS_SGX = os.environ.get('SGX') == '1'
+IS_VM = os.environ.get('IS_VM') == '1'
 ON_X86 = os.uname().machine in ['x86_64']
 USES_MUSL = os.environ.get('GRAMINE_MUSL') == '1'