[CI] Add a VM-based Jenkins pipeline

.ci/lib/stage-build-sgx-vm.jenkinsfile

@@ -0,0 +1,92 @@
+stage('build') {
+    sh '''
+        # we add `/sbin` to PATH to find the `modprobe` program
+        export PATH="/sbin:$PATH"
+
+        git clone https://github.com/gramineproject/device-testing-tools.git
+        cd device-testing-tools
+
+        cd initramfs_builder
+        {
+           echo '#!/bin/sh'
+           echo 'if test -n $SGX; then GRAMINE=gramine-sgx; else GRAMINE=gramine-direct; fi'
+           echo 'cd $PWD_FOR_VM'
+           echo '( cd device-testing-tools/gramine-device-testing-module; insmod gramine-testing-dev.ko )'
+
+           # only couple tests -- executing in a VM with virtio-9p-pci FS passthrough is very slow
+           echo 'cd libos/test/regression'
+           echo 'gramine-test build helloworld; $GRAMINE helloworld'
+           echo 'gramine-test build device_ioctl; $GRAMINE device_ioctl'
+           echo 'echo "TESTS OK"'
+           echo 'poweroff -n -f'
+        } > new_init
+        make ${MAKEOPTS}
+
+        cd ../gramine-device-testing-module
+        make ${MAKEOPTS}
+    '''
+
+    env.MESON_OPTIONS = ''
+    if (env.UBSAN == '1') {
+        env.MESON_OPTIONS += ' -Dubsan=enabled'
+    }
+    if (env.ASAN == '1') {
+        env.MESON_OPTIONS += ' -Dasan=enabled'
+    }
+    if (env.CC == 'clang') {
+        env.MESON_OPTIONS += ' -Dmusl=disabled'
+    }
+
+    try {
+        // copy gramine_test_dev_ioctl.h device header for `device_ioctl` LibOS test
+        sh '''
+            cp -f device-testing-tools/gramine-device-testing-module/gramine_test_dev_ioctl.h \
+                libos/test/regression/
+        '''
+
+        sh '''
+            meson setup build/ \
+                --werror \
+                --prefix="$PREFIX" \
+                --buildtype="$BUILDTYPE" \
+                -Ddirect=disabled \
+                -Dsgx=enabled \
+                -Dtests=enabled \
+                -Dsgx_driver=upstream \
+                $MESON_OPTIONS
+            ninja -vC build/
+        '''
+
+        // install
+        sh '''
+            ninja -vC build/ install
+            gramine-sgx-gen-private-key
+        '''
+    } finally {
+        archiveArtifacts 'build/meson-logs/**/*'
+        archiveArtifacts 'build/subprojects/glibc-*/glibc-build.log'
+    }
+
+    // archive all installed files
+    // NOTE we can't use ${env.PREFIX} here, because path needs to be relative to workdir
+    archiveArtifacts "usr/**/*"
+
+    // Absolute path to libdir, as configured by Meson.
+    // For our current builds this should be "$WORKSPACE/usr/lib/x86_64-linux-gnu":
+    // --prefix is set from $PREFIX above (see config-docker.jenkinsfile) and should be "$WORKSPACE/usr";
+    // --libdir is distro-dependent, but on Debian and derivatives it's "lib/x86_64-linux-gnu"
+    libdir = sh(returnStdout: true, script: '''
+        meson introspect build/ --buildoptions \
+        | jq -r '(map(select(.name == "prefix")) + map(select(.name == "libdir"))) | map(.value) | join("/")'
+    ''').trim()
+
+    env.GRAMINE_PKGLIBDIR = libdir + '/gramine'
+
+    // In CI we install to non-standard --prefix (see above). This makes sure the libraries are
+    // available anyway (e.g. gramine-sgx-pf-crypt needs libsgx_util.so).
+    env.PKG_CONFIG_PATH = libdir + '/pkgconfig'
+
+    // prevent cheating and testing from repo
+    sh 'rm -rf build'
+    sh 'git clean -Xf subprojects'
+}

.ci/lib/stage-clean-vm.jenkinsfile

@@ -0,0 +1,6 @@
+stage('clean-vm') {
+    sh '''
+        rm -rf device-testing-tools driver
+        rm -rf libos/test/regression/gramine_test_dev_ioctl.h
+    '''
+}

.ci/lib/stage-test-vm.jenkinsfile

@@ -0,0 +1,15 @@
+stage('test') {
+    timeout(time: 15, unit: 'MINUTES') {
+        sh '''
+            export PWD_FOR_VM=$PWD
+
+            cd device-testing-tools/initramfs_builder
+
+            # we add `/sbin` to PATH to find insmod and poweroff programs
+            ./run.sh PWD_FOR_VM=$PWD_FOR_VM SGX=$SGX IS_VM=$IS_VM PATH=/sbin:$PATH \
+                PKG_CONFIG_PATH=$PKG_CONFIG_PATH PYTHONPATH=$PYTHONPATH \
+                XDG_CONFIG_HOME=$XDG_CONFIG_HOME GRAMINE_PKGLIBDIR=$GRAMINE_PKGLIBDIR | tee OUTPUT
+            grep "TESTS OK" OUTPUT
+        '''
+    }
+}

.ci/linux-sgx-vm-gcc-release.jenkinsfile

@@ -0,0 +1,48 @@
+node('whatnots') {
+    checkout scm
+
+    env.SGX = '1'
+    env.IS_VM = '1'
+
+    load '.ci/lib/config-docker.jenkinsfile'
+
+    env.DOCKER_ARGS_SGX += '''
+        --volume=/usr/include/x86_64-linux-gnu/asm/sgx.h:/usr/include/asm/sgx.h:ro
+    '''
+
+    // Overwrite Gramine-specific seccomp policy because it conflicts with KVM requirements, see
+    // https://github.com/moby/moby/issues/42963 for details.
+    env.DOCKER_ARGS_COMMON +=
+        " --security-opt seccomp=${env.WORKSPACE}/scripts/docker_seccomp_aug_2022.json"
+
+    // Required by QEMU to run the same Linux kernel in VM (because we use host kernel as guest
+    // kernel for simplicity)
+    env.DOCKER_ARGS_COMMON += ' --volume=/boot:/boot:ro'
+
+    // only root and `kvm` group can access /dev/kvm, so add `kvm` GID to the in-Docker user
+    kvm_gid = sh(returnStdout: true, script: 'getent group kvm | cut -d: -f3').trim()
+    env.DOCKER_ARGS_COMMON += ' --group-add ' + kvm_gid
+
+    env.DOCKER_ARGS_COMMON += ' --device=/dev/kvm:/dev/kvm'
+
+    // only root and `sgx` group can access /dev/sgx_vepc, so add `sgx` GID to the in-Docker user
+    sgx_gid = sh(returnStdout: true, script: 'getent group sgx | cut -d: -f3').trim()
+    env.DOCKER_ARGS_SGX += ' --group-add ' + sgx_gid
+
+    env.DOCKER_ARGS_SGX += ' --device=/dev/sgx_vepc:/dev/sgx_vepc'
+
+    docker.build(
+        "local:${env.BUILD_TAG}",
+        '-f .ci/ubuntu22.04.dockerfile .'
+    ).inside("${env.DOCKER_ARGS_COMMON} ${env.DOCKER_ARGS_SGX}") {
+        load '.ci/lib/config.jenkinsfile'
+        load '.ci/lib/config-release.jenkinsfile'
+
+        load '.ci/lib/stage-lint.jenkinsfile'
+        load '.ci/lib/stage-clean-check-prepare.jenkinsfile'
+        load '.ci/lib/stage-build-sgx-vm.jenkinsfile'
+        load '.ci/lib/stage-test-vm.jenkinsfile'
+        load '.ci/lib/stage-clean-vm.jenkinsfile'
+        load '.ci/lib/stage-clean-check.jenkinsfile'
+    }
+}

.ci/ubuntu22.04.dockerfile

@@ -0,0 +1,99 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    autoconf \
+    bc \
+    bison \
+    build-essential \
+    cargo \
+    clang \
+    curl \
+    flex \
+    gawk \
+    gdb \
+    gettext \
+    git \
+    jq \
+    libapr1-dev \
+    libaprutil1-dev \
+    libcjson-dev \
+    libelf-dev \
+    libevent-dev \
+    libexpat1 \
+    libexpat1-dev \
+    libmemcached-tools \
+    libnss-mdns \
+    libnuma1 \
+    libomp-dev \
+    libpcre2-dev \
+    libpcre3-dev \
+    libprotobuf-c-dev \
+    libssl-dev \
+    libunwind8 \
+    libxfixes3 \
+    libxi6 \
+    libxml2-dev \
+    libxrender1 \
+    libxxf86vm1 \
+    linux-headers-generic \
+    musl \
+    musl-tools \
+    nasm \
+    net-tools \
+    netcat-openbsd \
+    ninja-build \
+    pkg-config \
+    protobuf-c-compiler \
+    protobuf-compiler \
+    pylint \
+    python3 \
+    python3-apport \
+    python3-apt \
+    python3-breathe \
+    python3-click \
+    python3-cryptography \
+    python3-jinja2 \
+    python3-lxml \
+    python3-numpy \
+    python3-pip \
+    python3-protobuf \
+    python3-pyelftools \
+    python3-pytest \
+    python3-pytest-xdist \
+    python3-scipy \
+    python3-sphinx-rtd-theme \
+    shellcheck \
+    sphinx-doc \
+    sqlite3 \
+    texinfo \
+    uthash-dev \
+    wget \
+    zlib1g \
+    zlib1g-dev
+
+# NOTE about meson version: we support "0.56 or newer", so in CI we pin to latest patch version of
+# the earliest supported minor version (pip implicitly installs latest version satisfying the
+# specification)
+RUN python3 -m pip install -U \
+    'tomli>=1.1.0' \
+    'tomli-w>=0.4.0' \
+    'meson>=0.56,<0.57' \
+    'recommonmark>=0.5.0,<=0.7.1' \
+    'docutils>=0.17,<0.18'
+
+# Dependencies required for building kernel modules and running VMs
+RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    cpio \
+    dwarves \
+    g++-10 \
+    gcc-10 \
+    kmod \
+    qemu-kvm
+
+# Kernel on the host machine is built with GCC-10, so we need to set it as default in Docker
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 && \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10 && \
+    update-alternatives --set gcc /usr/bin/gcc-10 && \
+    update-alternatives --set g++ /usr/bin/g++-10
+
+CMD ["bash"]

libos/test/regression/device_ioctl.c

@@ -0,0 +1,26 @@
+#define _GNU_SOURCE /* for loff_t */
+#include <err.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "common.h"
+#include "rw_file.h"
+
+#include "gramine_test_dev_ioctl.h" /* currently unused */
+
+#define STRING_READWRITE "Hello world via read/write\n"
+
+int main(void) {
+    int devfd = CHECK(open("/dev/gramine_test_dev", O_RDWR));
+
+    ssize_t bytes = posix_fd_write(devfd, STRING_READWRITE, sizeof(STRING_READWRITE));
+    if (bytes != sizeof(STRING_READWRITE))
+        CHECK(-1);
+
+    CHECK(close(devfd));
+    puts("TEST OK");
+    return 0;
+}

libos/test/regression/device_ioctl.manifest.template

@@ -0,0 +1,18 @@
+loader.entrypoint = "file:{{ gramine.libos }}"
+libos.entrypoint = "{{ entrypoint }}"
+
+loader.env.LD_LIBRARY_PATH = "/lib"
+
+fs.mounts = [
+  { path = "/lib", uri = "file:{{ gramine.runtimedir(libc) }}" },
+  { path = "/{{ entrypoint }}", uri = "file:{{ binary_dir }}/{{ entrypoint }}" },
+  { path = "/dev/gramine_test_dev", uri = "dev:/dev/gramine_test_dev" },
+]
+
+sgx.debug = true
+
+sgx.trusted_files = [
+  "file:{{ gramine.libos }}",
+  "file:{{ gramine.runtimedir(libc) }}/",
+  "file:{{ binary_dir }}/{{ entrypoint }}",
+]

libos/test/regression/meson.build

@@ -1,3 +1,5 @@
+fs = import('fs')
+
 tests = {
     'abort': {},
     'abort_multithread': {},
@@ -149,6 +151,11 @@ if host_machine.cpu_family() == 'x86_64'
     }
 endif
 
+# device_ioctl test may only be executed in a VM environment that prepares the below header file
+if fs.exists('gramine_test_dev_ioctl.h')
+    tests += { 'device_ioctl': {} }
+endif
+
 tests_musl = tests
 
 if host_machine.cpu_family() == 'x86_64'

libos/test/regression/test_libos.py

@@ -12,6 +12,7 @@
 from graminelibos.regression import (
     HAS_AVX,
     HAS_SGX,
+    IS_VM,
     ON_X86,
     USES_MUSL,
     RegressionTestCase,
@@ -1032,6 +1033,11 @@ def test_002_device_passthrough(self):
         stdout, _ = self.run_binary(['device_passthrough'])
         self.assertIn('TEST OK', stdout)
 
+    @unittest.skipUnless(IS_VM, '/dev/gramine_test_dev is available only on some Jenkins machines')
+    def test_003_device_ioctl(self):
+        stdout, _ = self.run_binary(['device_ioctl'])
+        self.assertIn('TEST OK', stdout)
+
     def test_010_path(self):
         stdout, _ = self.run_binary(['proc_path'])
         self.assertIn('proc path test success', stdout)

libos/test/regression/tests.toml

@@ -138,3 +138,10 @@ manifests = [
 manifests = [
   "attestation",
 ]
+
+
+[vm]
+
+manifests = [
+  "device_ioctl",
+]

libos/test/regression/tests_musl.toml

@@ -131,3 +131,9 @@ manifests = [
   "rdtsc",
   "sighandler_divbyzero",
 ]
+
+[vm]
+
+manifests = [
+  "device_ioctl",
+]

python/graminelibos/regression.py

@@ -19,6 +19,7 @@
 HAS_AVX = os.environ.get('AVX') == '1'
 HAS_EDMM = os.environ.get('EDMM') == '1'
 HAS_SGX = os.environ.get('SGX') == '1'
+IS_VM = os.environ.get('IS_VM') == '1'
 ON_X86 = os.uname().machine in ['x86_64']
 USES_MUSL = os.environ.get('GRAMINE_MUSL') == '1'