-
Notifications
You must be signed in to change notification settings - Fork 193
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[CI] WIP: Add a VM-based Jenkins pipeline
New Jenkins pipeline runs a Docker container that creates a minimal VM based on https://github.com/gramineproject/device-testing-tools repo and runs a subset of Gramine tests, in particular, the device IOCTL tests. The pipeline uses Ubuntu 22.04 with modern Linux kernel (to have an upstream SGX driver and support for SGX in KVM) and QEMU/KVM to run the VM. Signed-off-by: Dmitrii Kuvaiskii <[email protected]>
- Loading branch information
Showing
5 changed files
with
272 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,83 @@ | ||
stage('build') { | ||
// we add `/sbin` to PATH to find the `modprobe` program | ||
sh ''' | ||
export PATH="/sbin:$PATH" | ||
|
||
git clone https://github.com/gramineproject/device-testing-tools.git | ||
cd device-testing-tools | ||
git checkout 5d924c0f611b56a41fb28b73bd4b04d55dcdd7d9 | ||
|
||
cd initramfs_builder | ||
cp -f new_init_jenkins new_init | ||
make | ||
|
||
cd ../gramine-device-testing-module | ||
make | ||
''' | ||
|
||
sh ''' | ||
mkdir -p driver/asm | ||
cd driver/asm | ||
wget --timeout=10 -O sgx.h \ | ||
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/plain/arch/x86/include/uapi/asm/sgx.h?h=v5.11 | ||
sha256sum sgx.h | grep -q a34a997ade42b61376b1c5d3d50f839fd28f2253fa047cb9c0e68a1b00477956 | ||
''' | ||
|
||
env.MESON_OPTIONS = '' | ||
if (env.UBSAN == '1') { | ||
env.MESON_OPTIONS += ' -Dubsan=enabled' | ||
} | ||
if (env.ASAN == '1') { | ||
env.MESON_OPTIONS += ' -Dasan=enabled' | ||
} | ||
if (env.CC == 'clang') { | ||
env.MESON_OPTIONS += ' -Dmusl=disabled' | ||
} | ||
|
||
try { | ||
sh ''' | ||
meson setup build/ \ | ||
--werror \ | ||
--prefix="$PREFIX" \ | ||
--buildtype="$BUILDTYPE" \ | ||
-Ddirect=disabled \ | ||
-Dsgx=enabled \ | ||
-Dtests=enabled \ | ||
-Dsgx_driver_include_path="$PWD/driver" \ | ||
$MESON_OPTIONS | ||
ninja -vC build/ | ||
''' | ||
|
||
// install | ||
sh ''' | ||
ninja -vC build/ install | ||
gramine-sgx-gen-private-key | ||
''' | ||
} finally { | ||
archiveArtifacts 'build/meson-logs/**/*' | ||
archiveArtifacts 'build/subprojects/glibc-*/glibc-build.log' | ||
} | ||
|
||
// archive all installed files | ||
// NOTE we can't use ${env.PREFIX} here, because path needs to be relative to workdir | ||
archiveArtifacts "usr/**/*" | ||
|
||
// Absolute path to libdir, as configured by Meson. | ||
// For our current builds this should be "$WORKSPACE/usr/lib/x86_64-linux-gnu": | ||
// --prefix is set from $PREFIX above (see config-docker.jenkinsfile) and should be "$WORKSPACE/usr"; | ||
// --libdir is distro-dependent, but on Debian and derivatives it's "lib/x86_64-linux-gnu" | ||
libdir = sh(returnStdout: true, script: ''' | ||
meson introspect build/ --buildoptions \ | ||
| jq -r '(map(select(.name == "prefix")) + map(select(.name == "libdir"))) | map(.value) | join("/")' | ||
''').trim() | ||
|
||
env.GRAMINE_PKGLIBDIR = libdir + '/gramine' | ||
|
||
// In CI we install to non-standard --prefix (see above). This makes sure the libraries are | ||
// available anyway (e.g. gramine-sgx-pf-crypt needs libsgx_util.so). | ||
env.PKG_CONFIG_PATH = libdir + '/pkgconfig' | ||
|
||
// prevent cheating and testing from repo | ||
sh 'rm -rf build' | ||
sh 'git clean -Xf subprojects' | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
stage('test') { | ||
// prepare files to pass current work dir and envvars into VM | ||
sh ''' | ||
mkdir -p files_for_vm | ||
echo $PWD > files_for_vm/pwd | ||
|
||
touch files_for_vm/envs | ||
echo "export SGX=$SGX" >> files_for_vm/envs | ||
echo "export IS_VM=$IS_VM" >> files_for_vm/envs | ||
|
||
echo "export PATH=\"$PATH\"" >> files_for_vm/envs | ||
echo "export PKG_CONFIG_PATH=\"$PKG_CONFIG_PATH\"" >> files_for_vm/envs | ||
echo "export PYTHONPATH=\"$PYTHONPATH\"" >> files_for_vm/envs | ||
echo "export GRAMINE_PKGLIBDIR=\"$GRAMINE_PKGLIBDIR\"" >> files_for_vm/envs | ||
echo "export XDG_CONFIG_HOME=\"$XDG_CONFIG_HOME\"" >> files_for_vm/envs | ||
''' | ||
|
||
timeout(time: 15, unit: 'MINUTES') { | ||
sh ''' | ||
cd device-testing-tools/initramfs_builder | ||
./run.sh | tee OUTPUT | ||
grep "TESTS OK" OUTPUT | ||
''' | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
node('whatnots') { | ||
checkout scm | ||
|
||
env.SGX = '1' | ||
env.IS_VM = '1' | ||
|
||
load '.ci/lib/config-docker.jenkinsfile' | ||
|
||
if (fileExists('/dev/kvm')) { | ||
env.DOCKER_ARGS_COMMON += ' --device=/dev/kvm:/dev/kvm' | ||
} | ||
|
||
// Overwrite Gramine-specific seccomp policy because it conflicts with KVM requirements, see | ||
// https://github.com/moby/moby/issues/42963 for details. | ||
// FIXME: remove this line once seccomp policy is updated in core Gramine repo. | ||
env.DOCKER_ARGS_COMMON += ' --security-opt seccomp=unconfined' | ||
|
||
// Required by `make` of gramine-device-testing-module because | ||
// `/usr/src/linux-headers-.../scripts` is a symlink to smth like `../../lib/linux-kbuild-...` | ||
// FIXME: either move to Ubuntu 22.04 (which seems to not require this hack) or ditch Docker | ||
env.DOCKER_ARGS_COMMON += ' --volume=/usr/lib/linux-kbuild-5.18:/usr/lib/linux-kbuild-5.18:ro' | ||
|
||
// only root and `kvm` group can access /dev/kvm, so add `kvm` GID to the in-Docker user | ||
kvm_gid = sh(returnStdout: true, script: 'getent group kvm | cut -d: -f3').trim() | ||
env.DOCKER_ARGS_COMMON += ' --group-add ${kvm_gid}' | ||
|
||
docker.build( | ||
"local:${env.BUILD_TAG}", | ||
'-f .ci/ubuntu22.04.dockerfile .' | ||
).inside("${env.DOCKER_ARGS_COMMON} ${env.DOCKER_ARGS_SGX}") { | ||
load '.ci/lib/config.jenkinsfile' | ||
load '.ci/lib/config-release.jenkinsfile' | ||
|
||
load '.ci/lib/stage-lint.jenkinsfile' | ||
load '.ci/lib/stage-clean-check-prepare.jenkinsfile' | ||
load '.ci/lib/stage-build-sgx-vm.jenkinsfile' | ||
load '.ci/lib/stage-test-vm.jenkinsfile' | ||
load '.ci/lib/stage-clean-check.jenkinsfile' | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,123 @@ | ||
FROM ubuntu:22.04 | ||
|
||
# Add steps here to set up dependencies | ||
RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ | ||
autoconf \ | ||
bc \ | ||
bison \ | ||
build-essential \ | ||
cargo \ | ||
clang \ | ||
curl \ | ||
flex \ | ||
gawk \ | ||
gdb \ | ||
gettext \ | ||
git \ | ||
jq \ | ||
libapr1-dev \ | ||
libaprutil1-dev \ | ||
libcjson-dev \ | ||
libcurl4-openssl-dev \ | ||
libelf-dev \ | ||
libevent-dev \ | ||
libexpat1 \ | ||
libexpat1-dev \ | ||
libmemcached-tools \ | ||
libnss-mdns \ | ||
libnuma1 \ | ||
libomp-dev \ | ||
libpcre2-dev \ | ||
libpcre3-dev \ | ||
libprotobuf-c-dev \ | ||
libssl-dev \ | ||
libunwind8 \ | ||
libxfixes3 \ | ||
libxi6 \ | ||
libxml2-dev \ | ||
libxrender1 \ | ||
libxxf86vm1 \ | ||
linux-headers-generic \ | ||
musl \ | ||
musl-tools \ | ||
nasm \ | ||
net-tools \ | ||
netcat-openbsd \ | ||
ninja-build \ | ||
pkg-config \ | ||
protobuf-c-compiler \ | ||
protobuf-compiler \ | ||
pylint \ | ||
python3 \ | ||
python3-apport \ | ||
python3-apt \ | ||
python3-breathe \ | ||
python3-click \ | ||
python3-cryptography \ | ||
python3-jinja2 \ | ||
python3-lxml \ | ||
python3-numpy \ | ||
python3-pip \ | ||
python3-protobuf \ | ||
python3-pyelftools \ | ||
python3-pytest \ | ||
python3-pytest-xdist \ | ||
python3-scipy \ | ||
python3-sphinx-rtd-theme \ | ||
python3-toml \ | ||
shellcheck \ | ||
sphinx-doc \ | ||
sqlite3 \ | ||
texinfo \ | ||
uthash-dev \ | ||
wget \ | ||
zlib1g \ | ||
zlib1g-dev | ||
|
||
# NOTE about meson version: we support "0.56 or newer", so in CI we pin to latest patch version of | ||
# the earliest supported minor version (pip implicitly installs latest version satisfying the | ||
# specification) | ||
RUN python3 -m pip install -U \ | ||
'meson>=0.56,<0.57' \ | ||
'docutils>=0.17,<0.18' | ||
|
||
# Dependencies required for building kernel modules and running VMs | ||
RUN env DEBIAN_FRONTEND=noninteractive apt-get install -y \ | ||
cpio \ | ||
g++-10 \ | ||
gcc-10 \ | ||
kmod \ | ||
qemu-kvm | ||
|
||
# Kernel on the host machine is built with GCC-10, so we need to set it as default in Docker | ||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10 && \ | ||
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10 && \ | ||
update-alternatives --set gcc /usr/bin/gcc-10 && \ | ||
update-alternatives --set g++ /usr/bin/g++-10 | ||
|
||
# Add the user UID:1001, GID:1001, home at /leeroy | ||
RUN \ | ||
groupadd -r leeroy -g 1001 && \ | ||
useradd -u 1001 -r -g leeroy -m -d /leeroy -c "Leeroy Jenkins" leeroy && \ | ||
chmod 755 /leeroy | ||
|
||
# Make sure /leeroy can be written by leeroy | ||
RUN chown 1001 /leeroy | ||
|
||
# Blow away any random state | ||
RUN rm -f /leeroy/.rnd | ||
|
||
# Make a directory for the intel driver | ||
RUN mkdir -p /opt/intel && chown 1001 /opt/intel | ||
|
||
# Set the working directory to leeroy home directory | ||
WORKDIR /leeroy | ||
|
||
# Specify the user to execute all commands below | ||
USER leeroy | ||
|
||
# Set environment variables. | ||
ENV HOME /leeroy | ||
|
||
# Define default command. | ||
CMD ["bash"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters