Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Flesh out NVIDIA support for biarch and multiarch systems #4207

Merged
merged 5 commits into from
Nov 15, 2017
Merged

Flesh out NVIDIA support for biarch and multiarch systems #4207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0de674f
cmd/snap-confine: Implement full 32-bit NVIDIA driver support
ikeydoherty Nov 13, 2017
b588a23
cmd: Support exposing NVIDIA Vulkan ICD files to the snaps
ikeydoherty Nov 13, 2017
b2911db
cmd/snap-confine: Add missing bi-arch NVIDIA files
ikeydoherty Nov 13, 2017
549ffb6
cmd: make fmt
zyga Nov 13, 2017
0d9a468
cmd/snap-confine: Loosen the NVIDIA Vulkan ICD glob
ikeydoherty Nov 14, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 89 additions & 13 deletions cmd/snap-confine/mount-support-nvidia.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,18 @@

#define SC_NVIDIA_DRIVER_VERSION_FILE "/sys/module/nvidia/version"

#define SC_LIBGL_DIR "/var/lib/snapd/lib/gl"
#define SC_LIBGL32_DIR "/var/lib/snapd/lib/gl32"
#define SC_VULKAN_DIR "/var/lib/snapd/lib/vulkan"

// Location for NVIDIA vulkan files (including _wayland)
static const char *vulkan_globs[] = {
"/usr/share/vulkan/icd.d/*nvidia*.json",
};

static const size_t vulkan_globs_len =
sizeof vulkan_globs / sizeof *vulkan_globs;

#ifdef NVIDIA_BIARCH

// List of globs that describe nvidia userspace libraries.
Expand Down Expand Up @@ -70,6 +82,7 @@ static const char *nvidia_globs[] = {
"/usr/lib/libnvidia-cfg.so*",
"/usr/lib/libnvidia-compiler.so*",
"/usr/lib/libnvidia-eglcore.so*",
"/usr/lib/libnvidia-egl-wayland*",
"/usr/lib/libnvidia-encode.so*",
"/usr/lib/libnvidia-fatbinaryloader.so*",
"/usr/lib/libnvidia-fbc.so*",
Expand All @@ -79,11 +92,51 @@ static const char *nvidia_globs[] = {
"/usr/lib/libnvidia-ml.so*",
"/usr/lib/libnvidia-ptxjitcompiler.so*",
"/usr/lib/libnvidia-tls.so*",
"/usr/lib/vdpau/libvdpau_nvidia.so*",
};

static const size_t nvidia_globs_len =
sizeof nvidia_globs / sizeof *nvidia_globs;

// 32-bit variants of the NVIDIA driver libraries
static const char *nvidia_globs32[] = {
"/usr/lib32/libEGL.so*",
"/usr/lib32/libEGL_nvidia.so*",
"/usr/lib32/libGL.so*",
"/usr/lib32/libOpenGL.so*",
"/usr/lib32/libGLESv1_CM.so*",
"/usr/lib32/libGLESv1_CM_nvidia.so*",
"/usr/lib32/libGLESv2.so*",
"/usr/lib32/libGLESv2_nvidia.so*",
"/usr/lib32/libGLX_indirect.so*",
"/usr/lib32/libGLX_nvidia.so*",
"/usr/lib32/libGLX.so*",
"/usr/lib32/libGLdispatch.so*",
"/usr/lib32/libGLU.so*",
"/usr/lib32/libXvMCNVIDIA.so*",
"/usr/lib32/libXvMCNVIDIA_dynamic.so*",
"/usr/lib32/libcuda.so*",
"/usr/lib32/libnvcuvid.so*",
"/usr/lib32/libnvidia-cfg.so*",
"/usr/lib32/libnvidia-compiler.so*",
"/usr/lib32/libnvidia-eglcore.so*",
"/usr/lib32/libnvidia-encode.so*",
"/usr/lib32/libnvidia-fatbinaryloader.so*",
"/usr/lib32/libnvidia-fbc.so*",
"/usr/lib32/libnvidia-glcore.so*",
"/usr/lib32/libnvidia-glsi.so*",
"/usr/lib32/libnvidia-ifr.so*",
"/usr/lib32/libnvidia-ml.so*",
"/usr/lib32/libnvidia-ptxjitcompiler.so*",
"/usr/lib32/libnvidia-tls.so*",
"/usr/lib32/vdpau/libvdpau_nvidia.so*",
};

static const size_t nvidia_globs32_len =
sizeof nvidia_globs32 / sizeof *nvidia_globs32;

#endif // ifdef NVIDIA_BIARCH

// Populate libgl_dir with a symlink farm to files matching glob_list.
//
// The symbolic links are made in one of two ways. If the library found is a
Expand Down Expand Up @@ -167,27 +220,39 @@ static void sc_populate_libgl_with_hostfs_symlinks(const char *libgl_dir,
}
}

static void sc_mount_nvidia_driver_biarch(const char *rootfs_dir)
static void sc_mount_and_glob_files(const char *rootfs_dir,
const char *tgt_dir,
const char *glob_list[],
size_t glob_list_len)
{
// Bind mount a tmpfs on $rootfs_dir/var/lib/snapd/lib/gl
// Bind mount a tmpfs on $rootfs_dir/$tgt_dir (i.e. /var/lib/snapd/lib/gl)
char buf[512] = { 0 };
sc_must_snprintf(buf, sizeof(buf), "%s%s", rootfs_dir,
"/var/lib/snapd/lib/gl");
sc_must_snprintf(buf, sizeof(buf), "%s%s", rootfs_dir, tgt_dir);
const char *libgl_dir = buf;
debug("mounting tmpfs at %s", libgl_dir);
if (mount("none", libgl_dir, "tmpfs", MS_NODEV | MS_NOEXEC, NULL) != 0) {
die("cannot mount tmpfs at %s", libgl_dir);
};
// Populate libgl_dir with symlinks to libraries from hostfs
sc_populate_libgl_with_hostfs_symlinks(libgl_dir, nvidia_globs,
nvidia_globs_len);
// Remount .../lib/gl read only
sc_populate_libgl_with_hostfs_symlinks(libgl_dir, glob_list,
glob_list_len);
// Remount $tgt_dir (i.e. .../lib/gl) read only
debug("remounting tmpfs as read-only %s", libgl_dir);
if (mount(NULL, buf, NULL, MS_REMOUNT | MS_RDONLY, NULL) != 0) {
die("cannot remount %s as read-only", buf);
}
}

#ifdef NVIDIA_BIARCH

static void sc_mount_nvidia_driver_biarch(const char *rootfs_dir)
{
sc_mount_and_glob_files(rootfs_dir, SC_LIBGL_DIR,
nvidia_globs, nvidia_globs_len);
sc_mount_and_glob_files(rootfs_dir, SC_LIBGL32_DIR,
nvidia_globs32, nvidia_globs32_len);
}

#endif // ifdef NVIDIA_BIARCH

#ifdef NVIDIA_MULTIARCH
Expand All @@ -197,8 +262,6 @@ struct sc_nvidia_driver {
int minor_version;
};

#define SC_LIBGL_DIR "/var/lib/snapd/lib/gl"

static void sc_probe_nvidia_driver(struct sc_nvidia_driver *driver)
{
FILE *file SC_CLEANUP(sc_cleanup_file) = NULL;
Expand All @@ -224,7 +287,8 @@ static void sc_probe_nvidia_driver(struct sc_nvidia_driver *driver)
driver->minor_version);
}

static void sc_mount_nvidia_driver_multiarch(const char *rootfs_dir)
static void sc_mount_and_bind(const char *rootfs_dir, const char *src_dir,
const char *tgt_dir)
{
struct sc_nvidia_driver driver;

Expand All @@ -239,9 +303,9 @@ static void sc_mount_nvidia_driver_multiarch(const char *rootfs_dir)
// and for the gl directory.
char src[PATH_MAX] = { 0 };
char dst[PATH_MAX] = { 0 };
sc_must_snprintf(src, sizeof src, "/usr/lib/nvidia-%d",
sc_must_snprintf(src, sizeof src, "%s-%d", src_dir,
driver.major_version);
sc_must_snprintf(dst, sizeof dst, "%s%s", rootfs_dir, SC_LIBGL_DIR);
sc_must_snprintf(dst, sizeof dst, "%s%s", rootfs_dir, tgt_dir);

// If there is no userspace driver available then don't try to mount it.
// This can happen for any number of reasons but one interesting one is
Expand All @@ -251,12 +315,20 @@ static void sc_mount_nvidia_driver_multiarch(const char *rootfs_dir)
if (access(src, F_OK) != 0) {
return;
}
// Bind mount the binary nvidia driver into /var/lib/snapd/lib/gl.
// Bind mount the binary nvidia driver into $tgt_dir (i.e. /var/lib/snapd/lib/gl).
debug("bind mounting nvidia driver %s -> %s", src, dst);
if (mount(src, dst, NULL, MS_BIND, NULL) != 0) {
die("cannot bind mount nvidia driver %s -> %s", src, dst);
}
}

static void sc_mount_nvidia_driver_multiarch(const char *rootfs_dir)
{
// Attempt mount of both the native and 32-bit variants of the driver if they exist
sc_mount_and_bind(rootfs_dir, "/usr/lib/nvidia", SC_LIBGL_DIR);
sc_mount_and_bind(rootfs_dir, "/usr/lib32/nvidia", SC_LIBGL32_DIR);
}

#endif // ifdef NVIDIA_MULTIARCH

void sc_mount_nvidia_driver(const char *rootfs_dir)
Expand All @@ -271,4 +343,8 @@ void sc_mount_nvidia_driver(const char *rootfs_dir)
#ifdef NVIDIA_BIARCH
sc_mount_nvidia_driver_biarch(rootfs_dir);
#endif // ifdef NVIDIA_BIARCH

// Common for both driver mechanisms
sc_mount_and_glob_files(rootfs_dir, SC_VULKAN_DIR,
vulkan_globs, vulkan_globs_len);
}
14 changes: 10 additions & 4 deletions cmd/snap-confine/snap-confine.apparmor.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,10 +252,16 @@
/dev/nvidiactl r,
/dev/nvidia-uvm r,
/usr/** r,
mount options=(rw bind) /usr/lib/nvidia-*/ -> /{tmp/snap.rootfs_*/,}var/lib/snapd/lib/gl/,
/tmp/snap.rootfs_*/var/lib/snapd/lib/gl/* w,
mount fstype=tmpfs options=(rw nodev noexec) none -> /tmp/snap.rootfs_*/var/lib/snapd/lib/gl/,
mount options=(remount ro) -> /tmp/snap.rootfs_*/var/lib/snapd/lib/gl/,
mount options=(rw bind) /usr/lib{,32}/nvidia-*/ -> /{tmp/snap.rootfs_*/,}var/lib/snapd/lib/gl{,32}/,
mount options=(rw bind) /usr/lib{,32}/nvidia-*/ -> /{tmp/snap.rootfs_*/,}var/lib/snapd/lib/gl{,32}/,
/tmp/snap.rootfs_*/var/lib/snapd/lib/gl{,32}/* w,
mount fstype=tmpfs options=(rw nodev noexec) none -> /tmp/snap.rootfs_*/var/lib/snapd/lib/gl{,32}/,
mount options=(remount ro) -> /tmp/snap.rootfs_*/var/lib/snapd/lib/gl{,32}/,

# Vulkan support
/tmp/snap.rootfs_*/var/lib/snapd/lib/vulkan/* w,
mount fstype=tmpfs options=(rw nodev noexec) none -> /tmp/snap.rootfs_*/var/lib/snapd/lib/vulkan/,
mount options=(remount ro) -> /tmp/snap.rootfs_*/var/lib/snapd/lib/vulkan/,

# for chroot on steroids, we use pivot_root as a better chroot that makes
# apparmor rules behave the same on classic and outside of classic.
Expand Down
9 changes: 7 additions & 2 deletions interfaces/builtin/opengl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ const openglConnectedPlugAppArmor = `
# Description: Can access opengl.

# specific gl libs
/var/lib/snapd/lib/gl/ r,
/var/lib/snapd/lib/gl/** rm,
/var/lib/snapd/lib/gl{,32}/ r,
/var/lib/snapd/lib/gl{,32}/** rm,

# Supports linux-driver-management from Solus (staged symlink trees into libdirs)
/var/lib/snapd/hostfs/{,usr/}lib{,32,64,x32}/{,@{multiarch}/}glx-provider/**.so{,.*} rm,
Expand All @@ -45,6 +45,11 @@ const openglConnectedPlugAppArmor = `
/var/lib/snapd/hostfs/{,usr/}lib{,32,64,x32}/{,@{multiarch}/}lib{GL,EGL}*nvidia.so{,.*} rm,
/var/lib/snapd/hostfs/{,usr/}lib{,32,64,x32}/{,@{multiarch}/}libGLdispatch.so{,.*} rm,

# Support reading the Vulkan ICD files
/var/lib/snapd/lib/vulkan/ r,
/var/lib/snapd/lib/vulkan/** r,
/var/lib/snapd/hostfs/usr/share/vulkan/icd.d/*nvidia*.json r,

# Main bi-arch GL libraries
/var/lib/snapd/hostfs/{,usr/}lib{,32,64,x32}/{,@{multiarch}/}lib{GL,EGL}.so{,.*} rm,

Expand Down
2 changes: 1 addition & 1 deletion snap/snapenv/snapenv.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ func basicEnv(info *snap.Info) map[string]string {
"SNAP_REVISION": info.Revision.String(),
"SNAP_ARCH": arch.UbuntuArchitecture(),
// see https://github.com/snapcore/snapd/pull/2732#pullrequestreview-18827193
"SNAP_LIBRARY_PATH": "/var/lib/snapd/lib/gl:/var/lib/snapd/void",
"SNAP_LIBRARY_PATH": "/var/lib/snapd/lib/gl:/var/lib/snapd/lib/gl32:/var/lib/snapd/void",
"SNAP_REEXEC": os.Getenv("SNAP_REEXEC"),
}
}
Expand Down
4 changes: 2 additions & 2 deletions snap/snapenv/snapenv_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ func (ts *HTestSuite) TestBasic(c *C) {
"SNAP_ARCH": arch.UbuntuArchitecture(),
"SNAP_COMMON": "/var/snap/foo/common",
"SNAP_DATA": "/var/snap/foo/17",
"SNAP_LIBRARY_PATH": "/var/lib/snapd/lib/gl:/var/lib/snapd/void",
"SNAP_LIBRARY_PATH": "/var/lib/snapd/lib/gl:/var/lib/snapd/lib/gl32:/var/lib/snapd/void",
"SNAP_NAME": "foo",
"SNAP_REEXEC": "",
"SNAP_REVISION": "17",
Expand Down Expand Up @@ -129,7 +129,7 @@ func (s *HTestSuite) TestSnapRunSnapExecEnv(c *C) {
"SNAP_ARCH": arch.UbuntuArchitecture(),
"SNAP_COMMON": "/var/snap/snapname/common",
"SNAP_DATA": "/var/snap/snapname/42",
"SNAP_LIBRARY_PATH": "/var/lib/snapd/lib/gl:/var/lib/snapd/void",
"SNAP_LIBRARY_PATH": "/var/lib/snapd/lib/gl:/var/lib/snapd/lib/gl32:/var/lib/snapd/void",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wasn't aware this can be in the same path at once.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, the linker will just skip invalid ELFCLASS files

"SNAP_NAME": "snapname",
"SNAP_REEXEC": "",
"SNAP_REVISION": "42",
Expand Down
2 changes: 1 addition & 1 deletion tests/main/snap-env/task.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ execute: |
MATCH '^SNAP_ARCH=(amd64|i386|arm64|armhf|ppc64el)$' < snap-vars.txt
MATCH '^SNAP_COMMON=/var/snap/test-snapd-tools/common$' < snap-vars.txt
MATCH '^SNAP_DATA=/var/snap/test-snapd-tools/x1$' < snap-vars.txt
MATCH '^SNAP_LIBRARY_PATH=/var/lib/snapd/lib/gl:/var/lib/snapd/void$' < snap-vars.txt
MATCH '^SNAP_LIBRARY_PATH=/var/lib/snapd/lib/gl:/var/lib/snapd/lib/gl32:/var/lib/snapd/void$' < snap-vars.txt
MATCH '^SNAP_NAME=test-snapd-tools$' < snap-vars.txt
# XXX: probably not something we ought to test
# egrep -q '^SNAP_REEXEC=0$' snap-vars.txt
Expand Down