From f27fd1439556a43a64182d98b3ffd90cb4694dff Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 25 Mar 2026 19:50:56 +0100 Subject: [PATCH 1/4] Fix CDI spec generation to respect driver root for Tegra CSV files On Tegra-based systems the nvcdi library resolves CSV mount specs from /etc/nvidia-container-runtime/host-files-for-container.d using hardcoded absolute paths, ignoring the configured driver root. When the device plugin runs in a container with the host filesystem mounted at a non-default driver root the CSV files are unreachable at their absolute paths, causing CDI spec generation to fail. Inject driver-root-aware CSV file paths via nvcdi.WithCSVFiles() when the platform is detected as Tegra. The helper csvFilesForRoot() checks for the CSV directory under the driver root first, falling back to the absolute path, and returns nil (letting nvcdi use its own defaults) if neither location exists. Also extend the NVML guard so that pure iGPU Tegra systems without NVML are not incorrectly reduced to a null CDI handler. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Evan Lezar --- internal/cdi/cdi.go | 51 +++++++++++++++++++++++------- internal/cdi/csv_test.go | 68 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 11 deletions(-) create mode 100644 internal/cdi/csv_test.go diff --git a/internal/cdi/cdi.go b/internal/cdi/cdi.go index 7cad67cf4a..94ce377dc5 100644 --- a/internal/cdi/cdi.go +++ b/internal/cdi/cdi.go @@ -18,6 +18,7 @@ package cdi import ( "fmt" + "os" "path/filepath" "strings" @@ -87,7 +88,8 @@ func New(infolib info.Interface, nvmllib nvml.Interface, devicelib device.Interf return &null{}, nil } hasNVML, _ := infolib.HasNvml() - if !hasNVML { + platform := infolib.ResolvePlatform() + if !hasNVML && platform != info.PlatformTegra { klog.Warning("No valid resources detected, creating a null CDI handler") return &null{}, nil } @@ -128,6 +130,13 @@ func New(infolib info.Interface, nvmllib nvml.Interface, devicelib device.Interf nvcdi.WithVendor(c.vendor), } + // On Tegra (CSV mode), the default CSV files live under the driver root. + // Inject driver-root-aware paths explicitly so the nvcdi library does not + // fall back to the hardcoded absolute paths returned by csv.DefaultFileList(). + if platform == info.PlatformTegra { + commonOptions = append(commonOptions, nvcdi.WithCSVFiles(csvFilesForRoot(c.driverRoot))) + } + c.cdilibs = make(map[string]nvcdi.SpecGenerator) c.cdilibs["gpu"], err = nvcdi.New( @@ -177,16 +186,6 @@ func (cdi *cdiHandler) CreateSpecFile() error { for class, cdilib := range cdi.cdilibs { cdi.logger.Infof("Generating CDI spec for resource: %s/%s", cdi.vendor, class) - if class == "gpu" { - ret := cdi.nvmllib.Init() - if ret != nvml.SUCCESS { - return fmt.Errorf("failed to initialize NVML: %v", ret) - } - defer func() { - _ = cdi.nvmllib.Shutdown() - }() - } - spec, err := cdilib.GetSpec() if err != nil { return fmt.Errorf("failed to get CDI spec: %v", err) @@ -268,3 +267,33 @@ func (cdi *cdiHandler) AdditionalDevices() []string { } return devices } + +// defaultCSVMountSpecPath mirrors the constant defined in the nvidia-container-toolkit's +// internal/platform-support/tegra/csv package. +const defaultCSVMountSpecPath = "/etc/nvidia-container-runtime/host-files-for-container.d" + +var defaultCSVFileNames = []string{"devices.csv", "drivers.csv", "l4t.csv"} + +// csvFilesForRoot returns the Tegra CSV file paths to use for CDI spec generation. +// It checks for the CSV directory at the driver root first, then falls back to the +// absolute path. Returns nil if the directory is not found at either location, +// allowing nvcdi to fall back to csv.DefaultFileList(). +func csvFilesForRoot(driverRoot string) []string { + roots := []string{driverRoot} + if driverRoot != "/" { + roots = append(roots, "/") + } + for _, root := range roots { + csvDir := filepath.Join(root, defaultCSVMountSpecPath) + stat, err := os.Stat(csvDir) + if err != nil || !stat.IsDir() { + continue + } + paths := make([]string, len(defaultCSVFileNames)) + for i, f := range defaultCSVFileNames { + paths[i] = filepath.Join(csvDir, f) + } + return paths + } + return nil +} diff --git a/internal/cdi/csv_test.go b/internal/cdi/csv_test.go new file mode 100644 index 0000000000..7cf426f580 --- /dev/null +++ b/internal/cdi/csv_test.go @@ -0,0 +1,68 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package cdi + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCSVFilesForRoot(t *testing.T) { + testCases := []struct { + description string + setup func(t *testing.T) string // returns driverRoot + expectedPaths func(driverRoot string) []string + }{ + { + description: "directory exists at driver root", + setup: func(t *testing.T) string { + root := t.TempDir() + csvDir := filepath.Join(root, defaultCSVMountSpecPath) + require.NoError(t, os.MkdirAll(csvDir, 0755)) + return root + }, + expectedPaths: func(driverRoot string) []string { + csvDir := filepath.Join(driverRoot, defaultCSVMountSpecPath) + return []string{ + filepath.Join(csvDir, "devices.csv"), + filepath.Join(csvDir, "drivers.csv"), + filepath.Join(csvDir, "l4t.csv"), + } + }, + }, + { + description: "directory does not exist at driver root or absolute path", + setup: func(t *testing.T) string { + return t.TempDir() + }, + expectedPaths: func(_ string) []string { + return nil + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + driverRoot := tc.setup(t) + got := csvFilesForRoot(driverRoot) + require.Equal(t, tc.expectedPaths(driverRoot), got) + }) + } +} From 22499fe306f1849ff3971337b59fba5952561f4c Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 26 Mar 2026 09:16:14 +0100 Subject: [PATCH 2/4] Factor device discovery strategy resolution out of plugin and cdi packages Resolve the device discovery strategy once in GetPlugins() using a local resolveStrategy() helper and pass the concrete value ("nvml", "tegra", etc.) to both cdi.New() and plugin.New() via new WithDeviceDiscoveryStrategy options. This eliminates duplicate infolib.ResolvePlatform() calls and makes the dependency on platform detection explicit at the call site. Both constructors now return an error when the strategy is not set rather than falling back to their own resolution logic. The CDI handler uses the strategy directly to gate CSV-mode behaviour instead of re-checking the platform independently. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Evan Lezar --- cmd/nvidia-device-plugin/plugin-manager.go | 25 ++++++++++++++++++++++ internal/cdi/cdi.go | 11 ++++++---- internal/cdi/options.go | 9 ++++++++ internal/plugin/factory.go | 24 ++++++--------------- internal/plugin/options.go | 8 +++++++ 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/cmd/nvidia-device-plugin/plugin-manager.go b/cmd/nvidia-device-plugin/plugin-manager.go index 677bc7956f..671bc4519f 100644 --- a/cmd/nvidia-device-plugin/plugin-manager.go +++ b/cmd/nvidia-device-plugin/plugin-manager.go @@ -23,6 +23,7 @@ import ( "github.com/NVIDIA/go-nvlib/pkg/nvlib/device" "github.com/NVIDIA/go-nvlib/pkg/nvlib/info" "github.com/NVIDIA/go-nvml/pkg/nvml" + "k8s.io/klog/v2" spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" "github.com/NVIDIA/k8s-device-plugin/internal/cdi" @@ -45,6 +46,9 @@ func GetPlugins(ctx context.Context, infolib info.Interface, nvmllib nvml.Interf return nil, fmt.Errorf("error querying IMEX channels: %w", err) } + resolvedStrategy := resolveStrategy(*config.Flags.DeviceDiscoveryStrategy, infolib) + klog.Infof("Using device discovery strategy: %s", resolvedStrategy) + cdiHandler, err := cdi.New(infolib, nvmllib, devicelib, cdi.WithDeviceListStrategies(deviceListStrategies), cdi.WithDriverRoot(string(driverRoot)), @@ -59,6 +63,7 @@ func GetPlugins(ctx context.Context, infolib info.Interface, nvmllib nvml.Interf cdi.WithMofedEnabled(*config.Flags.MOFEDEnabled), cdi.WithImexChannels(imexChannels), cdi.WithFeatureFlags(o.cdiFeatureFlags.Value()...), + cdi.WithDeviceDiscoveryStrategy(resolvedStrategy), ) if err != nil { return nil, fmt.Errorf("unable to create cdi handler: %v", err) @@ -70,6 +75,7 @@ func GetPlugins(ctx context.Context, infolib info.Interface, nvmllib nvml.Interf plugin.WithDeviceListStrategies(deviceListStrategies), plugin.WithFailOnInitError(*config.Flags.FailOnInitError), plugin.WithImexChannels(imexChannels), + plugin.WithDeviceDiscoveryStrategy(resolvedStrategy), ) if err != nil { return nil, fmt.Errorf("unable to create plugins: %w", err) @@ -81,3 +87,22 @@ func GetPlugins(ctx context.Context, infolib info.Interface, nvmllib nvml.Interf return plugins, nil } + +// resolveStrategy resolves an "auto" device discovery strategy to a concrete +// value based on the detected platform. Non-auto values are returned unchanged. +func resolveStrategy(strategy string, infolib info.Interface) string { + if strategy != "" && strategy != "auto" { + klog.Infof("Using requested device discovery strategy: %s", strategy) + return strategy + } + + platform := infolib.ResolvePlatform() + klog.Infof("Detected platform: %s", platform) + switch platform { + case info.PlatformNVML, info.PlatformWSL: + return "nvml" + case info.PlatformTegra: + return "tegra" + } + return strategy +} diff --git a/internal/cdi/cdi.go b/internal/cdi/cdi.go index 94ce377dc5..3f89167ace 100644 --- a/internal/cdi/cdi.go +++ b/internal/cdi/cdi.go @@ -56,7 +56,8 @@ type cdiHandler struct { vendor string deviceIDStrategy string - deviceListStrategies spec.DeviceListStrategies + deviceListStrategies spec.DeviceListStrategies + deviceDiscoveryStrategy string // nvcdiFeatureFlags allows finer control over CDI spec generation. nvcdiFeatureFlags []string @@ -87,9 +88,11 @@ func New(infolib info.Interface, nvmllib nvml.Interface, devicelib device.Interf if !c.deviceListStrategies.AnyCDIEnabled() { return &null{}, nil } + if c.deviceDiscoveryStrategy == "" { + return nil, fmt.Errorf("device discovery strategy not set") + } hasNVML, _ := infolib.HasNvml() - platform := infolib.ResolvePlatform() - if !hasNVML && platform != info.PlatformTegra { + if !hasNVML && c.deviceDiscoveryStrategy != "tegra" { klog.Warning("No valid resources detected, creating a null CDI handler") return &null{}, nil } @@ -133,7 +136,7 @@ func New(infolib info.Interface, nvmllib nvml.Interface, devicelib device.Interf // On Tegra (CSV mode), the default CSV files live under the driver root. // Inject driver-root-aware paths explicitly so the nvcdi library does not // fall back to the hardcoded absolute paths returned by csv.DefaultFileList(). - if platform == info.PlatformTegra { + if c.deviceDiscoveryStrategy == "tegra" { commonOptions = append(commonOptions, nvcdi.WithCSVFiles(csvFilesForRoot(c.driverRoot))) } diff --git a/internal/cdi/options.go b/internal/cdi/options.go index 8faabed355..f9f62cac80 100644 --- a/internal/cdi/options.go +++ b/internal/cdi/options.go @@ -115,3 +115,12 @@ func WithImexChannels(imexChannels imex.Channels) Option { c.imexChannels = imexChannels } } + +// WithDeviceDiscoveryStrategy sets a pre-resolved device discovery strategy. +// When "tegra", the CDI handler uses CSV-based spec generation with +// driver-root-aware file paths. +func WithDeviceDiscoveryStrategy(strategy string) Option { + return func(c *cdiHandler) { + c.deviceDiscoveryStrategy = strategy + } +} diff --git a/internal/plugin/factory.go b/internal/plugin/factory.go index 89b2b7b51d..2fadfbd1fc 100644 --- a/internal/plugin/factory.go +++ b/internal/plugin/factory.go @@ -41,7 +41,8 @@ type options struct { cdiHandler cdi.Interface config *spec.Config - deviceListStrategies spec.DeviceListStrategies + deviceListStrategies spec.DeviceListStrategies + deviceDiscoveryStrategy string imexChannels imex.Channels } @@ -66,6 +67,10 @@ func New(ctx context.Context, infolib info.Interface, nvmllib nvml.Interface, de o.cdiHandler = cdi.NewNullHandler() } + if o.deviceDiscoveryStrategy == "" { + return nil, fmt.Errorf("device discovery strategy not set") + } + resourceManagers, err := o.getResourceManagers() if err != nil { return nil, fmt.Errorf("failed to construct resource managers: %w", err) @@ -86,7 +91,7 @@ func New(ctx context.Context, infolib info.Interface, nvmllib nvml.Interface, de // Each resource manager maps to a specific named extended resource and may // include full GPUs or MIG devices. func (o *options) getResourceManagers() ([]rm.ResourceManager, error) { - strategy := o.resolveStrategy(*o.config.Flags.DeviceDiscoveryStrategy) + strategy := o.deviceDiscoveryStrategy switch strategy { case "nvml": ret := o.nvmllib.Init() @@ -121,18 +126,3 @@ func (o *options) getResourceManagers() ([]rm.ResourceManager, error) { return nil, nil } } - -func (o *options) resolveStrategy(strategy string) string { - if strategy != "" && strategy != "auto" { - return strategy - } - - platform := o.infolib.ResolvePlatform() - switch platform { - case info.PlatformNVML, info.PlatformWSL: - return "nvml" - case info.PlatformTegra: - return "tegra" - } - return strategy -} diff --git a/internal/plugin/options.go b/internal/plugin/options.go index 894b3892f1..b673a95228 100644 --- a/internal/plugin/options.go +++ b/internal/plugin/options.go @@ -76,3 +76,11 @@ func WithImexChannels(imexChannels imex.Channels) Option { m.imexChannels = imexChannels } } + +// WithDeviceDiscoveryStrategy sets a pre-resolved device discovery strategy, +// bypassing the automatic platform-based resolution. +func WithDeviceDiscoveryStrategy(strategy string) Option { + return func(m *options) { + m.deviceDiscoveryStrategy = strategy + } +} From 6ca73d9e536eeb59707399e552fbf2400edec100 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 26 Mar 2026 09:22:38 +0100 Subject: [PATCH 3/4] Default to nvml strategy when platform cannot be detected When the device discovery strategy is "auto" but no supported platform is detected, default to "nvml" instead of returning the unresolved "auto" value. Log a warning so that the fallback is visible in the device plugin output. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Evan Lezar --- cmd/nvidia-device-plugin/plugin-manager.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/nvidia-device-plugin/plugin-manager.go b/cmd/nvidia-device-plugin/plugin-manager.go index 671bc4519f..e2681b5fb2 100644 --- a/cmd/nvidia-device-plugin/plugin-manager.go +++ b/cmd/nvidia-device-plugin/plugin-manager.go @@ -104,5 +104,6 @@ func resolveStrategy(strategy string, infolib info.Interface) string { case info.PlatformTegra: return "tegra" } - return strategy + klog.Warning("Unsupported platform detected; defaulting to nvml") + return "nvml" } From fe18b02e2a5f6d41e825f9e1dac71a329948c6f9 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 26 Mar 2026 13:51:51 +0100 Subject: [PATCH 4/4] Update nvcdi API to 2c0c91d0ebaa45873d997b42beffad3706e8d734 This change updates the nvcdi API to 2c0c91d0ebaa45873d997b42beffad3706e8d734 to include the changes for https://github.com/NVIDIA/nvidia-container-toolkit/pull/1745. Signed-off-by: Evan Lezar --- go.mod | 2 +- go.sum | 4 ++-- .../nvidia-container-toolkit/internal/discover/graphics.go | 2 ++ .../NVIDIA/nvidia-container-toolkit/internal/edits/device.go | 3 --- vendor/modules.txt | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 5fd7126395..44da7c70dd 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/NVIDIA/go-gpuallocator v0.6.0 github.com/NVIDIA/go-nvlib v0.9.1-0.20251202135446-d0f42ba016dd github.com/NVIDIA/go-nvml v0.13.0-1.0.20260212130905-92cf8c963449 - github.com/NVIDIA/nvidia-container-toolkit v1.19.0 + github.com/NVIDIA/nvidia-container-toolkit v1.19.1-0.20260410190841-2c0c91d0ebaa github.com/fsnotify/fsnotify v1.9.0 github.com/google/renameio v1.0.1 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index be9e21cc08..f889b6a7fd 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/NVIDIA/go-nvlib v0.9.1-0.20251202135446-d0f42ba016dd h1:MC1w/VYuo9Zt0 github.com/NVIDIA/go-nvlib v0.9.1-0.20251202135446-d0f42ba016dd/go.mod h1:7mzx9FSdO9fXWP9NKuZmWkCwhkEcSWQFe2tmFwtLb9c= github.com/NVIDIA/go-nvml v0.13.0-1.0.20260212130905-92cf8c963449 h1:UrArFAaPhj9av2yzEN35CvzWw68BeQjp2MaQFUIoJSU= github.com/NVIDIA/go-nvml v0.13.0-1.0.20260212130905-92cf8c963449/go.mod h1:ahi2psRYoa+wYUBIrZPRO+wJs9lcvMhxSSkjjvsJJNQ= -github.com/NVIDIA/nvidia-container-toolkit v1.19.0 h1:iuttW93jclh0+y3RdBw8DR7v6Gfv6h+HjjY87cTyn9M= -github.com/NVIDIA/nvidia-container-toolkit v1.19.0/go.mod h1:4aDFESAMdSDUGqFD6q3CBRaScL1ZNbMVDifD67cPxUw= +github.com/NVIDIA/nvidia-container-toolkit v1.19.1-0.20260410190841-2c0c91d0ebaa h1:/NWUzIuzA33H1hm9ze8RXdlKAsA1LVccZgdnt93MrQg= +github.com/NVIDIA/nvidia-container-toolkit v1.19.1-0.20260410190841-2c0c91d0ebaa/go.mod h1:4aDFESAMdSDUGqFD6q3CBRaScL1ZNbMVDifD67cPxUw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/discover/graphics.go b/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/discover/graphics.go index 4f58771c15..21c9c14db8 100644 --- a/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/discover/graphics.go +++ b/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/discover/graphics.go @@ -63,6 +63,7 @@ func NewGraphicsMountsDiscoverer(logger logger.Interface, driver *root.Driver, h "glvnd/egl_vendor.d/10_nvidia.json", "egl/egl_external_platform.d/15_nvidia_gbm.json", "egl/egl_external_platform.d/10_nvidia_wayland.json", + "egl/egl_external_platform.d/09_nvidia_wayland2.json", "nvidia/nvoptix.bin", "X11/xorg.conf.d/10-nvidia.conf", "X11/xorg.conf.d/nvidia-drm-outputclass.conf", @@ -134,6 +135,7 @@ func newGraphicsLibrariesDiscoverer(logger logger.Interface, driver *root.Driver // have the RM version. Use the *.* pattern to match X.Y.Z versions. "libnvidia-egl-gbm.so.*.*", "libnvidia-egl-wayland.so.*.*", + "libnvidia-egl-wayland2.so.*.*", // We include the following libraries to have them available for // symlink creation below: // If CDI injection is used, these should already be detected as: diff --git a/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/edits/device.go b/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/edits/device.go index 8ce05c00b3..2b12e0492f 100644 --- a/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/edits/device.go +++ b/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/edits/device.go @@ -130,9 +130,6 @@ func (d *device) getAdditionalGIDs(dn *specs.DeviceNode) []uint32 { if dn.FileMode == nil { return nil } - if dn.FileMode.Type()&os.ModeCharDevice == 0 { - return nil - } if permission := dn.FileMode.Perm(); isWorldReadable(permission) && isWorldWriteable(permission) { return nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index 4c9b3a574e..fbbe84c0d2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -20,7 +20,7 @@ github.com/NVIDIA/go-nvlib/pkg/pciids ## explicit; go 1.20 github.com/NVIDIA/go-nvml/pkg/dl github.com/NVIDIA/go-nvml/pkg/nvml -# github.com/NVIDIA/nvidia-container-toolkit v1.19.0 +# github.com/NVIDIA/nvidia-container-toolkit v1.19.1-0.20260410190841-2c0c91d0ebaa ## explicit; go 1.25.0 github.com/NVIDIA/nvidia-container-toolkit/internal/config/image github.com/NVIDIA/nvidia-container-toolkit/internal/devices