diff --git a/cmd/nvidia-device-plugin/plugin-manager.go b/cmd/nvidia-device-plugin/plugin-manager.go index 677bc7956f..e2681b5fb2 100644 --- a/cmd/nvidia-device-plugin/plugin-manager.go +++ b/cmd/nvidia-device-plugin/plugin-manager.go @@ -23,6 +23,7 @@ import ( "github.com/NVIDIA/go-nvlib/pkg/nvlib/device" "github.com/NVIDIA/go-nvlib/pkg/nvlib/info" "github.com/NVIDIA/go-nvml/pkg/nvml" + "k8s.io/klog/v2" spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" "github.com/NVIDIA/k8s-device-plugin/internal/cdi" @@ -45,6 +46,9 @@ func GetPlugins(ctx context.Context, infolib info.Interface, nvmllib nvml.Interf return nil, fmt.Errorf("error querying IMEX channels: %w", err) } + resolvedStrategy := resolveStrategy(*config.Flags.DeviceDiscoveryStrategy, infolib) + klog.Infof("Using device discovery strategy: %s", resolvedStrategy) + cdiHandler, err := cdi.New(infolib, nvmllib, devicelib, cdi.WithDeviceListStrategies(deviceListStrategies), cdi.WithDriverRoot(string(driverRoot)), @@ -59,6 +63,7 @@ func GetPlugins(ctx context.Context, infolib info.Interface, nvmllib nvml.Interf cdi.WithMofedEnabled(*config.Flags.MOFEDEnabled), cdi.WithImexChannels(imexChannels), cdi.WithFeatureFlags(o.cdiFeatureFlags.Value()...), + cdi.WithDeviceDiscoveryStrategy(resolvedStrategy), ) if err != nil { return nil, fmt.Errorf("unable to create cdi handler: %v", err) @@ -70,6 +75,7 @@ func GetPlugins(ctx context.Context, infolib info.Interface, nvmllib nvml.Interf plugin.WithDeviceListStrategies(deviceListStrategies), plugin.WithFailOnInitError(*config.Flags.FailOnInitError), plugin.WithImexChannels(imexChannels), + plugin.WithDeviceDiscoveryStrategy(resolvedStrategy), ) if err != nil { return nil, fmt.Errorf("unable to create plugins: %w", err) @@ -81,3 +87,23 @@ func GetPlugins(ctx context.Context, infolib info.Interface, nvmllib nvml.Interf return plugins, nil } + +// resolveStrategy resolves an "auto" device discovery strategy to a concrete +// value based on the detected platform. Non-auto values are returned unchanged. +func resolveStrategy(strategy string, infolib info.Interface) string { + if strategy != "" && strategy != "auto" { + klog.Infof("Using requested device discovery strategy: %s", strategy) + return strategy + } + + platform := infolib.ResolvePlatform() + klog.Infof("Detected platform: %s", platform) + switch platform { + case info.PlatformNVML, info.PlatformWSL: + return "nvml" + case info.PlatformTegra: + return "tegra" + } + klog.Warning("Unsupported platform detected; defaulting to nvml") + return "nvml" +} diff --git a/go.mod b/go.mod index 5fd7126395..44da7c70dd 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/NVIDIA/go-gpuallocator v0.6.0 github.com/NVIDIA/go-nvlib v0.9.1-0.20251202135446-d0f42ba016dd github.com/NVIDIA/go-nvml v0.13.0-1.0.20260212130905-92cf8c963449 - github.com/NVIDIA/nvidia-container-toolkit v1.19.0 + github.com/NVIDIA/nvidia-container-toolkit v1.19.1-0.20260410190841-2c0c91d0ebaa github.com/fsnotify/fsnotify v1.9.0 github.com/google/renameio v1.0.1 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index be9e21cc08..f889b6a7fd 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/NVIDIA/go-nvlib v0.9.1-0.20251202135446-d0f42ba016dd h1:MC1w/VYuo9Zt0 github.com/NVIDIA/go-nvlib v0.9.1-0.20251202135446-d0f42ba016dd/go.mod h1:7mzx9FSdO9fXWP9NKuZmWkCwhkEcSWQFe2tmFwtLb9c= github.com/NVIDIA/go-nvml v0.13.0-1.0.20260212130905-92cf8c963449 h1:UrArFAaPhj9av2yzEN35CvzWw68BeQjp2MaQFUIoJSU= github.com/NVIDIA/go-nvml v0.13.0-1.0.20260212130905-92cf8c963449/go.mod h1:ahi2psRYoa+wYUBIrZPRO+wJs9lcvMhxSSkjjvsJJNQ= -github.com/NVIDIA/nvidia-container-toolkit v1.19.0 h1:iuttW93jclh0+y3RdBw8DR7v6Gfv6h+HjjY87cTyn9M= -github.com/NVIDIA/nvidia-container-toolkit v1.19.0/go.mod h1:4aDFESAMdSDUGqFD6q3CBRaScL1ZNbMVDifD67cPxUw= +github.com/NVIDIA/nvidia-container-toolkit v1.19.1-0.20260410190841-2c0c91d0ebaa h1:/NWUzIuzA33H1hm9ze8RXdlKAsA1LVccZgdnt93MrQg= +github.com/NVIDIA/nvidia-container-toolkit v1.19.1-0.20260410190841-2c0c91d0ebaa/go.mod h1:4aDFESAMdSDUGqFD6q3CBRaScL1ZNbMVDifD67cPxUw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/internal/cdi/cdi.go b/internal/cdi/cdi.go index 7cad67cf4a..3f89167ace 100644 --- a/internal/cdi/cdi.go +++ b/internal/cdi/cdi.go @@ -18,6 +18,7 @@ package cdi import ( "fmt" + "os" "path/filepath" "strings" @@ -55,7 +56,8 @@ type cdiHandler struct { vendor string deviceIDStrategy string - deviceListStrategies spec.DeviceListStrategies + deviceListStrategies spec.DeviceListStrategies + deviceDiscoveryStrategy string // nvcdiFeatureFlags allows finer control over CDI spec generation. nvcdiFeatureFlags []string @@ -86,8 +88,11 @@ func New(infolib info.Interface, nvmllib nvml.Interface, devicelib device.Interf if !c.deviceListStrategies.AnyCDIEnabled() { return &null{}, nil } + if c.deviceDiscoveryStrategy == "" { + return nil, fmt.Errorf("device discovery strategy not set") + } hasNVML, _ := infolib.HasNvml() - if !hasNVML { + if !hasNVML && c.deviceDiscoveryStrategy != "tegra" { klog.Warning("No valid resources detected, creating a null CDI handler") return &null{}, nil } @@ -128,6 +133,13 @@ func New(infolib info.Interface, nvmllib nvml.Interface, devicelib device.Interf nvcdi.WithVendor(c.vendor), } + // On Tegra (CSV mode), the default CSV files live under the driver root. + // Inject driver-root-aware paths explicitly so the nvcdi library does not + // fall back to the hardcoded absolute paths returned by csv.DefaultFileList(). + if c.deviceDiscoveryStrategy == "tegra" { + commonOptions = append(commonOptions, nvcdi.WithCSVFiles(csvFilesForRoot(c.driverRoot))) + } + c.cdilibs = make(map[string]nvcdi.SpecGenerator) c.cdilibs["gpu"], err = nvcdi.New( @@ -177,16 +189,6 @@ func (cdi *cdiHandler) CreateSpecFile() error { for class, cdilib := range cdi.cdilibs { cdi.logger.Infof("Generating CDI spec for resource: %s/%s", cdi.vendor, class) - if class == "gpu" { - ret := cdi.nvmllib.Init() - if ret != nvml.SUCCESS { - return fmt.Errorf("failed to initialize NVML: %v", ret) - } - defer func() { - _ = cdi.nvmllib.Shutdown() - }() - } - spec, err := cdilib.GetSpec() if err != nil { return fmt.Errorf("failed to get CDI spec: %v", err) @@ -268,3 +270,33 @@ func (cdi *cdiHandler) AdditionalDevices() []string { } return devices } + +// defaultCSVMountSpecPath mirrors the constant defined in the nvidia-container-toolkit's +// internal/platform-support/tegra/csv package. +const defaultCSVMountSpecPath = "/etc/nvidia-container-runtime/host-files-for-container.d" + +var defaultCSVFileNames = []string{"devices.csv", "drivers.csv", "l4t.csv"} + +// csvFilesForRoot returns the Tegra CSV file paths to use for CDI spec generation. +// It checks for the CSV directory at the driver root first, then falls back to the +// absolute path. Returns nil if the directory is not found at either location, +// allowing nvcdi to fall back to csv.DefaultFileList(). +func csvFilesForRoot(driverRoot string) []string { + roots := []string{driverRoot} + if driverRoot != "/" { + roots = append(roots, "/") + } + for _, root := range roots { + csvDir := filepath.Join(root, defaultCSVMountSpecPath) + stat, err := os.Stat(csvDir) + if err != nil || !stat.IsDir() { + continue + } + paths := make([]string, len(defaultCSVFileNames)) + for i, f := range defaultCSVFileNames { + paths[i] = filepath.Join(csvDir, f) + } + return paths + } + return nil +} diff --git a/internal/cdi/csv_test.go b/internal/cdi/csv_test.go new file mode 100644 index 0000000000..7cf426f580 --- /dev/null +++ b/internal/cdi/csv_test.go @@ -0,0 +1,68 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package cdi + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCSVFilesForRoot(t *testing.T) { + testCases := []struct { + description string + setup func(t *testing.T) string // returns driverRoot + expectedPaths func(driverRoot string) []string + }{ + { + description: "directory exists at driver root", + setup: func(t *testing.T) string { + root := t.TempDir() + csvDir := filepath.Join(root, defaultCSVMountSpecPath) + require.NoError(t, os.MkdirAll(csvDir, 0755)) + return root + }, + expectedPaths: func(driverRoot string) []string { + csvDir := filepath.Join(driverRoot, defaultCSVMountSpecPath) + return []string{ + filepath.Join(csvDir, "devices.csv"), + filepath.Join(csvDir, "drivers.csv"), + filepath.Join(csvDir, "l4t.csv"), + } + }, + }, + { + description: "directory does not exist at driver root or absolute path", + setup: func(t *testing.T) string { + return t.TempDir() + }, + expectedPaths: func(_ string) []string { + return nil + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + driverRoot := tc.setup(t) + got := csvFilesForRoot(driverRoot) + require.Equal(t, tc.expectedPaths(driverRoot), got) + }) + } +} diff --git a/internal/cdi/options.go b/internal/cdi/options.go index 8faabed355..f9f62cac80 100644 --- a/internal/cdi/options.go +++ b/internal/cdi/options.go @@ -115,3 +115,12 @@ func WithImexChannels(imexChannels imex.Channels) Option { c.imexChannels = imexChannels } } + +// WithDeviceDiscoveryStrategy sets a pre-resolved device discovery strategy. +// When "tegra", the CDI handler uses CSV-based spec generation with +// driver-root-aware file paths. +func WithDeviceDiscoveryStrategy(strategy string) Option { + return func(c *cdiHandler) { + c.deviceDiscoveryStrategy = strategy + } +} diff --git a/internal/plugin/factory.go b/internal/plugin/factory.go index 89b2b7b51d..2fadfbd1fc 100644 --- a/internal/plugin/factory.go +++ b/internal/plugin/factory.go @@ -41,7 +41,8 @@ type options struct { cdiHandler cdi.Interface config *spec.Config - deviceListStrategies spec.DeviceListStrategies + deviceListStrategies spec.DeviceListStrategies + deviceDiscoveryStrategy string imexChannels imex.Channels } @@ -66,6 +67,10 @@ func New(ctx context.Context, infolib info.Interface, nvmllib nvml.Interface, de o.cdiHandler = cdi.NewNullHandler() } + if o.deviceDiscoveryStrategy == "" { + return nil, fmt.Errorf("device discovery strategy not set") + } + resourceManagers, err := o.getResourceManagers() if err != nil { return nil, fmt.Errorf("failed to construct resource managers: %w", err) @@ -86,7 +91,7 @@ func New(ctx context.Context, infolib info.Interface, nvmllib nvml.Interface, de // Each resource manager maps to a specific named extended resource and may // include full GPUs or MIG devices. func (o *options) getResourceManagers() ([]rm.ResourceManager, error) { - strategy := o.resolveStrategy(*o.config.Flags.DeviceDiscoveryStrategy) + strategy := o.deviceDiscoveryStrategy switch strategy { case "nvml": ret := o.nvmllib.Init() @@ -121,18 +126,3 @@ func (o *options) getResourceManagers() ([]rm.ResourceManager, error) { return nil, nil } } - -func (o *options) resolveStrategy(strategy string) string { - if strategy != "" && strategy != "auto" { - return strategy - } - - platform := o.infolib.ResolvePlatform() - switch platform { - case info.PlatformNVML, info.PlatformWSL: - return "nvml" - case info.PlatformTegra: - return "tegra" - } - return strategy -} diff --git a/internal/plugin/options.go b/internal/plugin/options.go index 894b3892f1..b673a95228 100644 --- a/internal/plugin/options.go +++ b/internal/plugin/options.go @@ -76,3 +76,11 @@ func WithImexChannels(imexChannels imex.Channels) Option { m.imexChannels = imexChannels } } + +// WithDeviceDiscoveryStrategy sets a pre-resolved device discovery strategy, +// bypassing the automatic platform-based resolution. +func WithDeviceDiscoveryStrategy(strategy string) Option { + return func(m *options) { + m.deviceDiscoveryStrategy = strategy + } +} diff --git a/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/discover/graphics.go b/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/discover/graphics.go index 4f58771c15..21c9c14db8 100644 --- a/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/discover/graphics.go +++ b/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/discover/graphics.go @@ -63,6 +63,7 @@ func NewGraphicsMountsDiscoverer(logger logger.Interface, driver *root.Driver, h "glvnd/egl_vendor.d/10_nvidia.json", "egl/egl_external_platform.d/15_nvidia_gbm.json", "egl/egl_external_platform.d/10_nvidia_wayland.json", + "egl/egl_external_platform.d/09_nvidia_wayland2.json", "nvidia/nvoptix.bin", "X11/xorg.conf.d/10-nvidia.conf", "X11/xorg.conf.d/nvidia-drm-outputclass.conf", @@ -134,6 +135,7 @@ func newGraphicsLibrariesDiscoverer(logger logger.Interface, driver *root.Driver // have the RM version. Use the *.* pattern to match X.Y.Z versions. "libnvidia-egl-gbm.so.*.*", "libnvidia-egl-wayland.so.*.*", + "libnvidia-egl-wayland2.so.*.*", // We include the following libraries to have them available for // symlink creation below: // If CDI injection is used, these should already be detected as: diff --git a/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/edits/device.go b/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/edits/device.go index 8ce05c00b3..2b12e0492f 100644 --- a/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/edits/device.go +++ b/vendor/github.com/NVIDIA/nvidia-container-toolkit/internal/edits/device.go @@ -130,9 +130,6 @@ func (d *device) getAdditionalGIDs(dn *specs.DeviceNode) []uint32 { if dn.FileMode == nil { return nil } - if dn.FileMode.Type()&os.ModeCharDevice == 0 { - return nil - } if permission := dn.FileMode.Perm(); isWorldReadable(permission) && isWorldWriteable(permission) { return nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index 4c9b3a574e..fbbe84c0d2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -20,7 +20,7 @@ github.com/NVIDIA/go-nvlib/pkg/pciids ## explicit; go 1.20 github.com/NVIDIA/go-nvml/pkg/dl github.com/NVIDIA/go-nvml/pkg/nvml -# github.com/NVIDIA/nvidia-container-toolkit v1.19.0 +# github.com/NVIDIA/nvidia-container-toolkit v1.19.1-0.20260410190841-2c0c91d0ebaa ## explicit; go 1.25.0 github.com/NVIDIA/nvidia-container-toolkit/internal/config/image github.com/NVIDIA/nvidia-container-toolkit/internal/devices