diff --git a/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/charts/hami/templates/device-plugin/daemonsetnvidia.yaml index 0e785c363b..56d74fb071 100644 --- a/charts/hami/templates/device-plugin/daemonsetnvidia.yaml +++ b/charts/hami/templates/device-plugin/daemonsetnvidia.yaml @@ -92,6 +92,10 @@ spec: value: all - name: DEVICE_LIST_STRATEGY value: {{ .Values.devicePlugin.deviceListStrategy }} + {{- if .Values.devicePlugin.deviceDiscoveryStrategy }} + - name: DEVICE_DISCOVERY_STRATEGY + value: {{ .Values.devicePlugin.deviceDiscoveryStrategy }} + {{- end }} - name: HOOK_PATH value: {{ .Values.global.gpuHookPath }} {{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }} diff --git a/charts/hami/templates/scheduler/device-configmap.yaml b/charts/hami/templates/scheduler/device-configmap.yaml index 7f0b2b254d..f11fb6c79c 100644 --- a/charts/hami/templates/scheduler/device-configmap.yaml +++ b/charts/hami/templates/scheduler/device-configmap.yaml @@ -22,6 +22,9 @@ data: defaultCores: 0 defaultGPUNum: 1 preConfiguredDeviceMemory: {{ .Values.devicePlugin.preConfiguredDeviceMemory | default 0 }} + {{- if .Values.devicePlugin.preConfiguredDeviceType }} + preConfiguredDeviceType: "{{ .Values.devicePlugin.preConfiguredDeviceType }}" + {{- end }} memoryFactor: 1 deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} diff --git a/charts/hami/values.yaml b/charts/hami/values.yaml index ed1a818f64..3f1ac89333 100644 --- a/charts/hami/values.yaml +++ b/charts/hami/values.yaml @@ -350,6 +350,9 @@ devicePlugin: # Set to 0 to use auto-detection (default). For unified memory GPUs, set to the total GPU memory (e.g., 131072 for 128GB). # Can be overridden per-node via nodeConfiguration.config. preConfiguredDeviceMemory: 0 + # Device type reported for GPUs discovered via CDI that cannot be enumerated by NVML (e.g., NVIDIA GB10). + # Only used with deviceDiscoveryStrategy=cdi. Empty uses the built-in default ("NVIDIA-GB10"). + preConfiguredDeviceType: "" # Node configuration for device plugin, Priority: externalConfigName > config > default config nodeConfiguration: # If you want to use a custom config.json, you can set the content here. @@ -384,6 +387,12 @@ devicePlugin: disablecorelimit: "false" passDeviceSpecsEnabled: false deviceListStrategy: "envvar" + # Strategy used by the NVIDIA device plugin to discover devices: + # "" / "auto" (default) auto-detects the platform; "nvml", "tegra", or "cdi". + # Use "cdi" (together with deviceListStrategy: "cdi-annotations,cdi-cri") for + # CDI-only accelerators such as the GB10 (Grace-Blackwell iGPU), which are not + # enumerable via NVML and are described by an on-node CDI spec instead. + deviceDiscoveryStrategy: "" nvidiaHookPath: null nvidiaDriverRoot: null gdrcopyEnabled: null diff --git a/cmd/device-plugin/nvidia/main.go b/cmd/device-plugin/nvidia/main.go index c91454bf8a..402b8132d4 100644 --- a/cmd/device-plugin/nvidia/main.go +++ b/cmd/device-plugin/nvidia/main.go @@ -175,7 +175,7 @@ func main() { &cli.StringFlag{ Name: "device-discovery-strategy", Value: "auto", - Usage: "the strategy to use to discover devices: 'auto', 'nvml', or 'tegra'", + Usage: "the strategy to use to discover devices: 'auto', 'nvml', 'tegra', or 'cdi'", EnvVars: []string{"DEVICE_DISCOVERY_STRATEGY"}, }, &cli.IntSliceFlag{ @@ -209,9 +209,16 @@ func validateFlags(infolib nvinfo.Interface, config *spec.Config) error { return fmt.Errorf("invalid --device-list-strategy option: %v", err) } + // CDI device-list strategies normally require NVML to generate the CDI spec. + // The exception is the "cdi" discovery strategy (or "auto", which falls back + // to CDI discovery), where devices are described by externally-managed CDI + // specs on the node — this is how CDI-only accelerators such as the GB10 + // (Grace-Blackwell iGPU) are supported without NVML. hasNvml, _ := infolib.HasNvml() - if deviceListStrategies.AnyCDIEnabled() && !hasNvml { - return fmt.Errorf("CDI --device-list-strategy options are only supported on NVML-based systems") + discoveryStrategy := *config.Flags.DeviceDiscoveryStrategy + cdiDiscovery := discoveryStrategy == "cdi" || discoveryStrategy == "auto" + if deviceListStrategies.AnyCDIEnabled() && !hasNvml && !cdiDiscovery { + return fmt.Errorf("CDI --device-list-strategy options are only supported on NVML-based systems or with --device-discovery-strategy=cdi") } if *config.Flags.Plugin.DeviceIDStrategy != spec.DeviceIDStrategyUUID && *config.Flags.Plugin.DeviceIDStrategy != spec.DeviceIDStrategyIndex { @@ -231,6 +238,7 @@ func validateFlags(infolib nvinfo.Interface, config *spec.Config) error { case "auto": case "nvml": case "tegra": + case "cdi": default: return fmt.Errorf("invalid --device-discovery-strategy option %v", *config.Flags.DeviceDiscoveryStrategy) } diff --git a/cmd/device-plugin/nvidia/plugin-manager.go b/cmd/device-plugin/nvidia/plugin-manager.go index a2d340606b..a09d07eb1b 100644 --- a/cmd/device-plugin/nvidia/plugin-manager.go +++ b/cmd/device-plugin/nvidia/plugin-manager.go @@ -54,7 +54,7 @@ func GetPlugins(ctx context.Context, infolib info.Interface, nvmllib nvml.Interf cdi.WithTargetDevRoot(*config.Flags.NvidiaDevRoot), cdi.WithNvidiaCTKPath(*config.Flags.Plugin.NvidiaCTKPath), cdi.WithDeviceIDStrategy(*config.Flags.Plugin.DeviceIDStrategy), - cdi.WithVendor("k8s.device-plugin.nvidia.com"), + cdi.WithVendor(cdi.DefaultVendor), cdi.WithGdrcopyEnabled(*config.Flags.GDRCopyEnabled), cdi.WithGdsEnabled(*config.Flags.GDSEnabled), cdi.WithMofedEnabled(*config.Flags.MOFEDEnabled), diff --git a/docs/nvidia-gb10-support.md b/docs/nvidia-gb10-support.md new file mode 100644 index 0000000000..000539bb98 --- /dev/null +++ b/docs/nvidia-gb10-support.md @@ -0,0 +1,127 @@ +# NVIDIA GB10 (Grace-Blackwell iGPU) Support + +## Introduction + +The NVIDIA GB10 (Grace-Blackwell "superchip", as shipped in the DGX Spark) is an +**integrated GPU** that shares unified LPDDR5X memory with the Grace CPU over a +coherent NVLink-C2C link. It differs from a discrete GPU in two ways that affect +HAMi's device plugin: + +1. **No dedicated framebuffer.** `nvmlDeviceGetMemoryInfo()` returns + `ERROR_NOT_SUPPORTED` because there is no fixed device-local memory pool to + report — the "GPU memory" is system memory allocated dynamically. +2. **CDI-only exposure.** The device is provisioned through the NVIDIA Container + Toolkit's [CDI](https://github.com/cncf-tags/container-device-interface) + mechanism. It is described by an on-node CDI spec + (`/var/run/cdi/k8s.device-plugin.nvidia.com-gpu.json`) and is **not** + enumerable via NVML inside the device-plugin container. + +Because HAMi's device plugin historically discovered devices only through NVML, +on a GB10 node `ResolvePlatform()` returns `unknown` and the plugin exits with: + +```text +factory.go] Incompatible strategy detected auto +main.go] error starting plugins: ... failed to construct resource managers: + invalid device discovery strategy +``` + +To support these accelerators, the NVIDIA device plugin can discover GPUs +directly from the node's CDI specs (no NVML required) using the `cdi` +device-discovery strategy. + +## Prerequisites + +- The **NVIDIA GPU Operator** (or NVIDIA Container Toolkit) is installed and has + generated the CDI spec at `/var/run/cdi/k8s.device-plugin.nvidia.com-gpu.json`. + You can verify the node's own device plugin advertises `nvidia.com/gpu` and + that GPU Feature Discovery has labelled the node, e.g. + `nvidia.com/gpu.product=NVIDIA-GB10`. +- The node carries the label HAMi's device-plugin DaemonSet selects on + (`gpu=on` by default): + + ```bash + kubectl label node gpu=on + ``` + +- The kernel's inotify limits are high enough for the plugin's filesystem + watcher. Busy nodes can exhaust the default `fs.inotify.max_user_instances` + (128), which makes the plugin fail at start-up with + `couldn't initialize inotify: too many open files`. Raise it persistently: + + ```bash + # /etc/sysctl.d/99-hami-inotify.conf + fs.inotify.max_user_instances = 8192 + fs.inotify.max_user_watches = 524288 + ``` + + ```bash + sudo sysctl --system + ``` + +## Configuration + +Set the following on the NVIDIA device plugin (Helm `values.yaml`): + +```yaml +devicePlugin: + # Discover devices from the on-node CDI specs instead of via NVML. + deviceDiscoveryStrategy: "cdi" + # Inject the device through CDI (matches the NVIDIA GPU Operator plugin). + deviceListStrategy: "cdi-annotations,cdi-cri" + # Unified memory (in MiB) HAMi should treat as schedulable per GPU. NVML + # cannot report it on a unified-memory GPU, so it is a policy value — set it + # at or below the node's total unified memory, leaving headroom for the OS. + # See the note below for deriving it from the node (the example is not a + # direct GiB conversion). + preConfiguredDeviceMemory: 122566 + # Optional. Device type recorded for scheduling/`use-gputype`. + # Defaults to "NVIDIA-GB10" when empty. + preConfiguredDeviceType: "NVIDIA-GB10" +``` + +Notes: + +- `deviceDiscoveryStrategy` also accepts `auto` (the default), which falls back + to `cdi` automatically when no NVML/Tegra platform is detected **and** a CDI + device-list strategy is active and CDI specs are present. Setting it to `cdi` + explicitly is recommended for clarity on GB10 nodes. +- `preConfiguredDeviceMemory` is a scheduling **policy** value — how much unified + memory HAMi treats as schedulable per GPU — interpreted as MiB (HAMi multiplies + it by 1024×1024 to get bytes). It is **not** a hardware GPU-memory readout: on a + unified-memory device the memory is shared with the CPU/OS. The example + `122566` is **this DGX Spark's total system memory as reported by Kubernetes**, + not a direct GiB conversion (120 GiB would be 122880 MiB): + + ```bash + kubectl get node -o jsonpath='{.status.capacity.memory}' + # 125506464Ki -> 125506464 / 1024 = 122566 MiB (~119.7 GiB) + ``` + + Choose a value at or below this, leaving headroom for the OS. + It can be overridden per node via `nodeConfiguration.config` (`preconfigureddevicememory`). +- `preConfiguredDeviceType` can be overridden per node via `preconfigureddevicetype`. + +## How it works + +1. **Discovery** — the plugin reads the CDI specs under `/etc/cdi` and + `/var/run/cdi`, enumerating GPU devices for vendor + `k8s.device-plugin.nvidia.com`, class `gpu` (the `all` meta-device is + excluded). Each CDI device becomes a schedulable GPU keyed by its CDI device + name (a GPU UUID). +2. **Registration** — because NVML is unavailable, per-GPU memory and type come + from `preConfiguredDeviceMemory` / `preConfiguredDeviceType` instead of being + queried from the driver. +3. **Allocation** — on `Allocate`, the plugin emits the CDI device reference + (`k8s.device-plugin.nvidia.com/gpu=`) via the `cdi-annotations` / + `cdi-cri` device-list strategy, so containerd injects the device. HAMi's + memory/core limiting (libvgpu) is applied on top as usual. + +## Limitations + +- Because there is no per-device NVML data, **all** GPUs discovered in `cdi` + mode share the same `preConfiguredDeviceMemory` and `preConfiguredDeviceType`. + This targets homogeneous CDI-only nodes (e.g. a single-GB10 DGX Spark). +- Health checks and MIG are not available in `cdi` mode (both require NVML). +- GPU topology scoring (`ENABLE_TOPOLOGY_SCORE`) is skipped in `cdi` mode: it + computes pairwise P2P/NVLink scores via NVML, which is unavailable and + meaningless for a single CDI-only iGPU. diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/cdi/cdi.go b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/cdi.go index a29871e457..e08153e306 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/cdi/cdi.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/cdi.go @@ -49,6 +49,7 @@ import ( cdiparser "tags.cncf.io/container-device-interface/pkg/parser" spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" + "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/imex" ) @@ -101,8 +102,12 @@ func New(infolib info.Interface, nvmllib nvml.Interface, devicelib device.Interf } hasNVML, _ := infolib.HasNvml() if !hasNVML { - klog.Warning("No valid resources detected, creating a null CDI handler") - return &null{}, nil + // NVML is unavailable but a CDI device-list strategy is requested. This + // is the case for CDI-only accelerators such as the GB10 (Grace-Blackwell + // iGPU), whose device is described by an externally-managed CDI spec on + // the node. Return a handler that can reference those specs for injection. + klog.Warning("NVML not detected; using externally-managed CDI specs for device injection") + return newExternalHandler(c.vendor), nil } if c.logger == nil { diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/cdi/cdi_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/cdi_test.go new file mode 100644 index 0000000000..cbe6c9ee7b --- /dev/null +++ b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/cdi_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cdi + +import ( + "testing" + + "github.com/NVIDIA/go-nvlib/pkg/nvlib/info" + spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" + "github.com/stretchr/testify/require" +) + +// fakeInfo is a minimal info.Interface used to drive the New() branches. +type fakeInfo struct { + platform info.Platform + hasNVML bool +} + +func (f fakeInfo) ResolvePlatform() info.Platform { return f.platform } +func (f fakeInfo) HasDXCore() (bool, string) { return false, "" } +func (f fakeInfo) HasNvml() (bool, string) { return f.hasNVML, "" } +func (f fakeInfo) HasTegraFiles() (bool, string) { return false, "" } +func (f fakeInfo) HasAnIntegratedGPU() (bool, string) { return false, "" } + +func mustStrategies(t *testing.T, s ...string) spec.DeviceListStrategies { + t.Helper() + ds, err := spec.NewDeviceListStrategies(s) + require.NoError(t, err) + return ds +} + +// When no CDI device-list strategy is enabled, New returns the null handler, +// whose QualifiedName yields an empty string. +func TestNew_NoCDIEnabled_ReturnsNull(t *testing.T) { + h, err := New(fakeInfo{hasNVML: false}, nil, nil, + WithDeviceListStrategies(mustStrategies(t, "envvar"))) + require.NoError(t, err) + require.Empty(t, h.QualifiedName("gpu", "0")) + require.NoError(t, h.CreateSpecFile()) +} + +// When a CDI device-list strategy is requested but NVML is unavailable (the +// GB10 / CDI-only case), New returns the external handler that can still build +// qualified device names for injection. +func TestNew_NoNVML_CDIEnabled_ReturnsExternal(t *testing.T) { + h, err := New(fakeInfo{hasNVML: false}, nil, nil, + WithDeviceListStrategies(mustStrategies(t, "cdi-annotations", "cdi-cri")), + WithVendor(DefaultVendor)) + require.NoError(t, err) + require.Equal(t, DefaultVendor+"/gpu=GPU-abc", h.QualifiedName("gpu", "GPU-abc")) + require.NoError(t, h.CreateSpecFile()) + require.Empty(t, h.AdditionalDevices()) +} + +// The external handler falls back to DefaultVendor when no vendor is set. +func TestNew_NoNVML_DefaultVendor(t *testing.T) { + h, err := New(fakeInfo{hasNVML: false}, nil, nil, + WithDeviceListStrategies(mustStrategies(t, "cdi-cri"))) + require.NoError(t, err) + require.Equal(t, DefaultVendor+"/gpu=0", h.QualifiedName("gpu", "0")) +} diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/cdi/external.go b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/external.go new file mode 100644 index 0000000000..50edcfb13f --- /dev/null +++ b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/external.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cdi + +import ( + cdiparser "tags.cncf.io/container-device-interface/pkg/parser" +) + +// DefaultVendor is the CDI vendor used by the NVIDIA device plugin. It matches +// the vendor of the CDI specs generated by the NVIDIA Container Toolkit / +// GPU Operator (e.g. /var/run/cdi/k8s.device-plugin.nvidia.com-gpu.json). +const DefaultVendor = "k8s.device-plugin.nvidia.com" + +// externalHandler is a CDI handler for nodes where the accelerator is exposed +// only through externally-managed CDI specs and NVML is not available to the +// device plugin. This is the case for CDI-only accelerators such as the GB10 +// (Grace-Blackwell iGPU), whose device is described by a CDI spec generated by +// the NVIDIA Container Toolkit rather than being enumerable via NVML. +// +// It can construct qualified CDI device names for injection but does not +// generate CDI spec files itself (the spec is provided by the node). +type externalHandler struct { + vendor string +} + +var _ Interface = &externalHandler{} + +// newExternalHandler returns a CDI handler that references externally-managed +// CDI specs. An empty vendor defaults to DefaultVendor. +func newExternalHandler(vendor string) Interface { + if vendor == "" { + vendor = DefaultVendor + } + return &externalHandler{vendor: vendor} +} + +// CreateSpecFile is a no-op: the CDI spec is managed externally on the node. +func (h *externalHandler) CreateSpecFile() error { + return nil +} + +// QualifiedName constructs a CDI qualified device name for the specified +// resources. It assumes the id matches a device name present in the +// externally-managed CDI spec. +func (h *externalHandler) QualifiedName(class string, id string) string { + return cdiparser.QualifiedName(h.vendor, class, id) +} + +// AdditionalDevices returns no additional devices for externally-managed specs. +func (h *externalHandler) AdditionalDevices() []string { + return nil +} diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/cdi/external_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/external_test.go new file mode 100644 index 0000000000..ff50fa0748 --- /dev/null +++ b/pkg/device-plugin/nvidiadevice/nvinternal/cdi/external_test.go @@ -0,0 +1,37 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cdi + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExternalHandlerQualifiedName(t *testing.T) { + h := newExternalHandler("k8s.device-plugin.nvidia.com") + require.Equal(t, "k8s.device-plugin.nvidia.com/gpu=0", h.QualifiedName("gpu", "0")) + // CreateSpecFile is a no-op and AdditionalDevices is empty for externally + // managed specs. + require.NoError(t, h.CreateSpecFile()) + require.Empty(t, h.AdditionalDevices()) +} + +func TestExternalHandlerDefaultVendor(t *testing.T) { + h := newExternalHandler("") + require.Equal(t, DefaultVendor+"/gpu=GPU-abc", h.QualifiedName("gpu", "GPU-abc")) +} diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/factory.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/factory.go index 950cc1662e..5af72bb299 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/factory.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/factory.go @@ -42,6 +42,7 @@ import ( "k8s.io/klog/v2" spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" + "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/cdi" "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/imex" "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/rm" @@ -61,6 +62,11 @@ type options struct { deviceListStrategies spec.DeviceListStrategies imexChannels imex.Channels + + // resolvedStrategy holds the device discovery strategy resolved from the + // configured strategy and the detected platform (see resolveStrategy). It is + // populated by getResourceManagers and consumed when constructing plugins. + resolvedStrategy string } // New a new set of plugins with the supplied options. @@ -109,6 +115,7 @@ func New(ctx context.Context, infolib info.Interface, nvmllib nvml.Interface, de // include full GPUs or MIG devices. func (o *options) getResourceManagers() ([]rm.ResourceManager, error) { strategy := o.resolveStrategy(*o.config.Flags.DeviceDiscoveryStrategy) + o.resolvedStrategy = strategy switch strategy { case "nvml": ret := o.nvmllib.Init() @@ -131,6 +138,12 @@ func (o *options) getResourceManagers() ([]rm.ResourceManager, error) { return rm.NewNVMLResourceManagers(o.infolib, o.nvmllib, o.devicelib, o.config.Config) case "tegra": return rm.NewTegraResourceManagers(o.config.Config) + case "cdi": + // CDI-only accelerators (e.g. the GB10 Grace-Blackwell iGPU) are not + // enumerable via NVML. Discover them from the externally-managed CDI + // specs present on the node instead. + klog.Info("Discovering devices from CDI specs (NVML enumeration unavailable)") + return rm.NewCDIResourceManagers(o.config.Config) default: klog.Errorf("Incompatible strategy detected %v", strategy) klog.Error("If this is a GPU node, did you configure the NVIDIA Container Toolkit?") @@ -156,5 +169,16 @@ func (o *options) resolveStrategy(strategy string) string { case info.PlatformTegra: return "tegra" } + + // No NVML/Tegra platform was detected. If a CDI device-list strategy is in + // use and CDI specs describing GPU devices are present on the node, discover + // devices from those specs. This supports CDI-only accelerators such as the + // GB10 (Grace-Blackwell iGPU), which are exposed exclusively through CDI and + // cannot be enumerated via NVML. + if o.deviceListStrategies.AnyCDIEnabled() && rm.HasCDISpecs(rm.CDISpecDirs()) { + klog.Info("No NVML/Tegra platform detected; falling back to CDI device discovery") + return "cdi" + } + return strategy } diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/factory_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/factory_test.go new file mode 100644 index 0000000000..7f9dbe47b4 --- /dev/null +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/factory_test.go @@ -0,0 +1,112 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "testing" + + "github.com/NVIDIA/go-nvlib/pkg/nvlib/info" + spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" + "github.com/stretchr/testify/require" + + "github.com/Project-HAMi/HAMi/pkg/device/nvidia" +) + +type fakeInfo struct { + platform info.Platform + hasNVML bool +} + +func (f fakeInfo) ResolvePlatform() info.Platform { return f.platform } +func (f fakeInfo) HasDXCore() (bool, string) { return false, "" } +func (f fakeInfo) HasNvml() (bool, string) { return f.hasNVML, "" } +func (f fakeInfo) HasTegraFiles() (bool, string) { return false, "" } +func (f fakeInfo) HasAnIntegratedGPU() (bool, string) { return false, "" } + +func strategies(t *testing.T, s ...string) spec.DeviceListStrategies { + t.Helper() + ds, err := spec.NewDeviceListStrategies(s) + require.NoError(t, err) + return ds +} + +func TestResolveStrategy(t *testing.T) { + cdi := strategies(t, "cdi-cri") + envvar := strategies(t, "envvar") + + cases := []struct { + name string + input string + platform info.Platform + strat spec.DeviceListStrategies + want string + }{ + {"explicit nvml", "nvml", "", envvar, "nvml"}, + {"explicit cdi", "cdi", "", envvar, "cdi"}, + {"explicit tegra", "tegra", "", envvar, "tegra"}, + {"auto->nvml", "auto", info.PlatformNVML, envvar, "nvml"}, + {"auto->wsl->nvml", "auto", info.PlatformWSL, envvar, "nvml"}, + {"auto->tegra", "auto", info.PlatformTegra, envvar, "tegra"}, + {"auto unknown, non-cdi list -> unchanged", "auto", info.PlatformUnknown, envvar, "auto"}, + // Unknown platform with a CDI list strategy but no CDI specs present on + // the test host: the fallback condition is false, so the input is returned. + {"empty unknown, cdi list, no specs -> unchanged", "", info.PlatformUnknown, cdi, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + o := &options{infolib: fakeInfo{platform: tc.platform}, deviceListStrategies: tc.strat} + require.Equal(t, tc.want, o.resolveStrategy(tc.input)) + }) + } +} + +func ptrStr(s string) *string { return &s } + +// getResourceManagers with an explicit "cdi" strategy exercises the CDI branch. +// No CDI specs exist on the test host, so it returns an empty (nil) set. +func TestGetResourceManagers_CDI(t *testing.T) { + cfg := &spec.Config{} + cfg.Flags.DeviceDiscoveryStrategy = ptrStr("cdi") + cfg.Resources.GPUs = []spec.Resource{{Pattern: "*", Name: "nvidia.com/gpu"}} + + o := &options{ + infolib: fakeInfo{}, + deviceListStrategies: strategies(t, "cdi-cri"), + config: &nvidia.DeviceConfig{Config: cfg}, + } + rms, err := o.getResourceManagers() + require.NoError(t, err) + require.Empty(t, rms) + require.Equal(t, "cdi", o.resolvedStrategy) +} + +// An unresolved strategy hits the default branch; with failOnInitError=false it +// returns no managers and no error. +func TestGetResourceManagers_Invalid(t *testing.T) { + cfg := &spec.Config{} + cfg.Flags.DeviceDiscoveryStrategy = ptrStr("auto") + + o := &options{ + infolib: fakeInfo{platform: info.PlatformUnknown}, + deviceListStrategies: strategies(t, "envvar"), + config: &nvidia.DeviceConfig{Config: cfg}, + failOnInitError: false, + } + rms, err := o.getResourceManagers() + require.NoError(t, err) + require.Empty(t, rms) +} diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go index 9f32accdaa..ed788f8319 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go @@ -86,7 +86,16 @@ func GetNumaNode(d nvml.Device) (bool, int, error) { return true, node, nil } +// cdiDefaultDeviceType is the device type reported for CDI-discovered GPUs when +// preConfiguredDeviceType is not set. It is used for CDI-only accelerators such +// as the GB10 (Grace-Blackwell iGPU). +const cdiDefaultDeviceType = "NVIDIA-GB10" + func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*device.DeviceInfo { + if plugin.cdiDiscovery { + return plugin.getCDIAPIDevices() + } + devs := plugin.Devices() defer nvml.Shutdown() klog.V(5).InfoS("getAPIDevices", "devices", devs) @@ -188,6 +197,60 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*device.DeviceInfo { return &res } +// getCDIAPIDevices builds the registration info for GPUs discovered from +// externally-managed CDI specs (e.g. the GB10 iGPU). NVML is unavailable in +// this mode, so device memory and type are taken from configuration +// (preConfiguredDeviceMemory / preConfiguredDeviceType) instead of being +// queried from the driver. +func (plugin *NvidiaDevicePlugin) getCDIAPIDevices() *[]*device.DeviceInfo { + devs := plugin.Devices() + res := make([]*device.DeviceInfo, 0, len(devs)) + + if plugin.schedulerConfig.PreConfiguredDeviceMemory == nil || *plugin.schedulerConfig.PreConfiguredDeviceMemory <= 0 { + klog.Error("CDI device discovery requires 'preConfiguredDeviceMemory' (in MB) to be set in the nvidia config; " + + "CDI-only accelerators cannot report memory via NVML. No devices registered.") + return &res + } + + registeredmem := int32(*plugin.schedulerConfig.PreConfiguredDeviceMemory) + if *plugin.schedulerConfig.DeviceMemoryScaling != 1 { + registeredmem = int32(float64(registeredmem) * *plugin.schedulerConfig.DeviceMemoryScaling) + klog.V(3).Infof("CDI device: MemoryScaling=%v, registeredmem=%vMB", *plugin.schedulerConfig.DeviceMemoryScaling, registeredmem) + } + + model := cdiDefaultDeviceType + if plugin.schedulerConfig.PreConfiguredDeviceType != nil && *plugin.schedulerConfig.PreConfiguredDeviceType != "" { + model = *plugin.schedulerConfig.PreConfiguredDeviceType + } + if !strings.HasPrefix(model, "NVIDIA") { + model = fmt.Sprintf("NVIDIA-%s", model) + } + + devcore := int32(*plugin.schedulerConfig.DeviceCoreScaling * 100) + + for _, d := range devs { + uuid := d.ID + idx, err := strconv.Atoi(d.Index) + if err != nil { + klog.ErrorS(err, "invalid CDI device index; skipping device", "index", d.Index, "id", uuid) + continue + } + res = append(res, &device.DeviceInfo{ + ID: uuid, + Index: uint(idx), + Count: int32(*plugin.schedulerConfig.DeviceSplitCount), + Devmem: registeredmem, + Devcore: devcore, + Type: model, + Numa: 0, + Mode: plugin.operatingMode, + Health: true, + }) + klog.V(3).Infof("Registered CDI device id=%v, index=%v, memory=%vMB, type=%v", uuid, idx, registeredmem, model) + } + return &res +} + // RegisterInAnnotation scans devices and patches node annotations. // Returns (changed, error) where changed indicates whether the annotation was actually updated. func (plugin *NvidiaDevicePlugin) RegisterInAnnotation() (bool, error) { @@ -211,7 +274,11 @@ func (plugin *NvidiaDevicePlugin) RegisterInAnnotation() (bool, error) { plugin.deviceCache = encodeddevices var data []byte - if os.Getenv("ENABLE_TOPOLOGY_SCORE") == "true" { + // Topology scoring computes pairwise P2P/NVLink scores via NVML. Skip it in + // CDI mode: NVML is unavailable (nvml.Init would fail and block registration), + // and pairwise topology is meaningless for a CDI-only accelerator such as the + // single GB10 iGPU. + if os.Getenv("ENABLE_TOPOLOGY_SCORE") == "true" && !plugin.cdiDiscovery { gpuScore, err := nvidia.CalculateGPUScore(device.GetDevicesUUIDList(*devices)) if err != nil { klog.ErrorS(err, "calculate gpu topo score error") diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register_cdi_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register_cdi_test.go new file mode 100644 index 0000000000..e63f5e1ffc --- /dev/null +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register_cdi_test.go @@ -0,0 +1,130 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/rm" + "github.com/Project-HAMi/HAMi/pkg/device/nvidia" +) + +func u(v uint) *uint { return &v } +func f64(v float64) *float64 { return &v } +func i64(v int64) *int64 { return &v } +func s(v string) *string { return &v } + +func cdiDevices(ids ...string) rm.Devices { + d := rm.Devices{} + for i, id := range ids { + dev := &rm.Device{Index: intToStr(i)} + dev.ID = id + d[id] = dev + } + return d +} + +func intToStr(i int) string { + return string(rune('0' + i)) +} + +func newCDIPlugin(devs rm.Devices, cfg nvidia.NodeDefaultConfig) *NvidiaDevicePlugin { + return &NvidiaDevicePlugin{ + cdiDiscovery: true, + operatingMode: "hami-core", + rm: &rm.ResourceManagerMock{DevicesFunc: func() rm.Devices { return devs }}, + schedulerConfig: nvidia.NvidiaConfig{NodeDefaultConfig: cfg}, + } +} + +func baseCfg() nvidia.NodeDefaultConfig { + return nvidia.NodeDefaultConfig{ + PreConfiguredDeviceMemory: i64(122566), + DeviceMemoryScaling: f64(1), + DeviceCoreScaling: f64(1), + DeviceSplitCount: u(10), + PreConfiguredDeviceType: s("NVIDIA-GB10"), + } +} + +func TestGetCDIAPIDevices_Basic(t *testing.T) { + p := newCDIPlugin(cdiDevices("GPU-abc"), baseCfg()) + res := *p.getCDIAPIDevices() + require.Len(t, res, 1) + d := res[0] + require.Equal(t, "GPU-abc", d.ID) + require.Equal(t, uint(0), d.Index) + require.Equal(t, int32(122566), d.Devmem) + require.Equal(t, int32(100), d.Devcore) + require.Equal(t, "NVIDIA-GB10", d.Type) + require.Equal(t, int32(10), d.Count) + require.True(t, d.Health) + require.Equal(t, "hami-core", d.Mode) +} + +func TestGetCDIAPIDevices_MemoryScaling(t *testing.T) { + cfg := baseCfg() + cfg.DeviceMemoryScaling = f64(2) + p := newCDIPlugin(cdiDevices("GPU-abc"), cfg) + res := *p.getCDIAPIDevices() + require.Len(t, res, 1) + require.Equal(t, int32(122566*2), res[0].Devmem) +} + +func TestGetCDIAPIDevices_DefaultTypeAndPrefix(t *testing.T) { + // nil type -> built-in default. + cfg := baseCfg() + cfg.PreConfiguredDeviceType = nil + p := newCDIPlugin(cdiDevices("GPU-abc"), cfg) + require.Equal(t, cdiDefaultDeviceType, (*p.getCDIAPIDevices())[0].Type) + + // type without an NVIDIA prefix gets one added. + cfg2 := baseCfg() + cfg2.PreConfiguredDeviceType = s("GB10") + p2 := newCDIPlugin(cdiDevices("GPU-abc"), cfg2) + require.Equal(t, "NVIDIA-GB10", (*p2.getCDIAPIDevices())[0].Type) +} + +func TestGetCDIAPIDevices_NoPreConfiguredMemory(t *testing.T) { + cfg := baseCfg() + cfg.PreConfiguredDeviceMemory = nil + p := newCDIPlugin(cdiDevices("GPU-abc"), cfg) + require.Empty(t, *p.getCDIAPIDevices()) + + cfg.PreConfiguredDeviceMemory = i64(0) + p = newCDIPlugin(cdiDevices("GPU-abc"), cfg) + require.Empty(t, *p.getCDIAPIDevices()) +} + +func TestGetCDIAPIDevices_InvalidIndexSkipped(t *testing.T) { + devs := rm.Devices{} + bad := &rm.Device{Index: "not-a-number"} + bad.ID = "GPU-bad" + devs["GPU-bad"] = bad + p := newCDIPlugin(devs, baseCfg()) + require.Empty(t, *p.getCDIAPIDevices()) +} + +// getAPIDevices dispatches to the CDI path when cdiDiscovery is set. +func TestGetAPIDevices_DispatchesToCDI(t *testing.T) { + p := newCDIPlugin(cdiDevices("GPU-abc"), baseCfg()) + res := *p.getAPIDevices() + require.Len(t, res, 1) + require.Equal(t, "GPU-abc", res[0].ID) +} diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go index 344f424aa5..889c553727 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go @@ -112,6 +112,12 @@ type NvidiaDevicePlugin struct { migCurrent nvidia.MigPartedSpec deviceCache string + // cdiDiscovery indicates that devices were discovered from externally-managed + // CDI specs rather than via NVML. In this mode the plugin must not call into + // NVML (it is unavailable) — this is the case for CDI-only accelerators such + // as the GB10 (Grace-Blackwell iGPU). + cdiDiscovery bool + imexChannels imex.Channels server *grpc.Server @@ -199,6 +205,7 @@ func (o *options) devicePluginForResource(ctx context.Context, nvconfig *nvidia. operatingMode: mode, migCurrent: nvidia.MigPartedSpec{}, deviceCache: "", + cdiDiscovery: o.resolvedStrategy == "cdi", // These will be reinitialized every // time the plugin server is restarted. @@ -249,14 +256,24 @@ func (plugin *NvidiaDevicePlugin) buildFallbackMigConfig(deviceNumbers int) { func (plugin *NvidiaDevicePlugin) Start(kubeletSocket string) error { plugin.initialize() - deviceNumbers, err := GetDeviceNums() - if err != nil { - return err - } + var deviceNumbers int + var deviceNames []string + var err error + if plugin.cdiDiscovery { + // NVML is unavailable for CDI-only accelerators (e.g. GB10). Derive the + // device count from the CDI-discovered resource manager and skip + // NVML-based enumeration and MIG handling entirely. + deviceNumbers = len(plugin.Devices()) + } else { + deviceNumbers, err = GetDeviceNums() + if err != nil { + return err + } - deviceNames, err := GetDeviceNames() - if err != nil { - return err + deviceNames, err = GetDeviceNames() + if err != nil { + return err + } } err = plugin.Serve() @@ -276,7 +293,7 @@ func (plugin *NvidiaDevicePlugin) Start(kubeletSocket string) error { klog.Infof("Registered device plugin for '%s' with Kubelet", plugin.rm.Resource()) migApplied := false - if plugin.operatingMode == "mig" { + if plugin.operatingMode == "mig" && !plugin.cdiDiscovery { deviceSupportMig := true for _, name := range deviceNames { supported := false diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/cdi_devices.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/cdi_devices.go new file mode 100644 index 0000000000..78e4a78016 --- /dev/null +++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/cdi_devices.go @@ -0,0 +1,153 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rm + +import ( + "fmt" + + spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" + "k8s.io/klog/v2" + cdiapi "tags.cncf.io/container-device-interface/pkg/cdi" + cdiparser "tags.cncf.io/container-device-interface/pkg/parser" +) + +const ( + // CDIVendor is the CDI vendor of the NVIDIA device-plugin CDI specs. It must + // match the vendor used when constructing qualified device names for + // injection (see the cdi handler) and the vendor of the CDI spec generated + // by the NVIDIA Container Toolkit / GPU Operator on the node. + CDIVendor = "k8s.device-plugin.nvidia.com" + // CDIClass is the CDI class for full GPU devices. + CDIClass = "gpu" + // cdiAllDevice is the meta-device that references every GPU in a CDI spec. + // It is excluded from discovery so that each physical device is advertised + // individually. + cdiAllDevice = "all" +) + +// cdiSpecDirs is the default set of directories scanned for CDI specs. It +// mirrors the CDI library defaults (/etc/cdi and /var/run/cdi). +var cdiSpecDirs = cdiapi.DefaultSpecDirs + +// CDISpecDirs returns the directories scanned for CDI specs. +func CDISpecDirs() []string { + return cdiSpecDirs +} + +// HasCDISpecs reports whether any GPU device for the NVIDIA device-plugin +// vendor/class is present in the CDI specs under the supplied directories. It +// is used to auto-detect CDI-only accelerators (e.g. the GB10 iGPU) when NVML +// discovery is unavailable. +func HasCDISpecs(specDirs []string) bool { + names, err := listCDIGPUDevices(specDirs) + if err != nil { + klog.V(3).InfoS("failed to scan CDI specs", "dirs", specDirs, "err", err) + return false + } + return len(names) > 0 +} + +// listCDIGPUDevices returns the CDI device names (e.g. "0", a GPU UUID) for the +// NVIDIA device-plugin GPU class found in the CDI specs under specDirs. The +// "all" meta-device is excluded. +func listCDIGPUDevices(specDirs []string) ([]string, error) { + cache, err := cdiapi.NewCache( + cdiapi.WithSpecDirs(specDirs...), + cdiapi.WithAutoRefresh(false), + ) + if err != nil { + return nil, fmt.Errorf("failed to create CDI cache: %w", err) + } + + var names []string + for _, qualified := range cache.ListDevices() { + vendor, class, name, err := cdiparser.ParseQualifiedName(qualified) + if err != nil { + klog.V(5).InfoS("skipping malformed CDI device", "device", qualified, "err", err) + continue + } + if vendor != CDIVendor || class != CDIClass || name == cdiAllDevice { + continue + } + names = append(names, name) + } + return names, nil +} + +// buildCDIDeviceMap creates a DeviceMap from the GPU devices described by the +// externally-managed CDI specs under specDirs. Devices are discovered without +// NVML, which is what allows CDI-only accelerators such as the GB10 to be +// advertised. +func buildCDIDeviceMap(config *spec.Config, specDirs []string) (DeviceMap, error) { + cdiNames, err := listCDIGPUDevices(specDirs) + if err != nil { + return nil, err + } + + devices := make(DeviceMap) + i := 0 + for _, cdiName := range cdiNames { + for _, resource := range config.Resources.GPUs { + if !resource.Pattern.Matches(cdiName) { + continue + } + index := fmt.Sprintf("%d", i) + if err := devices.setEntry(resource.Name, index, &cdiDevice{name: cdiName}); err != nil { + return nil, err + } + i++ + break + } + } + return devices, nil +} + +// cdiDevice represents a single GPU discovered from a CDI spec. +type cdiDevice struct { + // name is the CDI device name (e.g. "0" or a GPU UUID). It is used as the + // device UUID so that the allocation path can build a matching qualified + // CDI device name for injection. + name string +} + +var _ deviceInfo = (*cdiDevice)(nil) + +// GetUUID returns the CDI device name used to reference this device. +func (d *cdiDevice) GetUUID() (string, error) { + return d.name, nil +} + +// GetPaths returns no paths: device access is configured through CDI injection. +func (d *cdiDevice) GetPaths() ([]string, error) { + return nil, nil +} + +// GetNumaNode is unsupported for a CDI-discovered device. +func (d *cdiDevice) GetNumaNode() (bool, int, error) { + return false, -1, nil +} + +// GetTotalMemory is unsupported for a CDI-discovered device; the total memory +// is provided out of band via configuration (preConfiguredDeviceMemory). +func (d *cdiDevice) GetTotalMemory() (uint64, error) { + return 0, nil +} + +// GetComputeCapability is unsupported for a CDI-discovered device. +func (d *cdiDevice) GetComputeCapability() (string, error) { + return "0.0", nil +} diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/cdi_devices_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/cdi_devices_test.go new file mode 100644 index 0000000000..e300fd64e6 --- /dev/null +++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/cdi_devices_test.go @@ -0,0 +1,115 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rm + +import ( + "os" + "path/filepath" + "sort" + "testing" + + spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" + "github.com/stretchr/testify/require" +) + +// writeCDISpec writes a minimal NVIDIA device-plugin CDI spec exposing the +// supplied device names (plus the "all" meta-device) into dir. +func writeCDISpec(t *testing.T, dir string, names ...string) { + t.Helper() + + devices := "" + for _, n := range append(names, cdiAllDevice) { + devices += ` {"name": "` + n + `", "containerEdits": {"deviceNodes": [{"path": "/dev/nvidia0"}]}}, +` + } + content := `{ + "cdiVersion": "0.5.0", + "kind": "` + CDIVendor + `/` + CDIClass + `", + "devices": [ +` + devices[:len(devices)-2] + ` + ] +}` + require.NoError(t, os.WriteFile(filepath.Join(dir, CDIVendor+"-"+CDIClass+".json"), []byte(content), 0600)) +} + +func gpuOnlyConfig() *spec.Config { + return &spec.Config{ + Resources: spec.Resources{ + GPUs: []spec.Resource{{Pattern: "*", Name: "nvidia.com/gpu"}}, + }, + } +} + +func TestListCDIGPUDevices(t *testing.T) { + dir := t.TempDir() + writeCDISpec(t, dir, "0", "1") + + names, err := listCDIGPUDevices([]string{dir}) + require.NoError(t, err) + sort.Strings(names) + // The "all" meta-device must be excluded. + require.Equal(t, []string{"0", "1"}, names) +} + +func TestListCDIGPUDevices_NoSpecs(t *testing.T) { + names, err := listCDIGPUDevices([]string{t.TempDir()}) + require.NoError(t, err) + require.Empty(t, names) +} + +func TestHasCDISpecs(t *testing.T) { + empty := t.TempDir() + require.False(t, HasCDISpecs([]string{empty})) + + withSpec := t.TempDir() + writeCDISpec(t, withSpec, "0") + require.True(t, HasCDISpecs([]string{withSpec})) +} + +func TestBuildCDIDeviceMap(t *testing.T) { + dir := t.TempDir() + writeCDISpec(t, dir, "0", "1") + + deviceMap, err := buildCDIDeviceMap(gpuOnlyConfig(), []string{dir}) + require.NoError(t, err) + + devices := deviceMap["nvidia.com/gpu"] + require.Len(t, devices, 2) + // Device IDs must be the CDI device names so the allocation path can build + // a matching qualified CDI device name for injection. + ids := devices.GetIDs() + sort.Strings(ids) + require.Equal(t, []string{"0", "1"}, ids) +} + +func TestNewCDIResourceManagers(t *testing.T) { + dir := t.TempDir() + writeCDISpec(t, dir, "0") + + // Point discovery at the temp spec dir for the duration of the test. + orig := cdiSpecDirs + cdiSpecDirs = []string{dir} + defer func() { cdiSpecDirs = orig }() + + rms, err := NewCDIResourceManagers(gpuOnlyConfig()) + require.NoError(t, err) + require.Len(t, rms, 1) + require.Equal(t, spec.ResourceName("nvidia.com/gpu"), rms[0].Resource()) + require.Len(t, rms[0].Devices(), 1) + // CDI resource managers must not perform NVML health checks. + require.NoError(t, rms[0].CheckHealth(nil, nil, nil, nil)) +} diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/cdi_manager.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/cdi_manager.go new file mode 100644 index 0000000000..3ccf606b49 --- /dev/null +++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/cdi_manager.go @@ -0,0 +1,78 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rm + +import ( + "fmt" + + spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" +) + +type cdiResourceManager struct { + resourceManager +} + +var _ ResourceManager = (*cdiResourceManager)(nil) + +// NewCDIResourceManagers returns a set of ResourceManagers for GPUs discovered +// from externally-managed CDI specs on the node. This path does not use NVML +// and is intended for CDI-only accelerators such as the GB10 (Grace-Blackwell +// iGPU). +func NewCDIResourceManagers(config *spec.Config) ([]ResourceManager, error) { + deviceMap, err := buildCDIDeviceMap(config, cdiSpecDirs) + if err != nil { + return nil, fmt.Errorf("error building CDI device map: %v", err) + } + + deviceMap, err = updateDeviceMapWithReplicas(config.Sharing.ReplicatedResources(), deviceMap) + if err != nil { + return nil, fmt.Errorf("error updating device map with replicas from sharing resources: %v", err) + } + + var rms []ResourceManager + for resourceName, devices := range deviceMap { + if len(devices) == 0 { + continue + } + r := &cdiResourceManager{ + resourceManager: resourceManager{ + config: config, + resource: resourceName, + devices: devices, + }, + } + rms = append(rms, r) + } + + return rms, nil +} + +// GetPreferredAllocation returns a standard allocation for the CDI resource manager. +func (r *cdiResourceManager) GetPreferredAllocation(available, required []string, size int) ([]string, error) { + return r.distributedAlloc(available, required, size) +} + +// GetDevicePaths returns an empty slice: CDI devices are injected via CDI, not +// through explicit device paths. +func (r *cdiResourceManager) GetDevicePaths(ids []string) []string { + return nil +} + +// CheckHealth is disabled for the cdiResourceManager: health checks require NVML. +func (r *cdiResourceManager) CheckHealth(stop <-chan any, unhealthy chan<- *Device, disableNVML <-chan bool, ackDisableHealthChecks chan<- bool) error { + return nil +} diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/cdi_manager_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/cdi_manager_test.go new file mode 100644 index 0000000000..eb4b6bc61b --- /dev/null +++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/cdi_manager_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rm + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCDISpecDirs(t *testing.T) { + require.NotEmpty(t, CDISpecDirs()) +} + +// listCDIGPUDevices must ignore specs for other vendors/classes. +func TestListCDIGPUDevices_FiltersOtherVendors(t *testing.T) { + dir := t.TempDir() + writeCDISpec(t, dir, "0") // our vendor/class + + // A spec from a different vendor/class that must be ignored. + other := `{ + "cdiVersion": "0.5.0", + "kind": "example.com/other", + "devices": [ + {"name": "x", "containerEdits": {"deviceNodes": [{"path": "/dev/null"}]}} + ] +}` + require.NoError(t, os.WriteFile(filepath.Join(dir, "example.com-other.json"), []byte(other), 0600)) + + names, err := listCDIGPUDevices([]string{dir}) + require.NoError(t, err) + require.Equal(t, []string{"0"}, names) +} + +func TestCDIResourceManager_AllocationHelpers(t *testing.T) { + dir := t.TempDir() + writeCDISpec(t, dir, "0", "1") + + orig := cdiSpecDirs + cdiSpecDirs = []string{dir} + defer func() { cdiSpecDirs = orig }() + + rms, err := NewCDIResourceManagers(gpuOnlyConfig()) + require.NoError(t, err) + require.Len(t, rms, 1) + r := rms[0] + + ids := r.Devices().GetIDs() + require.Len(t, ids, 2) + + // GetPreferredAllocation returns a size-limited distributed allocation. + alloc, err := r.GetPreferredAllocation(ids, nil, 1) + require.NoError(t, err) + require.Len(t, alloc, 1) + require.Contains(t, ids, alloc[0]) + + // CDI devices are injected via CDI, so no explicit device paths. + require.Nil(t, r.GetDevicePaths(ids)) +} diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index 54e160d7c9..f570b26cc2 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -127,6 +127,10 @@ type NodeDefaultConfig struct { DeviceMemoryScaling *float64 `yaml:"deviceMemoryScaling" json:"devicememoryscaling"` DeviceCoreScaling *float64 `yaml:"deviceCoreScaling" json:"devicecorescaling"` PreConfiguredDeviceMemory *int64 `yaml:"preConfiguredDeviceMemory" json:"preconfigureddevicememory"` + // PreConfiguredDeviceType is the device type reported for CDI-discovered + // GPUs that cannot be enumerated via NVML (e.g. the GB10 iGPU). Defaults to + // "NVIDIA-GB10" when unset. + PreConfiguredDeviceType *string `yaml:"preConfiguredDeviceType" json:"preconfigureddevicetype"` // LogLevel is LIBCUDA_LOG_LEVEL value LogLevel *LibCudaLogLevel `yaml:"libCudaLogLevel" json:"libcudaloglevel"` // EnableNUMATopology advertises the physical GPU's NUMA node on each vGPU