diff --git a/charts/hami/templates/scheduler/device-configmap.yaml b/charts/hami/templates/scheduler/device-configmap.yaml index ccb945ac6f..74c2c1f941 100644 --- a/charts/hami/templates/scheduler/device-configmap.yaml +++ b/charts/hami/templates/scheduler/device-configmap.yaml @@ -22,6 +22,7 @@ data: defaultMemory: 0 defaultCores: 0 defaultGPUNum: 1 + preConfiguredDeviceMemory: {{ .Values.devicePlugin.preConfiguredDeviceMemory | default 0 }} memoryFactor: 1 deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} diff --git a/charts/hami/values.yaml b/charts/hami/values.yaml index 9fbeca706c..8efff8dedd 100644 --- a/charts/hami/values.yaml +++ b/charts/hami/values.yaml @@ -302,6 +302,10 @@ devicePlugin: deviceSplitCount: 10 deviceMemoryScaling: 1 deviceCoreScaling: 1 + # Pre-configured device memory in MB for GPUs that don't support memory query (e.g., unified memory architecture GPUs like NVIDIA GB10/DGX Spark). + # Set to 0 to use auto-detection (default). For unified memory GPUs, set to the total GPU memory (e.g., 131072 for 128GB). + # Can be overridden per-node via nodeConfiguration.config. + preConfiguredDeviceMemory: 0 # Node configuration for device plugin, Priority: externalConfigName > config > default config nodeConfiguration: # If you want to use a custom config.json, you can set the content here. @@ -314,6 +318,7 @@ devicePlugin: "operatingmode": "hami-core", "devicememoryscaling": 1, "devicesplitcount": 10, + "preconfigureddevicememory": 0, "migstrategy": "none", "filterdevices": { "uuid": [], diff --git a/cmd/vGPUmonitor/metrics.go b/cmd/vGPUmonitor/metrics.go index 3a2a5f221f..62d5eb49d4 100644 --- a/cmd/vGPUmonitor/metrics.go +++ b/cmd/vGPUmonitor/metrics.go @@ -262,6 +262,10 @@ func (cc ClusterManagerCollector) collectGPUDeviceMetrics(ch chan<- prometheus.M func (cc ClusterManagerCollector) collectGPUMemoryMetrics(ch chan<- prometheus.Metric, hdev nvml.Device, index int) error { memory, ret := hdev.GetMemoryInfo() + if ret == nvml.ERROR_NOT_SUPPORTED { + klog.V(3).Infof("Memory metrics not supported for device %d (unified memory architecture), skipping", index) + return nil + } if ret != nvml.SUCCESS { return fmt.Errorf("nvml get memory error ret=%d", ret) } diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go index d89ac82abc..c2e38e6718 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go @@ -108,9 +108,23 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*device.DeviceInfo { } memoryTotal := 0 memory, ret := ndev.GetMemoryInfo() - if ret == nvml.SUCCESS { + switch ret { + case nvml.SUCCESS: memoryTotal = int(memory.Total) - } else { + case nvml.ERROR_NOT_SUPPORTED: + // Unified memory architecture GPUs (e.g., NVIDIA GB10/DGX Spark) don't support + // traditional memory queries. Use PreConfiguredDeviceMemory from config as fallback. + if plugin.schedulerConfig.PreConfiguredDeviceMemory != nil && *plugin.schedulerConfig.PreConfiguredDeviceMemory > 0 { + memoryTotal = int(*plugin.schedulerConfig.PreConfiguredDeviceMemory) * 1024 * 1024 + klog.Warningf("GetMemoryInfo not supported for device %s, using configured PreConfiguredDeviceMemory: %d MB", + UUID, *plugin.schedulerConfig.PreConfiguredDeviceMemory) + } else { + klog.Errorf("GetMemoryInfo not supported for device %s (unified memory architecture) "+ + "and PreConfiguredDeviceMemory not configured. Skipping this device. "+ + "Set 'preConfiguredDeviceMemory' in nvidia config to the total GPU memory in MB.", UUID) + continue + } + default: klog.Error("nvml get memory error ret=", ret) panic(0) } diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index e2c94019bb..c6f5f68d81 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -124,9 +124,10 @@ type NvidiaConfig struct { // These configs can be specified for each node by using Nodeconfig. type NodeDefaultConfig struct { - DeviceSplitCount *uint `yaml:"deviceSplitCount" json:"devicesplitcount"` - DeviceMemoryScaling *float64 `yaml:"deviceMemoryScaling" json:"devicememoryscaling"` - DeviceCoreScaling *float64 `yaml:"deviceCoreScaling" json:"devicecorescaling"` + DeviceSplitCount *uint `yaml:"deviceSplitCount" json:"devicesplitcount"` + DeviceMemoryScaling *float64 `yaml:"deviceMemoryScaling" json:"devicememoryscaling"` + DeviceCoreScaling *float64 `yaml:"deviceCoreScaling" json:"devicecorescaling"` + PreConfiguredDeviceMemory *int64 `yaml:"preConfiguredDeviceMemory" json:"preconfigureddevicememory"` // LogLevel is LIBCUDA_LOG_LEVEL value LogLevel *LibCudaLogLevel `yaml:"libCudaLogLevel" json:"libcudaloglevel"` }