fix: handle GetMemoryInfo NOT_SUPPORTED for unified memory GPUs (cherry-pick #1637 to release-v2.8)

charts/hami/templates/scheduler/device-configmap.yaml

@@ -22,6 +22,7 @@ data:
       defaultMemory: 0
       defaultCores: 0
       defaultGPUNum: 1
+      preConfiguredDeviceMemory: {{ .Values.devicePlugin.preConfiguredDeviceMemory | default 0 }}
       memoryFactor: 1
       deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }}
       deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }}

charts/hami/values.yaml

@@ -302,6 +302,10 @@ devicePlugin:
   deviceSplitCount: 10
   deviceMemoryScaling: 1
   deviceCoreScaling: 1
+  # Pre-configured device memory in MB for GPUs that don't support memory query (e.g., unified memory architecture GPUs like NVIDIA GB10/DGX Spark).
+  # Set to 0 to use auto-detection (default). For unified memory GPUs, set to the total GPU memory (e.g., 131072 for 128GB).
+  # Can be overridden per-node via nodeConfiguration.config.
+  preConfiguredDeviceMemory: 0
   # Node configuration for device plugin, Priority: externalConfigName > config > default config
   nodeConfiguration:
     # If you want to use a custom config.json, you can set the content here.
@@ -314,6 +318,7 @@ devicePlugin:
             "operatingmode": "hami-core",
             "devicememoryscaling": 1,
             "devicesplitcount": 10,
+            "preconfigureddevicememory": 0,
             "migstrategy": "none",
             "filterdevices": {
               "uuid": [],

cmd/vGPUmonitor/metrics.go

@@ -262,6 +262,10 @@ func (cc ClusterManagerCollector) collectGPUDeviceMetrics(ch chan<- prometheus.M
 
 func (cc ClusterManagerCollector) collectGPUMemoryMetrics(ch chan<- prometheus.Metric, hdev nvml.Device, index int) error {
 	memory, ret := hdev.GetMemoryInfo()
+	if ret == nvml.ERROR_NOT_SUPPORTED {
+		klog.V(3).Infof("Memory metrics not supported for device %d (unified memory architecture), skipping", index)
+		return nil
+	}
 	if ret != nvml.SUCCESS {
 		return fmt.Errorf("nvml get memory error ret=%d", ret)
 	}

pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go

@@ -108,9 +108,23 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*device.DeviceInfo {
 		}
 		memoryTotal := 0
 		memory, ret := ndev.GetMemoryInfo()
-		if ret == nvml.SUCCESS {
+		switch ret {
+		case nvml.SUCCESS:
 			memoryTotal = int(memory.Total)
-		} else {
+		case nvml.ERROR_NOT_SUPPORTED:
+			// Unified memory architecture GPUs (e.g., NVIDIA GB10/DGX Spark) don't support
+			// traditional memory queries. Use PreConfiguredDeviceMemory from config as fallback.
+			if plugin.schedulerConfig.PreConfiguredDeviceMemory != nil && *plugin.schedulerConfig.PreConfiguredDeviceMemory > 0 {
+				memoryTotal = int(*plugin.schedulerConfig.PreConfiguredDeviceMemory) * 1024 * 1024
+				klog.Warningf("GetMemoryInfo not supported for device %s, using configured PreConfiguredDeviceMemory: %d MB",
+					UUID, *plugin.schedulerConfig.PreConfiguredDeviceMemory)
+			} else {
+				klog.Errorf("GetMemoryInfo not supported for device %s (unified memory architecture) "+
+					"and PreConfiguredDeviceMemory not configured. Skipping this device. "+
+					"Set 'preConfiguredDeviceMemory' in nvidia config to the total GPU memory in MB.", UUID)
+				continue
+			}
+		default:
 			klog.Error("nvml get memory error ret=", ret)
 			panic(0)
 		}

pkg/device/nvidia/device.go

@@ -124,9 +124,10 @@ type NvidiaConfig struct {
 
 // These configs can be specified for each node by using Nodeconfig.
 type NodeDefaultConfig struct {
-	DeviceSplitCount    *uint    `yaml:"deviceSplitCount" json:"devicesplitcount"`
-	DeviceMemoryScaling *float64 `yaml:"deviceMemoryScaling" json:"devicememoryscaling"`
-	DeviceCoreScaling   *float64 `yaml:"deviceCoreScaling" json:"devicecorescaling"`
+	DeviceSplitCount          *uint    `yaml:"deviceSplitCount" json:"devicesplitcount"`
+	DeviceMemoryScaling       *float64 `yaml:"deviceMemoryScaling" json:"devicememoryscaling"`
+	DeviceCoreScaling         *float64 `yaml:"deviceCoreScaling" json:"devicecorescaling"`
+	PreConfiguredDeviceMemory *int64   `yaml:"preConfiguredDeviceMemory" json:"preconfigureddevicememory"`
 	// LogLevel is LIBCUDA_LOG_LEVEL value
 	LogLevel *LibCudaLogLevel `yaml:"libCudaLogLevel" json:"libcudaloglevel"`
 }