From ff191e037deca9cd99f304860f177b1024939e26 Mon Sep 17 00:00:00 2001 From: alehhu <159355663+alehhu@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:24:56 +0200 Subject: [PATCH] feat: allow setting memory factor per node Signed-off-by: alehhu <159355663+alehhu@users.noreply.github.com> --- charts/hami/values.yaml | 1 + .../nvidiadevice/nvinternal/plugin/server.go | 3 ++ .../nvinternal/plugin/server_test.go | 48 ++++++++++++++++++- pkg/device/nvidia/device.go | 3 +- 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/charts/hami/values.yaml b/charts/hami/values.yaml index 6d39789737..e1989bb04c 100644 --- a/charts/hami/values.yaml +++ b/charts/hami/values.yaml @@ -370,6 +370,7 @@ devicePlugin: "preconfigureddevicememory": 0, "enablenumatopology": false, "migstrategy": "none", + "memoryfactor": 1, "filterdevices": { "uuid": [], "index": [] diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go index 8a3890ead5..2976cb98e6 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go @@ -137,6 +137,9 @@ func readFromConfigFile(sConfig *nvidia.NvidiaConfig, path string) (string, erro if err := mergo.Merge(&sConfig.NodeDefaultConfig, val.NodeDefaultConfig, mergo.WithOverride); err != nil { return "", err } + if val.NodeDefaultConfig.MemoryFactor != nil { + sConfig.MemoryFactor = *val.NodeDefaultConfig.MemoryFactor + } if val.FilterDevice != nil && (len(val.FilterDevice.UUID) > 0 || len(val.FilterDevice.Index) > 0) { nvidia.DevicePluginFilterDevice = val.FilterDevice } diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server_test.go index 21ff292b90..bfb6efea57 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server_test.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server_test.go @@ -629,12 +629,12 @@ func Test_pathGeneration(t *testing.T) { } func Test_configOverride(t *testing.T) { - t.Setenv("NODE_NAME", "testnode") logLevel1 := nvidia.Debugs logLevel2 := nvidia.Infos split1 := uint(2) memScale1 := 1.5 coreScale1 := 1.2 + memFactor1 := int32(2) split2 := uint(3) memScale2 := 0.8 @@ -655,6 +655,7 @@ func Test_configOverride(t *testing.T) { DeviceMemoryScaling: &memScale1, DeviceCoreScaling: &coreScale1, LogLevel: &logLevel1, + MemoryFactor: &memFactor1, }, Name: "node-1", OperatingMode: "default", @@ -684,6 +685,9 @@ func Test_configOverride(t *testing.T) { } path := t.TempDir() os.WriteFile(path+"/config.json", bytes, 0644) + + // Test 1: testnode (no MemoryFactor override, should fall back to global value 5) + t.Setenv("NODE_NAME", "testnode") nvconfig := nvidia.NvidiaConfig{ NodeDefaultConfig: nvidia.NodeDefaultConfig{ DeviceSplitCount: func() *uint { v := uint(1); return &v }(), @@ -696,6 +700,7 @@ func Test_configOverride(t *testing.T) { ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage", ResourceCoreName: "nvidia.com/gpucores", DefaultGPUNum: int32(2), + MemoryFactor: int32(5), // global value } _, err = readFromConfigFile(&nvconfig, path+"/config.json") if err != nil { @@ -707,16 +712,57 @@ func Test_configOverride(t *testing.T) { DeviceMemoryScaling: func() *float64 { v := 0.8; return &v }(), DeviceCoreScaling: func() *float64 { v := 1.4; return &v }(), LogLevel: func() *nvidia.LibCudaLogLevel { v := nvidia.Infos; return &v }(), + MemoryFactor: nil, }, ResourceCountName: "nvidia.com/gpu", ResourceMemoryName: "nvidia.com/gpumem", ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage", ResourceCoreName: "nvidia.com/gpucores", DefaultGPUNum: int32(2), + MemoryFactor: int32(5), // remains global value } if !reflect.DeepEqual(nvconfig, expected) { t.Errorf("Expected %v, got %v", expected, nvconfig) } + + // Test 2: node-1 (with MemoryFactor override, should override global value 5 to 2) + t.Setenv("NODE_NAME", "node-1") + nvconfig2 := nvidia.NvidiaConfig{ + NodeDefaultConfig: nvidia.NodeDefaultConfig{ + DeviceSplitCount: func() *uint { v := uint(1); return &v }(), + DeviceMemoryScaling: func() *float64 { v := 1.0; return &v }(), + DeviceCoreScaling: func() *float64 { v := 1.0; return &v }(), + LogLevel: func() *nvidia.LibCudaLogLevel { v := nvidia.Error; return &v }(), + }, + ResourceCountName: "nvidia.com/gpu", + ResourceMemoryName: "nvidia.com/gpumem", + ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage", + ResourceCoreName: "nvidia.com/gpucores", + DefaultGPUNum: int32(2), + MemoryFactor: int32(5), // global value + } + _, err = readFromConfigFile(&nvconfig2, path+"/config.json") + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + expected2 := nvidia.NvidiaConfig{ + NodeDefaultConfig: nvidia.NodeDefaultConfig{ + DeviceSplitCount: func() *uint { v := uint(2); return &v }(), + DeviceMemoryScaling: func() *float64 { v := 1.5; return &v }(), + DeviceCoreScaling: func() *float64 { v := 1.2; return &v }(), + LogLevel: func() *nvidia.LibCudaLogLevel { v := nvidia.Debugs; return &v }(), + MemoryFactor: func() *int32 { v := int32(2); return &v }(), + }, + ResourceCountName: "nvidia.com/gpu", + ResourceMemoryName: "nvidia.com/gpumem", + ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage", + ResourceCoreName: "nvidia.com/gpucores", + DefaultGPUNum: int32(2), + MemoryFactor: int32(2), // overridden value + } + if !reflect.DeepEqual(nvconfig2, expected2) { + t.Errorf("Expected %v, got %v", expected2, nvconfig2) + } } func TestGetPreferredAllocationSkipsEmptyAnnotations(t *testing.T) { diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index 7f0baa8af5..514c1ae3d1 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -132,7 +132,8 @@ type NodeDefaultConfig struct { // EnableNUMATopology advertises the physical GPU's NUMA node on each vGPU // replica so kubelet's TopologyManager can align CPU and GPU NUMA nodes. // Defaults to false to preserve existing admission behavior. - EnableNUMATopology *bool `yaml:"enableNumaTopology" json:"enablenumatopology"` + EnableNUMATopology *bool `yaml:"enableNumaTopology" json:"enablenumatopology"` + MemoryFactor *int32 `yaml:"memoryFactor,omitempty" json:"memoryfactor,omitempty"` } type FilterDevice struct {