Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions pkg/device/nvidia/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,9 @@ func (dev *NvidiaGPUDevices) GetNodeDevices(n corev1.Node) ([]*device.DeviceInfo

func (dev *NvidiaGPUDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) {
/*gpu related */
if err := dev.validateMemoryPercentage(ctr); err != nil {
Comment thread
saiyam1814 marked this conversation as resolved.
return false, err
}
priority, ok := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourcePriority)]
if ok {
ctr.Env = append(ctr.Env, corev1.EnvVar{
Expand Down Expand Up @@ -381,6 +384,15 @@ func (dev *NvidiaGPUDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Po
return hasResource, nil
}

func (dev *NvidiaGPUDevices) validateMemoryPercentage(ctr *corev1.Container) error {
if pct, ok := resourceValue(ctr, corev1.ResourceName(dev.config.ResourceMemoryPercentageName)); ok {
if pct < 0 || pct > 100 {
return fmt.Errorf("invalid %s value %d in container %s: must be an integer between 0 and 100", dev.config.ResourceMemoryPercentageName, pct, ctr.Name)
}
}
return nil
}

func (dev *NvidiaGPUDevices) mutateContainerResource(ctr *corev1.Container) bool {
_, resourceNameOK := ctr.Resources.Limits[corev1.ResourceName(dev.config.ResourceCountName)]
if resourceNameOK {
Expand Down Expand Up @@ -552,6 +564,10 @@ func (dev *NvidiaGPUDevices) GenerateResourceRequests(ctr *corev1.Container) dev
if ok {
mempnums, ok := mem.AsInt64()
if ok {
if mempnums < 0 || mempnums > 100 {
klog.ErrorS(nil, "memory percentage request out of range, clamping to 100", "container", ctr.Name, "requested", mempnums)
mempnums = 100
}
mempnum = int32(mempnums)
}
}
Expand Down
112 changes: 112 additions & 0 deletions pkg/device/nvidia/device_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,64 @@ func Test_MutateAdmission(t *testing.T) {
}
}

func Test_MutateAdmission_MemoryPercentageValidation(t *testing.T) {
gpuDevices := &NvidiaGPUDevices{
config: NvidiaConfig{
ResourceCountName: "nvidia.com/gpu",
ResourceMemoryName: "nvidia.com/gpumem",
ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage",
ResourceCoreName: "nvidia.com/gpucores",
DefaultGPUNum: int32(1),
},
}
tests := []struct {
name string
pct int64
wantErr bool
}{
{
name: "percentage of 0 is accepted",
pct: 0,
wantErr: false,
},
{
name: "percentage of 100 is accepted",
pct: 100,
wantErr: false,
},
{
name: "percentage of 101 is rejected",
pct: 101,
wantErr: true,
},
{
name: "percentage above 100 is rejected",
pct: 150,
wantErr: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctr := &corev1.Container{
Name: "test",
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"nvidia.com/gpu": *resource.NewQuantity(1, resource.BinarySI),
"nvidia.com/gpumem-percentage": *resource.NewQuantity(test.pct, resource.DecimalSI),
},
},
}
_, err := gpuDevices.MutateAdmission(ctr, &corev1.Pod{})
if test.wantErr && err == nil {
t.Fatalf("expected MutateAdmission to reject percentage %d, but got no error", test.pct)
}
if !test.wantErr && err != nil {
t.Fatalf("expected MutateAdmission to accept percentage %d, but got error: %v", test.pct, err)
}
})
}
}

func TestMutateAdmissionDefaultsExclusiveCore(t *testing.T) {
ptr := func(v int64) *int64 { return &v }
clone := func(in corev1.ResourceList) corev1.ResourceList {
Expand Down Expand Up @@ -1725,6 +1783,60 @@ func TestGenerateResourceRequests(t *testing.T) {
Coresreq: 0,
},
},
{
name: "gpu count + memory percentage above 100 — clamped to 100",
ctr: &corev1.Container{
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"nvidia.com/gpu": *resource.NewQuantity(1, resource.BinarySI),
"nvidia.com/gpumem-percentage": *resource.NewQuantity(150, resource.DecimalSI),
},
},
},
want: device.ContainerDeviceRequest{
Nums: 1,
Type: NvidiaGPUDevice,
Memreq: 0,
MemPercentagereq: 100,
Coresreq: 0,
},
},
{
name: "gpu count + memory percentage beyond int32 range — clamped to 100 without wrapping",
ctr: &corev1.Container{
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"nvidia.com/gpu": *resource.NewQuantity(1, resource.BinarySI),
"nvidia.com/gpumem-percentage": *resource.NewQuantity(int64(1)<<32+50, resource.DecimalSI),
},
},
},
want: device.ContainerDeviceRequest{
Nums: 1,
Type: NvidiaGPUDevice,
Memreq: 0,
MemPercentagereq: 100,
Coresreq: 0,
},
},
{
name: "gpu count + memory percentage equal to sentinel 101 — clamped to 100",
ctr: &corev1.Container{
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"nvidia.com/gpu": *resource.NewQuantity(1, resource.BinarySI),
"nvidia.com/gpumem-percentage": *resource.NewQuantity(101, resource.DecimalSI),
},
},
},
want: device.ContainerDeviceRequest{
Nums: 1,
Type: NvidiaGPUDevice,
Memreq: 0,
MemPercentagereq: 100,
Coresreq: 0,
},
},
{
name: "gpu count + explicit cores",
ctr: &corev1.Container{
Expand Down
Loading