Skip to content
10 changes: 7 additions & 3 deletions pkg/device/cambricon/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"encoding/json"
"flag"
"fmt"
"math"
"math/rand"
"slices"
"strings"
Expand Down Expand Up @@ -251,11 +252,14 @@ func (dev *CambriconDevices) GenerateResourceRequests(ctr *corev1.Container) dev
}
klog.Infoln("mluResourceMem", mem, "ok=", ok, "memoryname=", mluResourceMem)
if ok {
memnums, ok := mem.AsInt64()
memnums, parsed := mem.AsInt64()
klog.Infoln("mluResourceMem", mem, memnums)
if ok {
memnum = int(memnums) * MemoryFactor
if !parsed || memnums < 0 || memnums > int64(math.MaxInt32)/int64(MemoryFactor) {
klog.ErrorS(nil, "cambricon memory request is not a plain integer within the int32 range; rejecting to avoid silent under-allocation",
"container", ctr.Name)
return device.ContainerDeviceRequest{}
}
memnum = int(memnums) * MemoryFactor
}
corenum := int32(100)
core, ok := ctr.Resources.Limits[mluResourceCores]
Expand Down
24 changes: 24 additions & 0 deletions pkg/device/cambricon/device_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,30 @@ func Test_GenerateResourceRequests(t *testing.T) {
Coresreq: int32(2),
},
},
{
name: "memory overflowing int32 is rejected, not truncated to zero",
args: corev1.Container{
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"cambricon.com/mlu": resource.MustParse("1"),
"cambricon.com/mlu.smlu.vmemory": resource.MustParse("16Gi"),
},
},
},
want: device.ContainerDeviceRequest{},
},
{
name: "decimal-form memory request is rejected, not treated as zero",
args: corev1.Container{
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"cambricon.com/mlu": resource.MustParse("1"),
"cambricon.com/mlu.smlu.vmemory": resource.MustParse("16.0Gi"),
},
},
},
want: device.ContainerDeviceRequest{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
Expand Down
10 changes: 7 additions & 3 deletions pkg/device/iluvatar/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"errors"
"flag"
"fmt"
"math"
"slices"
"strings"

Expand Down Expand Up @@ -215,10 +216,13 @@ func (dev *IluvatarDevices) GenerateResourceRequests(ctr *corev1.Container) devi
mem, ok = ctr.Resources.Requests[iluvatarResourceMem]
}
if ok {
memnums, ok := mem.AsInt64()
if ok {
memnum = int(memnums) * MemoryFactor
memnums, parsed := mem.AsInt64()
if !parsed || memnums < 0 || memnums > int64(math.MaxInt32)/int64(MemoryFactor) {
klog.ErrorS(nil, "iluvatar memory request is not a plain integer within the int32 range; rejecting to avoid silent under-allocation",
"container", ctr.Name)
return device.ContainerDeviceRequest{}
}
memnum = int(memnums) * MemoryFactor
}
corenum := int32(0)
core, ok := ctr.Resources.Limits[iluvatarResourceCores]
Expand Down
24 changes: 24 additions & 0 deletions pkg/device/iluvatar/device_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,30 @@ func Test_GenerateResourceRequests(t *testing.T) {
Coresreq: int32(0),
},
},
{
name: "memory overflowing int32 is rejected, not truncated to zero",
args: &corev1.Container{
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"iluvatar.ai/MR-V100-vgpu": resource.MustParse("1"),
"iluvatar.ai/MR-V100.vMem": resource.MustParse("16Gi"),
},
},
},
want: device.ContainerDeviceRequest{},
},
{
name: "decimal-form memory request is rejected, not treated as zero",
args: &corev1.Container{
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"iluvatar.ai/MR-V100-vgpu": resource.MustParse("1"),
"iluvatar.ai/MR-V100.vMem": resource.MustParse("16.0Gi"),
},
},
},
want: device.ContainerDeviceRequest{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
Expand Down
17 changes: 10 additions & 7 deletions pkg/device/mthreads/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,14 +222,17 @@ func (dev *MthreadsDevices) GenerateResourceRequests(ctr *corev1.Container) devi
mem, ok = ctr.Resources.Requests[mthreadsResourceMem]
}
if ok {
memnums, ok := mem.AsInt64()
if ok {
memnum = int(memnums) * MemoryFactor
klog.InfoS("Memory allocation calculated",
"container", ctr.Name,
"requestedMem", memnums,
"allocatedMem", memnum)
memnums, parsed := mem.AsInt64()
if !parsed || memnums < 0 || memnums > int64(math.MaxInt32)/int64(MemoryFactor) {
klog.ErrorS(nil, "mthreads memory request is not a plain integer within the int32 range; rejecting to avoid silent under-allocation",
"container", ctr.Name)
return device.ContainerDeviceRequest{}
}
memnum = int(memnums) * MemoryFactor
klog.InfoS("Memory allocation calculated",
"container", ctr.Name,
"requestedMem", memnums,
"allocatedMem", memnum)
}
corenum := int32(0)
core, ok := ctr.Resources.Limits[mthreadsResourceCores]
Expand Down
26 changes: 26 additions & 0 deletions pkg/device/mthreads/device_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,32 @@ func Test_GenerateResourceRequests(t *testing.T) {
},
want: device.ContainerDeviceRequest{},
},
{
name: "memory overflowing int32 is rejected, not truncated to zero",
args: &corev1.Container{
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"mthreads.com/vgpu": resource.MustParse("1"),
"mthreads.com/sgpu-memory": resource.MustParse("16Gi"),
"mthreads.com/sgpu-core": resource.MustParse("1"),
},
},
},
want: device.ContainerDeviceRequest{},
},
{
name: "decimal-form memory request is rejected, not treated as zero",
args: &corev1.Container{
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"mthreads.com/vgpu": resource.MustParse("1"),
"mthreads.com/sgpu-memory": resource.MustParse("16.0Gi"),
"mthreads.com/sgpu-core": resource.MustParse("1"),
},
},
},
want: device.ContainerDeviceRequest{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
Expand Down
21 changes: 13 additions & 8 deletions pkg/device/nvidia/device.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"errors"
"flag"
"fmt"
"math"
Comment thread
archlitchi marked this conversation as resolved.
Resolved
"sort"
"strconv"
"strings"
Expand Down Expand Up @@ -515,15 +516,19 @@ func (dev *NvidiaGPUDevices) GenerateResourceRequests(ctr *corev1.Container) dev
mem, ok = ctr.Resources.Requests[resourceMem]
}
if ok {
memnums, ok := mem.AsInt64()
if ok {
if dev.config.MemoryFactor > 1 {
rawMemnums := memnums
memnums = memnums * int64(dev.config.MemoryFactor)
klog.V(4).Infof("Update memory request. before %d, after %d, factor %d", rawMemnums, memnums, dev.config.MemoryFactor)
}
memnum = int(memnums)
memnums, parsed := mem.AsInt64()
factor := max(int64(dev.config.MemoryFactor), 1)
if !parsed || memnums < 0 || memnums > int64(math.MaxInt32)/factor {
klog.ErrorS(nil, "nvidia memory request is not a plain integer within the int32 range; rejecting to avoid silent under-allocation",
"container", ctr.Name)
return device.ContainerDeviceRequest{}
}
if factor > 1 {
rawMemnums := memnums
memnums = memnums * factor
klog.V(4).Infof("Update memory request. before %d, after %d, factor %d", rawMemnums, memnums, factor)
}
memnum = int(memnums)
}
mempnum := int32(101)
mem, ok = ctr.Resources.Limits[resourceMemPercentage]
Expand Down
28 changes: 28 additions & 0 deletions pkg/device/nvidia/device_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1884,6 +1884,30 @@ func TestGenerateResourceRequests(t *testing.T) {
Coresreq: 0,
},
},
{
name: "byte-scale memory quantity that overflows int32 is rejected",
ctr: &corev1.Container{
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"nvidia.com/gpu": *resource.NewQuantity(1, resource.BinarySI),
"nvidia.com/gpumem": resource.MustParse("16Gi"),
},
},
},
want: device.ContainerDeviceRequest{},
},
{
name: "decimal-form memory request is rejected, not treated as zero",
ctr: &corev1.Container{
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
"nvidia.com/gpu": *resource.NewQuantity(1, resource.BinarySI),
"nvidia.com/gpumem": resource.MustParse("16.0Gi"),
},
},
},
want: device.ContainerDeviceRequest{},
},
}

for _, tt := range tests {
Expand Down Expand Up @@ -1913,6 +1937,10 @@ func TestGenerateResourceRequests_MemoryFactor(t *testing.T) {
}
result := dev.GenerateResourceRequests(ctr)
assert.Equal(t, result.Memreq, int32(2048))

ctr.Resources.Limits["nvidia.com/gpumem"] = resource.MustParse("1Gi")
result = dev.GenerateResourceRequests(ctr)
assert.DeepEqual(t, result, device.ContainerDeviceRequest{})
}

func TestGenerateResourceRequests_DefaultMemory(t *testing.T) {
Expand Down
Loading