diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 6496d115ee..0e222d244d 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -512,6 +512,12 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } + if !needTopology && !device.FitQuotaWithPodDevices(tmpDevs, allocated, pod.Namespace, k.Type, + int64(memreq), int64(k.Coresreq), npu.GetResourceNames().MemoryFactor) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", klog.KObj(pod), "memreq", memreq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) @@ -575,6 +581,9 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD if needTopology { if len(tmpDevs[k.Type]) == int(originReq) { + if !npu.fitQuotaForSelection(tmpDevs, allocated, pod, k.Type, reason) { + return false, tmpDevs, common.GenReason(reason, len(devices)) + } klog.V(5).InfoS("device allocate success", "pod", klog.KObj(pod), "allocate device", tmpDevs) return true, tmpDevs, "" } else if len(tmpDevs[k.Type]) > int(originReq) { @@ -585,6 +594,9 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD combination := npu.computeBestCombination(nodeInfo, int(originReq), tmpDevs[k.Type]) tmpDevs[k.Type] = combination } + if !npu.fitQuotaForSelection(tmpDevs, allocated, pod, k.Type, reason) { + return false, tmpDevs, common.GenReason(reason, len(devices)) + } klog.V(5).InfoS("device allocate success", "pod", klog.KObj(pod), "best device combination", tmpDevs) return true, tmpDevs, "" } @@ -597,6 +609,18 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD return false, tmpDevs, common.GenReason(reason, len(devices)) } +// fitQuotaForSelection charges the namespace for exactly the cards that were +// chosen. Passing no extra request means only the contents of tmpDevs and +// anything already allocated to the pod are weighed. +func (npu *Devices) fitQuotaForSelection(tmpDevs map[string]device.ContainerDevices, allocated *device.PodDevices, pod *corev1.Pod, devType string, reason map[string]int) bool { + if device.FitQuotaWithPodDevices(tmpDevs, allocated, pod.Namespace, devType, 0, 0, npu.GetResourceNames().MemoryFactor) { + return true + } + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", klog.KObj(pod), "selected", len(tmpDevs[devType])) + return false +} + func hasNetworkID(devices []*device.DeviceUsage) bool { for _, dev := range devices { if dev.CustomInfo == nil { diff --git a/pkg/device/ascend/device_test.go b/pkg/device/ascend/device_test.go index 742b4981b3..8a8d8068ab 100644 --- a/pkg/device/ascend/device_test.go +++ b/pkg/device/ascend/device_test.go @@ -32,6 +32,7 @@ import ( "k8s.io/klog/v2" "github.com/Project-HAMi/HAMi/pkg/device" + "github.com/Project-HAMi/HAMi/pkg/device/common" "github.com/Project-HAMi/HAMi/pkg/util" ) @@ -2457,3 +2458,149 @@ func TestDevices_AddResourceUsage(t *testing.T) { }) } } + +// Fit() is the second gate on namespace quota, after admission. Usage is only +// recorded at Filter time, so pods created together all clear admission against +// the same figure and this is what stops them. +func TestDevices_FitResourceQuota(t *testing.T) { + dev := &Devices{ + config: VNPUConfig{ + CommonWord: "Ascend910A", + ChipName: "910A", + ResourceName: "huawei.com/Ascend910A", + ResourceMemoryName: "huawei.com/Ascend910A-memory", + MemoryAllocatable: 32768, + MemoryCapacity: 32768, + AICore: 30, + }, + } + + prevDevices := device.DevicesMap + device.DevicesMap = map[string]device.Devices{"Ascend910A": dev} + t.Cleanup(func() { device.DevicesMap = prevDevices }) + + usable := func() []*device.DeviceUsage { + return []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Count: 100, + Totalmem: 32768, + Totalcore: 30, + Type: "Ascend910A", + Health: true, + }} + } + request := device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 2184, + MemPercentagereq: 0, + Coresreq: 0, + Type: "Ascend910A", + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "ascend-quota"}} + + qm := device.NewQuotaManager() + t.Cleanup(func() { delete(qm.Quotas, "ascend-quota") }) + + qm.Quotas["ascend-quota"] = &device.DeviceQuota{ + "huawei.com/Ascend910A-memory": &device.Quota{Used: 0, Limit: 32768, LimitSet: true}, + } + if fit, _, _ := dev.Fit(usable(), request, pod, &device.NodeInfo{}, &device.PodDevices{}); !fit { + t.Fatal("Fit() = false, want true: the request is inside the namespace quota") + } + + qm.Quotas["ascend-quota"] = &device.DeviceQuota{ + "huawei.com/Ascend910A-memory": &device.Quota{Used: 31000, Limit: 32768, LimitSet: true}, + } + fit, _, reason := dev.Fit(usable(), request, pod, &device.NodeInfo{}, &device.PodDevices{}) + if fit { + t.Error("Fit() = true, want false: the namespace quota is exhausted") + } + if !strings.Contains(reason, common.ResourceQuotaNotFit) { + t.Errorf("reason = %q, want it to mention %s", reason, common.ResourceQuotaNotFit) + } +} + +// In topology mode the loop gathers every candidate card and picks the best +// originReq subset afterwards, so quota has to be charged for the selection +// rather than for each candidate. Charging per candidate would refuse cards the +// final combination never uses and starve computeBestCombination. +func TestDevices_FitResourceQuotaTopology(t *testing.T) { + dev := &Devices{ + config: VNPUConfig{ + CommonWord: "Ascend910B2", + ChipName: "910B2", + ResourceName: "huawei.com/Ascend910B2", + ResourceMemoryName: "huawei.com/Ascend910B2-memory", + MemoryAllocatable: 65536, + MemoryCapacity: 65536, + AICore: 24, + }, + } + + prevDevices := device.DevicesMap + device.DevicesMap = map[string]device.Devices{"Ascend910B2": dev} + t.Cleanup(func() { device.DevicesMap = prevDevices }) + + usable := func() []*device.DeviceUsage { + devs := make([]*device.DeviceUsage, 0, 4) + for i := range 4 { + devs = append(devs, &device.DeviceUsage{ + ID: fmt.Sprintf("dev-%d", i), + Index: uint(i), + Count: 100, + Totalmem: 65536, + Totalcore: 24, + Type: "Ascend910B2", + Health: true, + CustomInfo: map[string]any{ + "NetworkID": float64(1), + }, + }) + } + return devs + } + request := device.ContainerDeviceRequest{ + Nums: 2, + Memreq: 16384, + MemPercentagereq: 0, + Coresreq: 0, + Type: "Ascend910B2", + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "ascend-topo"}} + + nodeInfo := &device.NodeInfo{ID: "node1", Devices: map[string][]device.DeviceInfo{}} + for i := range 4 { + nodeInfo.Devices["Ascend910B2"] = append(nodeInfo.Devices["Ascend910B2"], device.DeviceInfo{ + ID: fmt.Sprintf("dev-%d", i), + Index: uint(i), + Health: true, + CustomInfo: map[string]any{"NetworkID": float64(1)}, + }) + } + + qm := device.NewQuotaManager() + t.Cleanup(func() { delete(qm.Quotas, "ascend-topo") }) + + qm.Quotas["ascend-topo"] = &device.DeviceQuota{ + "huawei.com/Ascend910B2-memory": &device.Quota{Used: 0, Limit: 40000, LimitSet: true}, + } + fit, res, reason := dev.Fit(usable(), request, pod, nodeInfo, &device.PodDevices{}) + if !fit { + t.Fatalf("Fit() = false (reason %q), want true: the chosen pair is inside quota even though the candidate pool is not", reason) + } + if got := len(res["Ascend910B2"]); got != 2 { + t.Errorf("selected %d cards, want 2", got) + } + + qm.Quotas["ascend-topo"] = &device.DeviceQuota{ + "huawei.com/Ascend910B2-memory": &device.Quota{Used: 0, Limit: 20000, LimitSet: true}, + } + fit, _, reason = dev.Fit(usable(), request, pod, nodeInfo, &device.PodDevices{}) + if fit { + t.Error("Fit() = true, want false: the selected cards exceed the namespace quota") + } + if !strings.Contains(reason, common.ResourceQuotaNotFit) { + t.Errorf("reason = %q, want it to mention %s", reason, common.ResourceQuotaNotFit) + } +} diff --git a/pkg/device/cambricon/device.go b/pkg/device/cambricon/device.go index 222b4cf3e0..94fabfb5ee 100644 --- a/pkg/device/cambricon/device.go +++ b/pkg/device/cambricon/device.go @@ -378,6 +378,12 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } + if !device.FitQuotaWithPodDevices(tmpDevs, allocated, pod.Namespace, k.Type, + int64(memreq), int64(k.Coresreq), cam.GetResourceNames().MemoryFactor) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", klog.KObj(pod), "memreq", memreq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) diff --git a/pkg/device/cambricon/device_test.go b/pkg/device/cambricon/device_test.go index eb3ffa007e..96ee916f83 100644 --- a/pkg/device/cambricon/device_test.go +++ b/pkg/device/cambricon/device_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/client-go/kubernetes/fake" "github.com/Project-HAMi/HAMi/pkg/device" + "github.com/Project-HAMi/HAMi/pkg/device/common" "github.com/Project-HAMi/HAMi/pkg/util/client" ) @@ -1164,3 +1165,79 @@ func TestDevices_AddResourceUsage(t *testing.T) { }) } } + +// Fit() is the second gate on quota. Admission alone cannot hold a namespace +// to its limit, because usage is only recorded at Filter time, so pods created +// together all pass admission against the same figure. Until now only nvidia +// re-checked here. +func TestDevices_FitResourceQuota(t *testing.T) { + dev := InitMLUDevice(CambriconConfig{ + ResourceCountName: "cambricon.com/mlu", + ResourceMemoryName: "cambricon.com/mlu.smlu.vmemory", + ResourceCoreName: "cambricon.com/mlu.smlu.vcore", + }) + + prevDevices := device.DevicesMap + device.DevicesMap = map[string]device.Devices{CambriconMLUDevice: dev} + t.Cleanup(func() { device.DevicesMap = prevDevices }) + + usable := func() []*device.DeviceUsage { + return []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Count: 100, + Usedmem: 0, + Totalmem: 100000, + Totalcore: 100, + Usedcores: 0, + Type: CambriconMLUDevice, + Health: true, + }} + } + request := device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 2560, + MemPercentagereq: 0, + Coresreq: 50, + Type: CambriconMLUDevice, + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "mlu-quota"}} + + qm := device.NewQuotaManager() + t.Cleanup(func() { delete(qm.Quotas, "mlu-quota") }) + + qm.Quotas["mlu-quota"] = &device.DeviceQuota{ + "cambricon.com/mlu.smlu.vmemory": &device.Quota{Used: 0, Limit: 20, LimitSet: true}, + } + fit, _, _ := dev.Fit(usable(), request, pod, &device.NodeInfo{}, &device.PodDevices{}) + if !fit { + t.Fatal("Fit() = false, want true: the request is inside the namespace quota") + } + + qm.Quotas["mlu-quota"] = &device.DeviceQuota{ + "cambricon.com/mlu.smlu.vmemory": &device.Quota{Used: 4864, Limit: 20, LimitSet: true}, + } + fit, _, reason := dev.Fit(usable(), request, pod, &device.NodeInfo{}, &device.PodDevices{}) + if fit { + t.Error("Fit() = true, want false: the namespace quota is exhausted") + } + if !strings.Contains(reason, common.ResourceQuotaNotFit) { + t.Errorf("reason = %q, want it to mention %s", reason, common.ResourceQuotaNotFit) + } + + qm.Quotas["mlu-quota"] = &device.DeviceQuota{ + "cambricon.com/mlu.smlu.vcore": &device.Quota{Used: 80, Limit: 100, LimitSet: true}, + } + fit, _, reason = dev.Fit(usable(), request, pod, &device.NodeInfo{}, &device.PodDevices{}) + if fit { + t.Error("Fit() = true, want false: the namespace core quota is exhausted") + } + if !strings.Contains(reason, common.ResourceQuotaNotFit) { + t.Errorf("reason = %q, want it to mention %s", reason, common.ResourceQuotaNotFit) + } + + delete(qm.Quotas, "mlu-quota") + if fit, _, _ = dev.Fit(usable(), request, pod, &device.NodeInfo{}, &device.PodDevices{}); !fit { + t.Error("Fit() = false, want true: no quota is set for this namespace") + } +} diff --git a/pkg/device/hygon/device.go b/pkg/device/hygon/device.go index 0852ba3e5d..80e2ca4acc 100644 --- a/pkg/device/hygon/device.go +++ b/pkg/device/hygon/device.go @@ -301,6 +301,12 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } + if !device.FitQuotaWithPodDevices(tmpDevs, allocated, pod.Namespace, k.Type, + int64(memreq), int64(k.Coresreq), dcu.GetResourceNames().MemoryFactor) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", klog.KObj(pod), "memreq", memreq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) diff --git a/pkg/device/hygon/device_test.go b/pkg/device/hygon/device_test.go index 0811002227..ed430d2b45 100644 --- a/pkg/device/hygon/device_test.go +++ b/pkg/device/hygon/device_test.go @@ -21,6 +21,7 @@ import ( "errors" "flag" "fmt" + "strings" "testing" "gotest.tools/v3/assert" @@ -31,6 +32,7 @@ import ( "k8s.io/klog/v2" "github.com/Project-HAMi/HAMi/pkg/device" + "github.com/Project-HAMi/HAMi/pkg/device/common" "github.com/Project-HAMi/HAMi/pkg/util" "github.com/Project-HAMi/HAMi/pkg/util/client" ) @@ -1257,3 +1259,59 @@ func TestDevices_AddResourceUsage(t *testing.T) { }) } } + +// Fit() is the second gate on namespace quota, after admission. Usage is only +// recorded at Filter time, so pods created together all clear admission against +// the same figure and this is what stops them. +func TestDevices_FitResourceQuota(t *testing.T) { + dev := InitDCUDevice(HygonConfig{ + ResourceCountName: "hygon.com/dcunum", + ResourceMemoryName: "hygon.com/dcumem", + ResourceCoreName: "hygon.com/dcucores", + }) + + prevDevices := device.DevicesMap + device.DevicesMap = map[string]device.Devices{HygonDCUDevice: dev} + t.Cleanup(func() { device.DevicesMap = prevDevices }) + + usable := func() []*device.DeviceUsage { + return []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Count: 100, + Totalmem: 10000, + Totalcore: 100, + Type: HygonDCUDevice, + Health: true, + }} + } + request := device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 200, + MemPercentagereq: 0, + Coresreq: 50, + Type: HygonDCUDevice, + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "dcu-quota"}} + + qm := device.NewQuotaManager() + t.Cleanup(func() { delete(qm.Quotas, "dcu-quota") }) + + qm.Quotas["dcu-quota"] = &device.DeviceQuota{ + "hygon.com/dcumem": &device.Quota{Used: 0, Limit: 1000, LimitSet: true}, + } + if fit, _, _ := dev.Fit(usable(), request, pod, &device.NodeInfo{}, &device.PodDevices{}); !fit { + t.Fatal("Fit() = false, want true: the request is inside the namespace quota") + } + + qm.Quotas["dcu-quota"] = &device.DeviceQuota{ + "hygon.com/dcumem": &device.Quota{Used: 900, Limit: 1000, LimitSet: true}, + } + fit, _, reason := dev.Fit(usable(), request, pod, &device.NodeInfo{}, &device.PodDevices{}) + if fit { + t.Error("Fit() = true, want false: the namespace quota is exhausted") + } + if !strings.Contains(reason, common.ResourceQuotaNotFit) { + t.Errorf("reason = %q, want it to mention %s", reason, common.ResourceQuotaNotFit) + } +} diff --git a/pkg/device/iluvatar/device.go b/pkg/device/iluvatar/device.go index 1ec0905152..f143cf6b53 100644 --- a/pkg/device/iluvatar/device.go +++ b/pkg/device/iluvatar/device.go @@ -317,6 +317,12 @@ func (ilu *IluvatarDevices) Fit(devices []*device.DeviceUsage, request device.Co //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } + if !device.FitQuotaWithPodDevices(tmpDevs, allocated, pod.Namespace, k.Type, + int64(memreq), int64(k.Coresreq), ilu.GetResourceNames().MemoryFactor) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", klog.KObj(pod), "memreq", memreq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) diff --git a/pkg/device/iluvatar/device_test.go b/pkg/device/iluvatar/device_test.go index b2cf0efb99..83444e8a72 100644 --- a/pkg/device/iluvatar/device_test.go +++ b/pkg/device/iluvatar/device_test.go @@ -21,9 +21,11 @@ import ( "flag" "fmt" "maps" + "strings" "testing" "github.com/Project-HAMi/HAMi/pkg/device" + "github.com/Project-HAMi/HAMi/pkg/device/common" "gotest.tools/v3/assert" corev1 "k8s.io/api/core/v1" @@ -707,3 +709,63 @@ func TestDevices_ReleaseNodeLock(t *testing.T) { }) } } + +// Fit() is the second gate on namespace quota, after admission. One vmemory +// unit is 256 MiB, so the limit has to be scaled by the same factor the request +// already carries. +func Test_FitResourceQuota(t *testing.T) { + dev := &IluvatarDevices{ + config: IluvatarConfig{ + CommonWord: "MR-V100", + ChipName: "MR-V100", + ResourceCountName: "iluvatar.ai/MR-V100-vgpu", + ResourceMemoryName: "iluvatar.ai/MR-V100.vMem", + ResourceCoreName: "iluvatar.ai/MR-V100.vCore", + }, + } + + prevDevices := device.DevicesMap + device.DevicesMap = map[string]device.Devices{"MR-V100": dev} + t.Cleanup(func() { device.DevicesMap = prevDevices }) + + usable := func() []*device.DeviceUsage { + return []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Count: 100, + Totalmem: 100000, + Totalcore: 100, + Type: "MR-V100", + Health: true, + }} + } + request := device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 512, + MemPercentagereq: 0, + Coresreq: 50, + Type: "MR-V100", + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "ilu-quota"}} + + qm := device.NewQuotaManager() + t.Cleanup(func() { delete(qm.Quotas, "ilu-quota") }) + + qm.Quotas["ilu-quota"] = &device.DeviceQuota{ + "iluvatar.ai/MR-V100.vMem": &device.Quota{Used: 0, Limit: 10, LimitSet: true}, + } + if fit, _, _ := dev.Fit(usable(), request, pod, &device.NodeInfo{}, &device.PodDevices{}); !fit { + t.Fatal("Fit() = false, want true: the request is inside the scaled namespace quota") + } + + qm.Quotas["ilu-quota"] = &device.DeviceQuota{ + "iluvatar.ai/MR-V100.vMem": &device.Quota{Used: 2200, Limit: 10, LimitSet: true}, + } + fit, _, reason := dev.Fit(usable(), request, pod, &device.NodeInfo{}, &device.PodDevices{}) + if fit { + t.Error("Fit() = true, want false: the namespace quota is exhausted") + } + if !strings.Contains(reason, common.ResourceQuotaNotFit) { + t.Errorf("reason = %q, want it to mention %s", reason, common.ResourceQuotaNotFit) + } +} diff --git a/pkg/device/mthreads/device.go b/pkg/device/mthreads/device.go index 15b4ebbf21..d58fb3ddce 100644 --- a/pkg/device/mthreads/device.go +++ b/pkg/device/mthreads/device.go @@ -346,6 +346,12 @@ func (mth *MthreadsDevices) Fit(devices []*device.DeviceUsage, request device.Co //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } + if !device.FitQuotaWithPodDevices(tmpDevs, allocated, pod.Namespace, k.Type, + int64(memreq), int64(k.Coresreq), mth.GetResourceNames().MemoryFactor) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", klog.KObj(pod), "memreq", memreq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) diff --git a/pkg/device/mthreads/device_test.go b/pkg/device/mthreads/device_test.go index 901129eeaf..f1e843cb42 100644 --- a/pkg/device/mthreads/device_test.go +++ b/pkg/device/mthreads/device_test.go @@ -18,9 +18,11 @@ package mthreads import ( "flag" + "strings" "testing" "github.com/Project-HAMi/HAMi/pkg/device" + "github.com/Project-HAMi/HAMi/pkg/device/common" "gotest.tools/v3/assert" corev1 "k8s.io/api/core/v1" @@ -1181,3 +1183,59 @@ func TestDevices_AddResourceUsage(t *testing.T) { }) } } + +// Fit() is the second gate on namespace quota, after admission. One vmemory +// unit is 512 MiB, so the limit has to be scaled by the same factor the request +// already carries. +func TestDevices_FitResourceQuota(t *testing.T) { + dev := InitMthreadsDevice(MthreadsConfig{ + ResourceCountName: "mthreads.com/vgpu", + ResourceMemoryName: "mthreads.com/sgpu-memory", + ResourceCoreName: "mthreads.com/sgpu-core", + }) + + prevDevices := device.DevicesMap + device.DevicesMap = map[string]device.Devices{MthreadsGPUDevice: dev} + t.Cleanup(func() { device.DevicesMap = prevDevices }) + + usable := func() []*device.DeviceUsage { + return []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Count: 100, + Totalmem: 100000, + Totalcore: 100, + Type: MthreadsGPUDevice, + Health: true, + }} + } + request := device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 1024, + MemPercentagereq: 0, + Coresreq: 8, + Type: MthreadsGPUDevice, + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "mth-quota"}} + + qm := device.NewQuotaManager() + t.Cleanup(func() { delete(qm.Quotas, "mth-quota") }) + + qm.Quotas["mth-quota"] = &device.DeviceQuota{ + "mthreads.com/sgpu-memory": &device.Quota{Used: 0, Limit: 10, LimitSet: true}, + } + if fit, _, _ := dev.Fit(usable(), request, pod, &device.NodeInfo{}, &device.PodDevices{}); !fit { + t.Fatal("Fit() = false, want true: the request is inside the scaled namespace quota") + } + + qm.Quotas["mth-quota"] = &device.DeviceQuota{ + "mthreads.com/sgpu-memory": &device.Quota{Used: 4500, Limit: 10, LimitSet: true}, + } + fit, _, reason := dev.Fit(usable(), request, pod, &device.NodeInfo{}, &device.PodDevices{}) + if fit { + t.Error("Fit() = true, want false: the namespace quota is exhausted") + } + if !strings.Contains(reason, common.ResourceQuotaNotFit) { + t.Errorf("reason = %q, want it to mention %s", reason, common.ResourceQuotaNotFit) + } +} diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index c7eaec0039..a18690dbf9 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -734,24 +734,7 @@ func (dev *NvidiaGPUDevices) AddResourceUsage(pod *corev1.Pod, n *device.DeviceU } func fitQuota(tmpDevs map[string]device.ContainerDevices, allocated *device.PodDevices, ns string, memreq int64, coresreq int64) bool { - mem := memreq - core := coresreq - for _, val := range tmpDevs[NvidiaGPUDevice] { - mem += int64(val.Usedmem) - core += int64(val.Usedcores) - } - if allocated != nil { - if podSingleDevice, exists := (*allocated)[NvidiaGPUDevice]; exists { - for _, containerDevices := range podSingleDevice { - for _, val := range containerDevices { - mem += int64(val.Usedmem) - core += int64(val.Usedcores) - } - } - } - } - klog.V(4).Infoln("Allocating...", mem, "cores", core) - return device.GetLocalCache().FitQuota(ns, mem, MemoryFactor, core, NvidiaGPUDevice) + return device.FitQuotaWithPodDevices(tmpDevs, allocated, ns, NvidiaGPUDevice, memreq, coresreq, MemoryFactor) } func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.ContainerDeviceRequest, pod *corev1.Pod, nodeInfo *device.NodeInfo, allocated *device.PodDevices) (bool, map[string]device.ContainerDevices, string) { diff --git a/pkg/device/quota.go b/pkg/device/quota.go index 9b12f5f3f4..8519075406 100644 --- a/pkg/device/quota.go +++ b/pkg/device/quota.go @@ -94,6 +94,35 @@ func (q *QuotaManager) FitQuota(ns string, memreq int64, memoryFactor int32, cor return true } +// FitQuotaWithPodDevices reports whether charging one more deviceName device to +// the namespace keeps it inside its ResourceQuota. Devices already chosen for +// this pod count too, both the ones picked so far in tmpDevs and the ones +// recorded in allocated, so a pod asking for several cards is weighed as a +// whole rather than one card at a time. +// +// Backends call this from Fit(). The admission webhook checks the same quota, +// but namespace usage is only recorded at Filter time, so pods created together +// all pass admission against the same figure. This is the check that catches +// them. +func FitQuotaWithPodDevices(tmpDevs map[string]ContainerDevices, allocated *PodDevices, ns, deviceName string, memreq, coresreq int64, memoryFactor int32) bool { + mem := memreq + core := coresreq + for _, val := range tmpDevs[deviceName] { + mem += int64(val.Usedmem) + core += int64(val.Usedcores) + } + if allocated != nil { + for _, containerDevices := range (*allocated)[deviceName] { + for _, val := range containerDevices { + mem += int64(val.Usedmem) + core += int64(val.Usedcores) + } + } + } + klog.V(4).Infoln("Allocating...", mem, "cores", core) + return GetLocalCache().FitQuota(ns, mem, memoryFactor, core, deviceName) +} + func countPodDevices(podDev PodDevices) map[string]int64 { res := make(map[string]int64) for deviceName, podSingle := range podDev { diff --git a/pkg/device/quota_test.go b/pkg/device/quota_test.go index 654768dee6..e2b2972473 100644 --- a/pkg/device/quota_test.go +++ b/pkg/device/quota_test.go @@ -400,3 +400,103 @@ func TestUpdateQuota(t *testing.T) { t.Errorf("memory limit = %d, want 3000", got) } } + +// FitQuotaWithPodDevices has to count the cards already picked for this pod, +// otherwise a multi-card request is waved through one card at a time and the +// namespace ends up over its limit. +func TestFitQuotaWithPodDevices(t *testing.T) { + initTest() + ns := "fit-quota" + qm := NewQuotaManager() + qm.Quotas[ns] = &DeviceQuota{ + "nvidia.com/gpumem": &Quota{Used: 0, Limit: 1000, LimitSet: true}, + "nvidia.com/gpucore": &Quota{Used: 0, Limit: 200, LimitSet: true}, + } + t.Cleanup(func() { delete(qm.Quotas, ns) }) + + tests := []struct { + name string + ns string + tmpDevs map[string]ContainerDevices + allocated *PodDevices + memreq int64 + coresreq int64 + factor int32 + want bool + }{ + { + name: "first card fits", + memreq: 400, coresreq: 50, factor: 1, + want: true, + }, + { + name: "second card counted against the first", + tmpDevs: map[string]ContainerDevices{ + "NVIDIA": {{UUID: "dev-0", Usedmem: 700, Usedcores: 50}}, + }, + memreq: 400, coresreq: 50, factor: 1, + want: false, + }, + { + name: "cards from an earlier container counted too", + allocated: &PodDevices{ + "NVIDIA": PodSingleDevice{{{UUID: "dev-0", Usedmem: 700, Usedcores: 50}}}, + }, + memreq: 400, coresreq: 50, factor: 1, + want: false, + }, + { + name: "cores alone can exceed", + memreq: 10, coresreq: 250, factor: 1, + want: false, + }, + { + name: "factor raises the limit, so a scaled request still fits", + memreq: 2000, coresreq: 50, factor: 4, + want: true, + }, + { + name: "factor applied, request still over", + memreq: 5000, coresreq: 50, factor: 4, + want: false, + }, + { + name: "namespace with no quota is unrestricted", + ns: "no-quota-here", + memreq: 999999, + coresreq: 999999, + factor: 1, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + namespace := tt.ns + if namespace == "" { + namespace = ns + } + got := FitQuotaWithPodDevices(tt.tmpDevs, tt.allocated, namespace, "NVIDIA", tt.memreq, tt.coresreq, tt.factor) + if got != tt.want { + t.Errorf("FitQuotaWithPodDevices() = %v, want %v", got, tt.want) + } + }) + } +} + +// A nil allocated pointer is the normal case for the first container in a pod. +func TestFitQuotaWithPodDevicesNilAllocated(t *testing.T) { + initTest() + qm := NewQuotaManager() + qm.Quotas["nil-alloc"] = &DeviceQuota{ + "nvidia.com/gpumem": &Quota{Used: 0, Limit: 100, LimitSet: true}, + } + t.Cleanup(func() { delete(qm.Quotas, "nil-alloc") }) + + if !FitQuotaWithPodDevices(nil, nil, "nil-alloc", "NVIDIA", 50, 0, 1) { + t.Error("FitQuotaWithPodDevices() = false, want true for a request inside the limit") + } + if FitQuotaWithPodDevices(nil, nil, "nil-alloc", "NVIDIA", 150, 0, 1) { + t.Error("FitQuotaWithPodDevices() = true, want false for a request past the limit") + } +}