diff --git a/pkg/device/amd/device.go b/pkg/device/amd/device.go index 98b748dc87..8611281dd5 100644 --- a/pkg/device/amd/device.go +++ b/pkg/device/amd/device.go @@ -312,6 +312,11 @@ func (amddevice *AMDDevices) Fit(devices []*device.DeviceUsage, request device.C if memReq <= 0 && dev.Totalmem > 0 { memReq = dev.Totalmem } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(memReq), int64(k.Coresreq), AMDDevice, amddevice.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", memReq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memReq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memReq) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 5aff293ce4..248aa485b7 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -512,6 +512,11 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(memreq), int64(k.Coresreq), k.Type, npu.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", memreq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) diff --git a/pkg/device/awsneuron/device.go b/pkg/device/awsneuron/device.go index 27c3023070..b3762f1ff4 100644 --- a/pkg/device/awsneuron/device.go +++ b/pkg/device/awsneuron/device.go @@ -428,6 +428,12 @@ func (neuron *AWSNeuronDevices) Fit(devices []*device.DeviceUsage, request devic continue } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(k.Memreq), int64(k.Coresreq), AWSNeuronDevice, neuron.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", k.Memreq, "coresreq", k.Coresreq) + continue + } + if countMaskAvailable(dev.Totalcore)-countMaskAvailable(dev.Usedcores) < k.Coresreq { reason[common.CardInsufficientCore]++ klog.V(5).InfoS(common.CardInsufficientCore, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total core", dev.Totalcore, "device used core", dev.Usedcores, "request cores", k.Coresreq) diff --git a/pkg/device/biren/device.go b/pkg/device/biren/device.go index 10b1655cd4..f17eff8cdf 100644 --- a/pkg/device/biren/device.go +++ b/pkg/device/biren/device.go @@ -219,6 +219,11 @@ func (br *BirenDevices) Fit(devices []*device.DeviceUsage, request device.Contai klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) continue } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(k.Memreq), int64(k.Coresreq), BirenDevice, br.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", k.Memreq, "coresreq", k.Coresreq) + continue + } if k.Nums > 0 { klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID) k.Nums-- diff --git a/pkg/device/cambricon/device.go b/pkg/device/cambricon/device.go index 222b4cf3e0..fb89741bb7 100644 --- a/pkg/device/cambricon/device.go +++ b/pkg/device/cambricon/device.go @@ -378,6 +378,11 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(memreq), int64(k.Coresreq), CambriconMLUDevice, cam.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", memreq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) diff --git a/pkg/device/enflame/device.go b/pkg/device/enflame/device.go index f5343a1a85..5236b4fdf2 100644 --- a/pkg/device/enflame/device.go +++ b/pkg/device/enflame/device.go @@ -431,6 +431,11 @@ func (enf *EnflameDevices) Fit(devices []*device.DeviceUsage, request device.Con klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used) continue } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(profileMemoryMiB), int64(profileCorePercent), EnflameVGCUDevice, enf.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", profileMemoryMiB, "coresreq", profileCorePercent) + continue + } if dev.Totalmem-dev.Usedmem < profileMemoryMiB { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", profileMemoryMiB) diff --git a/pkg/device/enflame/gcu.go b/pkg/device/enflame/gcu.go index 5d1bc4e836..c3fbbc5f34 100644 --- a/pkg/device/enflame/gcu.go +++ b/pkg/device/enflame/gcu.go @@ -163,6 +163,12 @@ func (gcuDev *GCUDevices) Fit(devices []*device.DeviceUsage, request device.Cont continue } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(k.Memreq), int64(k.Coresreq), EnflameGCUDevice, gcuDev.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", k.Memreq, "coresreq", k.Coresreq) + continue + } + if k.Nums > 0 { klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID) k.Nums-- diff --git a/pkg/device/hygon/device.go b/pkg/device/hygon/device.go index 0852ba3e5d..097eb2af2a 100644 --- a/pkg/device/hygon/device.go +++ b/pkg/device/hygon/device.go @@ -301,6 +301,11 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(memreq), int64(k.Coresreq), HygonDCUDevice, dcu.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", memreq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) diff --git a/pkg/device/iluvatar/device.go b/pkg/device/iluvatar/device.go index 1ec0905152..fc86bb136e 100644 --- a/pkg/device/iluvatar/device.go +++ b/pkg/device/iluvatar/device.go @@ -317,6 +317,11 @@ func (ilu *IluvatarDevices) Fit(devices []*device.DeviceUsage, request device.Co //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(memreq), int64(k.Coresreq), k.Type, ilu.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", memreq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) diff --git a/pkg/device/kunlun/device.go b/pkg/device/kunlun/device.go index 4ed39e771e..0456bb383c 100644 --- a/pkg/device/kunlun/device.go +++ b/pkg/device/kunlun/device.go @@ -205,6 +205,11 @@ func (kl *KunlunDevices) Fit(devices []*device.DeviceUsage, request device.Conta } } } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(request.Memreq), int64(request.Coresreq), request.Type, kl.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", request.Memreq, "coresreq", request.Coresreq) + return false, tmpDevs, common.GenReason(reason, len(devices)) + } return true, tmpDevs, "" } diff --git a/pkg/device/kunlun/vdevice.go b/pkg/device/kunlun/vdevice.go index 242844b2b3..17697b7581 100644 --- a/pkg/device/kunlun/vdevice.go +++ b/pkg/device/kunlun/vdevice.go @@ -271,6 +271,11 @@ func (dev *KunlunVDevices) Fit(devices []*device.DeviceUsage, request device.Con } } } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(request.Memreq), int64(request.Coresreq), request.Type, dev.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", request.Memreq, "coresreq", request.Coresreq) + return false, tmpDevs, common.GenReason(reason, len(devices)) + } return true, tmpDevs, "" } diff --git a/pkg/device/metax/device.go b/pkg/device/metax/device.go index 1f7fe4648f..aacd701297 100644 --- a/pkg/device/metax/device.go +++ b/pkg/device/metax/device.go @@ -277,6 +277,11 @@ func (mat *MetaxDevices) Fit(devices []*device.DeviceUsage, request device.Conta //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(memreq), int64(k.Coresreq), MetaxGPUDevice, mat.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", memreq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) diff --git a/pkg/device/metax/sdevice.go b/pkg/device/metax/sdevice.go index d716fdc59c..e70fd55421 100644 --- a/pkg/device/metax/sdevice.go +++ b/pkg/device/metax/sdevice.go @@ -371,6 +371,12 @@ func (mats *MetaxSDevices) Fit(devices []*device.DeviceUsage, request device.Con memreq = dev.Totalmem * request.MemPercentagereq / 100 } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(memreq), int64(request.Coresreq), MetaxSGPUDevice, mats.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", memreq, "coresreq", request.Coresreq) + continue + } + if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) diff --git a/pkg/device/mthreads/device.go b/pkg/device/mthreads/device.go index 15b4ebbf21..2ac34dee8b 100644 --- a/pkg/device/mthreads/device.go +++ b/pkg/device/mthreads/device.go @@ -346,6 +346,11 @@ func (mth *MthreadsDevices) Fit(devices []*device.DeviceUsage, request device.Co //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(memreq), int64(k.Coresreq), MthreadsGPUDevice, mth.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", memreq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index 57553c5210..90b25bb08a 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -728,27 +728,6 @@ func (dev *NvidiaGPUDevices) AddResourceUsage(pod *corev1.Pod, n *device.DeviceU return nil } -func fitQuota(tmpDevs map[string]device.ContainerDevices, allocated *device.PodDevices, ns string, memreq int64, coresreq int64) bool { - mem := memreq - core := coresreq - for _, val := range tmpDevs[NvidiaGPUDevice] { - mem += int64(val.Usedmem) - core += int64(val.Usedcores) - } - if allocated != nil { - if podSingleDevice, exists := (*allocated)[NvidiaGPUDevice]; exists { - for _, containerDevices := range podSingleDevice { - for _, val := range containerDevices { - mem += int64(val.Usedmem) - core += int64(val.Usedcores) - } - } - } - } - klog.V(4).Infoln("Allocating...", mem, "cores", core) - return device.GetLocalCache().FitQuota(ns, mem, MemoryFactor, core, NvidiaGPUDevice) -} - func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.ContainerDeviceRequest, pod *corev1.Pod, nodeInfo *device.NodeInfo, allocated *device.PodDevices) (bool, map[string]device.ContainerDevices, string) { k := request originReq := k.Nums @@ -812,7 +791,7 @@ func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.Co //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } - if !fitQuota(tmpDevs, allocated, pod.Namespace, int64(memreq), int64(k.Coresreq)) { + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(memreq), int64(k.Coresreq), NvidiaGPUDevice, nv.GetResourceNames()) { reason[common.ResourceQuotaNotFit]++ klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", memreq, "coresreq", k.Coresreq) continue diff --git a/pkg/device/quota.go b/pkg/device/quota.go index dbcf2a4b47..5243a639ef 100644 --- a/pkg/device/quota.go +++ b/pkg/device/quota.go @@ -88,6 +88,31 @@ func (q *QuotaManager) FitQuota(ns string, memreq int64, memoryFactor int32, cor return true } +// FitQuotaForDevice checks whether a pod's total device request (current round +// plus previously allocated containers) fits the namespace ResourceQuota for the +// given device type. Every backend should call this from its Fit() method to +// enforce quota at scheduling time. +func FitQuotaForDevice(tmpDevs map[string]ContainerDevices, allocated *PodDevices, ns string, memreq int64, coresreq int64, deviceName string, resourceNames ResourceNames) bool { + mem := memreq + core := coresreq + for _, val := range tmpDevs[deviceName] { + mem += int64(val.Usedmem) + core += int64(val.Usedcores) + } + if allocated != nil { + if podSingleDevice, exists := (*allocated)[deviceName]; exists { + for _, containerDevices := range podSingleDevice { + for _, val := range containerDevices { + mem += int64(val.Usedmem) + core += int64(val.Usedcores) + } + } + } + } + klog.V(4).Infoln("FitQuotaForDevice: device", deviceName, "mem", mem, "cores", core) + return GetLocalCache().FitQuota(ns, mem, resourceNames.MemoryFactor, core, deviceName) +} + func countPodDevices(podDev PodDevices) map[string]int64 { res := make(map[string]int64) for deviceName, podSingle := range podDev { diff --git a/pkg/device/quota_test.go b/pkg/device/quota_test.go index 89c627867e..cb30b24b51 100644 --- a/pkg/device/quota_test.go +++ b/pkg/device/quota_test.go @@ -282,3 +282,81 @@ func TestDelQuotaNonLimitsKey(t *testing.T) { t.Errorf("DelQuota: expected memory limit 100 after deleting non-limits quota, got %d", (*qm.Quotas[ns])[memName].Limit) } } + +func TestFitQuotaForDevice(t *testing.T) { + initTest() + qm := NewQuotaManager() + ns := "testns" + deviceName := "NVIDIA" + memName := "nvidia.com/gpumem" + coreName := "nvidia.com/gpucore" + rns := ResourceNames{ + ResourceMemoryName: memName, + ResourceCoreName: coreName, + MemoryFactor: 1, + } + + // Set up quota: limit 1000 mem, 100 cores + qm.Quotas[ns] = &DeviceQuota{ + memName: &Quota{Used: 800, Limit: 1000}, + coreName: &Quota{Used: 50, Limit: 100}, + } + + // Should fit: 800 used + 100 requested = 900 <= 1000 + tmpDevs := map[string]ContainerDevices{} + if !FitQuotaForDevice(tmpDevs, nil, ns, 100, 10, deviceName, rns) { + t.Error("FitQuotaForDevice should return true when within limits") + } + + // Should not fit: 800 used + 300 requested = 1100 > 1000 + if FitQuotaForDevice(tmpDevs, nil, ns, 300, 10, deviceName, rns) { + t.Error("FitQuotaForDevice should return false when memory exceeds limit") + } + + // Should not fit cores: 50 used + 60 requested = 110 > 100 + if FitQuotaForDevice(tmpDevs, nil, ns, 10, 60, deviceName, rns) { + t.Error("FitQuotaForDevice should return false when cores exceed limit") + } + + // Should account for devices in tmpDevs + tmpDevs2 := map[string]ContainerDevices{ + deviceName: { + {Usedmem: 100, Usedcores: 20}, + }, + } + // 800 + 100 (tmpDevs) + 50 (request) = 950 <= 1000 + if !FitQuotaForDevice(tmpDevs2, nil, ns, 50, 10, deviceName, rns) { + t.Error("FitQuotaForDevice should return true accounting for tmpDevs") + } + // 800 + 100 (tmpDevs) + 200 (request) = 1100 > 1000 + if FitQuotaForDevice(tmpDevs2, nil, ns, 200, 10, deviceName, rns) { + t.Error("FitQuotaForDevice should return false accounting for tmpDevs") + } + + // Should account for allocated devices + allocated := PodDevices{ + deviceName: PodSingleDevice{ + { + {Usedmem: 50, Usedcores: 5}, + }, + }, + } + // 800 + 50 (allocated) + 50 (request) = 900 <= 1000 + if !FitQuotaForDevice(tmpDevs, &allocated, ns, 50, 10, deviceName, rns) { + t.Error("FitQuotaForDevice should return true accounting for allocated") + } + // 800 + 50 (allocated) + 200 (request) = 1050 > 1000 + if FitQuotaForDevice(tmpDevs, &allocated, ns, 200, 10, deviceName, rns) { + t.Error("FitQuotaForDevice should return false accounting for allocated") + } + + // Should fit if namespace not present + if !FitQuotaForDevice(tmpDevs, nil, "otherns", 5000, 100, deviceName, rns) { + t.Error("FitQuotaForDevice should return true if namespace not present") + } + + // Should fit if device not in quota + if !FitQuotaForDevice(tmpDevs, nil, ns, 5000, 100, "unknown-device", rns) { + t.Error("FitQuotaForDevice should return true if device not in quota") + } +} diff --git a/pkg/device/vastai/device.go b/pkg/device/vastai/device.go index beccf16baa..b186190cd8 100644 --- a/pkg/device/vastai/device.go +++ b/pkg/device/vastai/device.go @@ -281,6 +281,11 @@ func (va *VastaiDevices) Fit(devices []*device.DeviceUsage, request device.Conta continue } } + if !device.FitQuotaForDevice(tmpDevs, allocated, pod.Namespace, int64(k.Memreq), int64(k.Coresreq), VastaiDevice, va.GetResourceNames()) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", k.Memreq, "coresreq", k.Coresreq) + continue + } if k.Nums > 0 { klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID) if !dieMode {