From b5a755ef79da4e2e688d3d0d26dfb33fb836690a Mon Sep 17 00:00:00 2001 From: gabesaba <15304068+gabesaba@users.noreply.github.com> Date: Thu, 25 Jul 2024 16:30:11 +0200 Subject: [PATCH 1/6] Cleanup to use FlavorResourceQuantities.Add in cache (#2696) --- pkg/cache/clusterqueue.go | 11 +++-------- pkg/cache/snapshot.go | 23 +++++------------------ 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/pkg/cache/clusterqueue.go b/pkg/cache/clusterqueue.go index 219989ab3f9..f2fc3b3e4ae 100644 --- a/pkg/cache/clusterqueue.go +++ b/pkg/cache/clusterqueue.go @@ -200,15 +200,10 @@ func (c *clusterQueue) update(in *kueue.ClusterQueue, resourceFlavors map[kueue. for _, rg := range c.ResourceGroups { for _, fName := range rg.Flavors { for rName := range rg.CoveredResources { - rQuota := c.QuotaFor(resources.FlavorResource{Flavor: fName, Resource: rName}) + fr := resources.FlavorResource{Flavor: fName, Resource: rName} + rQuota := c.QuotaFor(fr) if rQuota.LendingLimit != nil { - if guaranteedQuota == nil { - guaranteedQuota = make(resources.FlavorResourceQuantities) - } - if guaranteedQuota[fName] == nil { - guaranteedQuota[fName] = make(map[corev1.ResourceName]int64) - } - guaranteedQuota[fName][rName] = rQuota.Nominal - *rQuota.LendingLimit + guaranteedQuota.Add(fr, rQuota.Nominal-*rQuota.LendingLimit) } } } diff --git a/pkg/cache/snapshot.go b/pkg/cache/snapshot.go index 6ff3e0a4bfb..6c4d4c87318 100644 --- a/pkg/cache/snapshot.go +++ b/pkg/cache/snapshot.go @@ -20,7 +20,6 @@ import ( "maps" "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" @@ -168,34 +167,22 @@ func (c *ClusterQueueSnapshot) accumulateResources(cohort *CohortSnapshot) { } for _, rg := range c.ResourceGroups { for _, fName := range rg.Flavors { - res := cohort.RequestableResources[fName] - if res == nil { - res = make(map[corev1.ResourceName]int64, len(rg.CoveredResources)) - cohort.RequestableResources[fName] = res - } for rName := range rg.CoveredResources { - rQuota := c.QuotaFor(resources.FlavorResource{Flavor: fName, Resource: rName}) + fr := resources.FlavorResource{Flavor: fName, Resource: rName} + rQuota := c.QuotaFor(fr) // When feature LendingLimit enabled, cohort.RequestableResources indicates // the sum of cq.NominalQuota and other cqs' LendingLimit (if not nil). // If LendingLimit is not nil, we should count the lendingLimit as the requestable // resource because we can't borrow more quota than lendingLimit. if features.Enabled(features.LendingLimit) && rQuota.LendingLimit != nil { - res[rName] += *rQuota.LendingLimit + cohort.RequestableResources.Add(fr, *rQuota.LendingLimit) } else { - res[rName] += rQuota.Nominal + cohort.RequestableResources.Add(fr, rQuota.Nominal) } } } } - if cohort.Usage == nil { - cohort.Usage = make(resources.FlavorResourceQuantities, len(c.Usage)) - } for fName, resUsages := range c.Usage { - used := cohort.Usage[fName] - if used == nil { - used = make(map[corev1.ResourceName]int64, len(resUsages)) - cohort.Usage[fName] = used - } for res, val := range resUsages { // Similar to cohort.RequestableResources, we accumulate the usage above the guaranteed resources, // here we should remove the guaranteed quota as well for that part can not be borrowed. @@ -204,7 +191,7 @@ func (c *ClusterQueueSnapshot) accumulateResources(cohort *CohortSnapshot) { if val < 0 { val = 0 } - used[res] += val + cohort.Usage.Add(resources.FlavorResource{Flavor: fName, Resource: res}, val) } } } From 46646ef00ded819ab24da316346e5f94344d5a05 Mon Sep 17 00:00:00 2001 From: s-shiraki <54130718+highpon@users.noreply.github.com> Date: Thu, 25 Jul 2024 17:05:35 +0900 Subject: [PATCH 2/6] fix: Refactor FitInCohort tests (#2655) * fix: Refactor FitInCohorot tests * fix: delete no-op function * fix: use new method to add usage * fix: to enforce resource group constraint for flavors and resources * fix: consolidated into a single resource group * fix: delete flavorNames * fix: adjusted test cases to align with existing expected conditions * fix: change FlavorResourceQuantitiesFlat value --- pkg/cache/clusterqueue_test.go | 339 +++++++++++++++++++-------------- 1 file changed, 192 insertions(+), 147 deletions(-) diff --git a/pkg/cache/clusterqueue_test.go b/pkg/cache/clusterqueue_test.go index 43fc91f5829..f86168b1505 100644 --- a/pkg/cache/clusterqueue_test.go +++ b/pkg/cache/clusterqueue_test.go @@ -98,206 +98,238 @@ func TestFitInCohort(t *testing.T) { cases := map[string]struct { request resources.FlavorResourceQuantitiesFlat wantFit bool - cq *ClusterQueueSnapshot + usage resources.FlavorResourceQuantitiesFlat + clusterQueue []*kueue.ClusterQueue enableLendingLimit bool }{ "full cohort, empty request": { request: resources.FlavorResourceQuantitiesFlat{}, wantFit: true, - cq: &ClusterQueueSnapshot{ - Name: "CQ", - Cohort: &CohortSnapshot{ - Name: "C", - RequestableResources: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, - {Flavor: "f2", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f2", Resource: corev1.ResourceMemory}: 5, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, - {Flavor: "f2", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f2", Resource: corev1.ResourceMemory}: 5, - }.Unflatten(), - }, - ResourceGroups: nil, + usage: resources.FlavorResourceQuantitiesFlat{ + {Flavor: "f1", Resource: corev1.ResourceCPU}: 5_000, + {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, + {Flavor: "f2", Resource: corev1.ResourceCPU}: 5_000, + {Flavor: "f2", Resource: corev1.ResourceMemory}: 5, + }, + clusterQueue: []*kueue.ClusterQueue{ + utiltesting. + MakeClusterQueue("CQ"). + ResourceGroup( + *utiltesting.MakeFlavorQuotas("f1"). + Resource(corev1.ResourceCPU, "5"). + Resource(corev1.ResourceMemory, "5"). + Obj(), + *utiltesting.MakeFlavorQuotas("f2"). + Resource(corev1.ResourceCPU, "5"). + Resource(corev1.ResourceMemory, "5"). + Obj(), + ). + Cohort("C"). + Obj(), }, }, "can fit": { request: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f2", Resource: corev1.ResourceCPU}: 1, + {Flavor: "f2", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f2", Resource: corev1.ResourceMemory}: 1, }, wantFit: true, - cq: &ClusterQueueSnapshot{ - Name: "CQ", - Cohort: &CohortSnapshot{ - Name: "C", - RequestableResources: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, - {Flavor: "f2", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f2", Resource: corev1.ResourceMemory}: 5, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, - {Flavor: "f2", Resource: corev1.ResourceCPU}: 4, - {Flavor: "f2", Resource: corev1.ResourceMemory}: 4, - }.Unflatten(), - }, - ResourceGroups: nil, + usage: resources.FlavorResourceQuantitiesFlat{ + {Flavor: "f1", Resource: corev1.ResourceCPU}: 5_000, + {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, + {Flavor: "f2", Resource: corev1.ResourceCPU}: 4_000, + {Flavor: "f2", Resource: corev1.ResourceMemory}: 4, + }, + clusterQueue: []*kueue.ClusterQueue{ + utiltesting. + MakeClusterQueue("CQ"). + ResourceGroup( + *utiltesting.MakeFlavorQuotas("f1"). + Resource(corev1.ResourceCPU, "5"). + Resource(corev1.ResourceMemory, "5"). + Obj(), + *utiltesting.MakeFlavorQuotas("f2"). + Resource(corev1.ResourceCPU, "5"). + Resource(corev1.ResourceMemory, "5"). + Obj(), + ). + Cohort("C"). + Obj(), }, }, "full cohort, none fit": { request: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 1, + {Flavor: "f1", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 1, - {Flavor: "f2", Resource: corev1.ResourceCPU}: 1, + {Flavor: "f2", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f2", Resource: corev1.ResourceMemory}: 1, }, wantFit: false, - cq: &ClusterQueueSnapshot{ - Name: "CQ", - Cohort: &CohortSnapshot{ - Name: "C", - RequestableResources: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, - {Flavor: "f2", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f2", Resource: corev1.ResourceMemory}: 5, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, - {Flavor: "f2", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f2", Resource: corev1.ResourceMemory}: 5, - }.Unflatten(), - }, - ResourceGroups: nil, + usage: resources.FlavorResourceQuantitiesFlat{ + {Flavor: "f1", Resource: corev1.ResourceCPU}: 5_000, + {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, + {Flavor: "f2", Resource: corev1.ResourceCPU}: 5_000, + {Flavor: "f2", Resource: corev1.ResourceMemory}: 5, + }, + clusterQueue: []*kueue.ClusterQueue{ + utiltesting. + MakeClusterQueue("CQ"). + ResourceGroup( + *utiltesting.MakeFlavorQuotas("f1"). + Resource(corev1.ResourceCPU, "5"). + Resource(corev1.ResourceMemory, "5"). + Obj(), + *utiltesting.MakeFlavorQuotas("f2"). + Resource(corev1.ResourceCPU, "5"). + Resource(corev1.ResourceMemory, "5"). + Obj(), + ). + Cohort("C"). + Obj(), }, }, "one cannot fit": { request: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 1, + {Flavor: "f1", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 1, - {Flavor: "f2", Resource: corev1.ResourceCPU}: 2, + {Flavor: "f2", Resource: corev1.ResourceCPU}: 2_000, {Flavor: "f2", Resource: corev1.ResourceMemory}: 1, }, wantFit: false, - cq: &ClusterQueueSnapshot{ - Name: "CQ", - Cohort: &CohortSnapshot{ - Name: "C", - RequestableResources: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, - {Flavor: "f2", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f2", Resource: corev1.ResourceMemory}: 5, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 4, - {Flavor: "f1", Resource: corev1.ResourceMemory}: 4, - {Flavor: "f2", Resource: corev1.ResourceCPU}: 4, - {Flavor: "f2", Resource: corev1.ResourceMemory}: 4, - }.Unflatten(), - }, - ResourceGroups: nil, + usage: resources.FlavorResourceQuantitiesFlat{ + {Flavor: "f1", Resource: corev1.ResourceCPU}: 4_000, + {Flavor: "f1", Resource: corev1.ResourceMemory}: 4, + {Flavor: "f2", Resource: corev1.ResourceCPU}: 4_000, + {Flavor: "f2", Resource: corev1.ResourceMemory}: 4, + }, + clusterQueue: []*kueue.ClusterQueue{ + utiltesting. + MakeClusterQueue("CQ"). + ResourceGroup( + *utiltesting.MakeFlavorQuotas("f1"). + Resource(corev1.ResourceCPU, "5"). + Resource(corev1.ResourceMemory, "5"). + Obj(), + *utiltesting.MakeFlavorQuotas("f2"). + Resource(corev1.ResourceCPU, "5"). + Resource(corev1.ResourceMemory, "5"). + Obj(), + ). + Cohort("C"). + Obj(), }, }, "missing flavor": { request: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f2", Resource: corev1.ResourceCPU}: 1, - {Flavor: "f2", Resource: corev1.ResourceMemory}: 1, + {Flavor: "non-existent-flavor", Resource: corev1.ResourceCPU}: 1_000, + {Flavor: "non-existent-flavor", Resource: corev1.ResourceMemory}: 1, }, wantFit: false, - cq: &ClusterQueueSnapshot{ - Name: "CQ", - Cohort: &CohortSnapshot{ - Name: "C", - RequestableResources: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 5, - {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, - }.Unflatten(), - }, - ResourceGroups: nil, + usage: resources.FlavorResourceQuantitiesFlat{ + {Flavor: "f1", Resource: corev1.ResourceCPU}: 5_000, + {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, + }, + clusterQueue: []*kueue.ClusterQueue{ + utiltesting. + MakeClusterQueue("CQ"). + ResourceGroup( + *utiltesting.MakeFlavorQuotas("f1"). + Resource(corev1.ResourceCPU, "5"). + Resource(corev1.ResourceMemory, "5"). + Obj(), + ). + Cohort("C"). + Obj(), }, }, "missing resource": { request: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 1, + {Flavor: "f1", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 1, }, wantFit: false, - cq: &ClusterQueueSnapshot{ - Name: "CQ", - Cohort: &CohortSnapshot{ - Name: "C", - RequestableResources: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 5, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 3, - }.Unflatten(), - }, - ResourceGroups: nil, + usage: resources.FlavorResourceQuantitiesFlat{ + {Flavor: "f1", Resource: corev1.ResourceCPU}: 3_000, + }, + clusterQueue: []*kueue.ClusterQueue{ + utiltesting. + MakeClusterQueue("CQ"). + ResourceGroup( + *utiltesting.MakeFlavorQuotas("f1"). + Resource(corev1.ResourceCPU, "5"). + Obj(), + ). + Cohort("C"). + Obj(), }, }, "lendingLimit enabled can't fit": { request: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 3, + {Flavor: "f1", Resource: corev1.ResourceCPU}: 3_000, }, wantFit: false, - cq: &ClusterQueueSnapshot{ - Name: "CQ-A", - Cohort: &CohortSnapshot{ - Name: "C", - RequestableResources: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: - // CQ-A has 2 nominal cpu, CQ-B has 3 nominal cpu and 2 lendingLimit, - // so when lendingLimit enabled, the cohort's RequestableResources is 4 cpu. - corev1.ResourceCPU}: 4, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 2, - }.Unflatten(), - }, - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + usage: resources.FlavorResourceQuantitiesFlat{ + {Flavor: "f1", Resource: corev1.ResourceCPU}: 2_000, + }, + clusterQueue: []*kueue.ClusterQueue{ + utiltesting. + MakeClusterQueue("CQ"). + ResourceGroup( + utiltesting.MakeFlavorQuotas("f1"). + ResourceQuotaWrapper(corev1.ResourceCPU). + NominalQuota("2"). + Append(). + FlavorQuotas, + ). + Cohort("C"). + Obj(), + utiltesting. + MakeClusterQueue("CQ-B"). + ResourceGroup( + utiltesting.MakeFlavorQuotas("f1"). + ResourceQuotaWrapper(corev1.ResourceCPU). + NominalQuota("3"). + LendingLimit("2"). + Append(). + FlavorQuotas, + ). + Cohort("C"). + Obj(), }, enableLendingLimit: true, }, "lendingLimit enabled can fit": { request: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 3, + {Flavor: "f1", Resource: corev1.ResourceCPU}: 3_000, }, wantFit: true, - cq: &ClusterQueueSnapshot{ - Name: "CQ-A", - Cohort: &CohortSnapshot{ - Name: "C", - RequestableResources: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: - // CQ-A has 2 nominal cpu, CQ-B has 3 nominal cpu and 2 lendingLimit, - // so when lendingLimit enabled, the cohort's RequestableResources is 4 cpu. - corev1.ResourceCPU}: 4, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: - // CQ-B has admitted a workload with 2 cpus, but with 1 GuaranteedQuota, - // so when lendingLimit enabled, Cohort.Usage should be 2 - 1 = 1. - corev1.ResourceCPU}: 1, - }.Unflatten(), - }, - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ - {Flavor: "f1", Resource: corev1.ResourceCPU}: 2, - }.Unflatten(), + usage: resources.FlavorResourceQuantitiesFlat{ + {Flavor: "f1", Resource: corev1.ResourceCPU}: 1_000, + }, + clusterQueue: []*kueue.ClusterQueue{ + utiltesting. + MakeClusterQueue("CQ"). + ResourceGroup( + utiltesting.MakeFlavorQuotas("f1"). + ResourceQuotaWrapper(corev1.ResourceCPU). + NominalQuota("2"). + Append(). + FlavorQuotas, + ). + Cohort("C"). + Obj(), + utiltesting. + MakeClusterQueue("CQ-B"). + ResourceGroup( + utiltesting.MakeFlavorQuotas("f1"). + ResourceQuotaWrapper(corev1.ResourceCPU). + NominalQuota("3"). + LendingLimit("2"). + Append(). + FlavorQuotas, + ). + Cohort("C"). + Obj(), }, enableLendingLimit: true, }, @@ -306,7 +338,20 @@ func TestFitInCohort(t *testing.T) { for name, tc := range cases { t.Run(name, func(t *testing.T) { defer features.SetFeatureGateDuringTest(t, features.LendingLimit, tc.enableLendingLimit)() - got := tc.cq.FitInCohort(tc.request) + cache := New(utiltesting.NewFakeClient()) + + cache.AddOrUpdateResourceFlavor(utiltesting.MakeResourceFlavor("f1").Obj()) + cache.AddOrUpdateResourceFlavor(utiltesting.MakeResourceFlavor("f2").Obj()) + + for _, cq := range tc.clusterQueue { + _ = cache.AddClusterQueue(context.Background(), cq) + } + + snapshot := cache.Snapshot() + cq := snapshot.ClusterQueues["CQ"] + cq.AddUsage(tc.usage) + + got := cq.FitInCohort(tc.request) if got != tc.wantFit { t.Errorf("Unexpected result, %v", got) } From d01fe01d80c38ec5f0b0ec10e4298f5e336445f3 Mon Sep 17 00:00:00 2001 From: gabesaba <15304068+gabesaba@users.noreply.github.com> Date: Tue, 30 Jul 2024 11:50:35 +0200 Subject: [PATCH 3/6] Finish flattening of FlavorResourceQuantities (#2721) * Finish Flattenning FlavorResourceQuantities * Rename FlavorResourceQuantitiesFlat to FlavorResourceQuantities --- pkg/cache/cache.go | 10 +- pkg/cache/cache_test.go | 320 +++++++++--------- pkg/cache/clusterqueue.go | 74 ++-- pkg/cache/clusterqueue_snapshot.go | 14 +- pkg/cache/clusterqueue_test.go | 66 ++-- pkg/cache/resource.go | 17 +- pkg/cache/snapshot.go | 40 +-- pkg/cache/snapshot_test.go | 220 ++++++------ pkg/resources/resource.go | 31 +- pkg/resources/resource_test.go | 40 --- .../flavorassigner/flavorassigner.go | 30 +- .../flavorassigner/flavorassigner_test.go | 192 +++++------ pkg/scheduler/preemption/preemption.go | 8 +- pkg/scheduler/scheduler.go | 20 +- pkg/scheduler/scheduler_test.go | 42 +-- pkg/workload/workload.go | 4 +- pkg/workload/workload_test.go | 10 +- 17 files changed, 529 insertions(+), 609 deletions(-) delete mode 100644 pkg/resources/resource_test.go diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index a2ac90f87f3..460b2a3a40a 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -619,14 +619,14 @@ func getUsage(frq resources.FlavorResourceQuantities, cq *clusterQueue, cohort * usage := make([]kueue.FlavorUsage, 0, len(frq)) for _, rg := range cq.ResourceGroups { for _, fName := range rg.Flavors { - flvUsage := frq[fName] outFlvUsage := kueue.FlavorUsage{ Name: fName, Resources: make([]kueue.ResourceUsage, 0, len(rg.CoveredResources)), } for rName := range rg.CoveredResources { - rQuota := cq.QuotaFor(resources.FlavorResource{Flavor: fName, Resource: rName}) - used := flvUsage[rName] + fr := resources.FlavorResource{Flavor: fName, Resource: rName} + rQuota := cq.QuotaFor(fr) + used := frq[fr] rUsage := kueue.ResourceUsage{ Name: rName, Total: resources.ResourceQuantity(rName, used), @@ -682,15 +682,15 @@ func filterLocalQueueUsage(orig resources.FlavorResourceQuantities, resourceGrou qFlvUsages := make([]kueue.LocalQueueFlavorUsage, 0, len(orig)) for _, rg := range resourceGroups { for _, fName := range rg.Flavors { - flvUsage := orig[fName] outFlvUsage := kueue.LocalQueueFlavorUsage{ Name: fName, Resources: make([]kueue.LocalQueueResourceUsage, 0, len(rg.CoveredResources)), } for rName := range rg.CoveredResources { + fr := resources.FlavorResource{Flavor: fName, Resource: rName} outFlvUsage.Resources = append(outFlvUsage.Resources, kueue.LocalQueueResourceUsage{ Name: rName, - Total: resources.ResourceQuantity(rName, flvUsage[rName]), + Total: resources.ResourceQuantity(rName, orig[fr]), }) } // The resourceUsages should be in a stable order to avoid endless creation of update events. diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index 17a9582e499..68468c15bde 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -110,12 +110,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -125,12 +125,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -160,12 +160,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: pending, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -257,12 +257,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, FlavorFungibility: defaultFlavorFungibility, NamespaceSelector: labels.Nothing(), - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -272,12 +272,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, FlavorFungibility: defaultFlavorFungibility, NamespaceSelector: labels.Nothing(), - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -307,12 +307,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: pending, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -379,12 +379,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 2, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -422,17 +422,17 @@ func TestCacheClusterQueueOperations(t *testing.T) { "e": { Name: "e", AllocatableResourceGeneration: 2, - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: "cpu"}: 1_000, - }.Unflatten(), + }, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -479,12 +479,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -504,12 +504,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: pending, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -551,12 +551,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -566,12 +566,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -601,12 +601,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -661,22 +661,22 @@ func TestCacheClusterQueueOperations(t *testing.T) { NamespaceSelector: labels.Everything(), AllocatableResourceGeneration: 1, FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "foo", Resource: "cpu"}: 0, {Flavor: "foo", Resource: "memory"}: 0, {Flavor: "bar", Resource: "cpu"}: 0, {Flavor: "bar", Resource: "memory"}: 0, {Flavor: "theta", Resource: "example.com/gpu"}: 0, {Flavor: "gamma", Resource: "example.com/gpu"}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "foo", Resource: "cpu"}: 0, {Flavor: "foo", Resource: "memory"}: 0, {Flavor: "bar", Resource: "cpu"}: 0, {Flavor: "bar", Resource: "memory"}: 0, {Flavor: "theta", Resource: "example.com/gpu"}: 0, {Flavor: "gamma", Resource: "example.com/gpu"}: 0, - }.Unflatten(), + }, Status: pending, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -852,12 +852,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { Preemption: defaultPreemption, AllocatableResourceGeneration: 1, FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 2000, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 1000, - }.Unflatten(), + }, FairWeight: oneQuantity, Workloads: map[string]*workload.Info{ "ns/reserving": { @@ -952,27 +952,27 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 2_000, {Flavor: "on-demand", Resource: corev1.ResourceMemory}: 2 * utiltesting.Gi, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceMemory}: 0, {Flavor: "license", Resource: "license"}: 4, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ + }, + Usage: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "on-demand", Resource: corev1.ResourceMemory}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceMemory}: 0, {Flavor: "license", Resource: "license"}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "on-demand", Resource: corev1.ResourceMemory}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceMemory}: 0, {Flavor: "license", Resource: "license"}: 0, - }.Unflatten(), + }, }, }, }, @@ -1099,17 +1099,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c", "/d"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1128,17 +1128,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1156,17 +1156,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1185,17 +1185,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/a", "/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, }, }, @@ -1214,17 +1214,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1243,17 +1243,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1271,17 +1271,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c", "/d"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1296,17 +1296,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1319,17 +1319,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1343,17 +1343,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1369,17 +1369,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1394,17 +1394,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1431,17 +1431,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b", "/d"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 20, {Flavor: "spot", Resource: corev1.ResourceCPU}: 30, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c", "/e"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, }, wantAssumedWorkloads: map[string]string{ @@ -1464,17 +1464,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, wantAssumedWorkloads: map[string]string{}, @@ -1504,17 +1504,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c", "/e"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, }, wantAssumedWorkloads: map[string]string{ @@ -1536,17 +1536,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1578,17 +1578,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b", "/d"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 20, {Flavor: "spot", Resource: corev1.ResourceCPU}: 30, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c", "/e"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, }, wantAssumedWorkloads: map[string]string{ @@ -2243,46 +2243,46 @@ func TestCacheQueueOperations(t *testing.T) { key: "ns1/alpha", reservingWorkloads: 1, admittedWorkloads: 1, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("2")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("8Gi")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), - admittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + admittedUsage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("2")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("8Gi")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), + }, }, "ns2/beta": { key: "ns2/beta", reservingWorkloads: 2, admittedWorkloads: 1, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("7")), - }.Unflatten(), - admittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + admittedUsage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("2")), - }.Unflatten(), + }, }, "ns1/gamma": { key: "ns1/gamma", reservingWorkloads: 1, admittedWorkloads: 0, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "ondemand", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("5")), {Flavor: "ondemand", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("16Gi")), {Flavor: "model-b", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), - admittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + admittedUsage: resources.FlavorResourceQuantities{ {Flavor: "ondemand", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "ondemand", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-b", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), + }, }, } cacheLocalQueuesAfterInsertingCqAndQ := map[string]*queue{ @@ -2290,46 +2290,46 @@ func TestCacheQueueOperations(t *testing.T) { key: "ns1/alpha", reservingWorkloads: 0, admittedWorkloads: 0, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), - admittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + admittedUsage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), + }, }, "ns2/beta": { key: "ns2/beta", reservingWorkloads: 0, admittedWorkloads: 0, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), - admittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + admittedUsage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), + }, }, "ns1/gamma": { key: "ns1/gamma", reservingWorkloads: 0, admittedWorkloads: 0, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "ondemand", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "ondemand", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-b", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), - admittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + admittedUsage: resources.FlavorResourceQuantities{ {Flavor: "ondemand", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "ondemand", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-b", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), + }, }, } cases := map[string]struct { diff --git a/pkg/cache/clusterqueue.go b/pkg/cache/clusterqueue.go index f2fc3b3e4ae..b9ab3ad74a1 100644 --- a/pkg/cache/clusterqueue.go +++ b/pkg/cache/clusterqueue.go @@ -134,9 +134,9 @@ func (c *cohort) CalculateLendable() map[corev1.ResourceName]int64 { return lendable } -func (c *ClusterQueueSnapshot) FitInCohort(q resources.FlavorResourceQuantitiesFlat) bool { +func (c *ClusterQueueSnapshot) FitInCohort(q resources.FlavorResourceQuantities) bool { for fr, value := range q { - available := c.RequestableCohortQuota(fr.Flavor, fr.Resource) - c.UsedCohortQuota(fr.Flavor, fr.Resource) + available := c.RequestableCohortQuota(fr) - c.UsedCohortQuota(fr) if available < value { return false } @@ -196,34 +196,31 @@ func (c *clusterQueue) update(in *kueue.ClusterQueue, resourceFlavors map[kueue. } if features.Enabled(features.LendingLimit) { - var guaranteedQuota resources.FlavorResourceQuantities + c.GuaranteedQuota = make(resources.FlavorResourceQuantities) for _, rg := range c.ResourceGroups { for _, fName := range rg.Flavors { for rName := range rg.CoveredResources { fr := resources.FlavorResource{Flavor: fName, Resource: rName} rQuota := c.QuotaFor(fr) if rQuota.LendingLimit != nil { - guaranteedQuota.Add(fr, rQuota.Nominal-*rQuota.LendingLimit) + c.GuaranteedQuota[fr] += rQuota.Nominal - *rQuota.LendingLimit } } } } - c.GuaranteedQuota = guaranteedQuota } return nil } func filterFlavorQuantities(orig resources.FlavorResourceQuantities, resourceGroups []kueue.ResourceGroup) resources.FlavorResourceQuantities { - ret := make(resources.FlavorResourceQuantities) + ret := make(resources.FlavorResourceQuantities, len(orig)) for _, rg := range resourceGroups { for _, f := range rg.Flavors { - existingUsedResources := orig[f.Name] - usedResources := make(map[corev1.ResourceName]int64, len(f.Resources)) for _, r := range f.Resources { - usedResources[r.Name] = existingUsedResources[r.Name] + fr := resources.FlavorResource{Flavor: f.Name, Resource: r.Name} + ret[fr] = orig[fr] } - ret[f.Name] = usedResources } } return ret @@ -458,23 +455,23 @@ func (c *clusterQueue) updateWorkloadUsage(wi *workload.Info, m int64) { } } -func updateFlavorUsage(newUsage resources.FlavorResourceQuantitiesFlat, oldUsage resources.FlavorResourceQuantities, m int64) { +func updateFlavorUsage(newUsage resources.FlavorResourceQuantities, oldUsage resources.FlavorResourceQuantities, m int64) { for fr, q := range newUsage { - oldUsage.Add(fr, q*m) + oldUsage[fr] += q * m } } -func updateCohortUsage(newUsage resources.FlavorResourceQuantitiesFlat, cq *ClusterQueueSnapshot, m int64) { +func updateCohortUsage(newUsage resources.FlavorResourceQuantities, cq *ClusterQueueSnapshot, m int64) { for fr, v := range newUsage { - after := cq.Usage.For(fr) - cq.guaranteedQuota(fr.Flavor, fr.Resource) + after := cq.Usage[fr] - cq.guaranteedQuota(fr) // rollback update cq.Usage before := after - v*m if before > 0 { - cq.Cohort.Usage.Add(fr, -before) + cq.Cohort.Usage[fr] -= before } // simulate updating cq.Usage if after > 0 { - cq.Cohort.Usage.Add(fr, after) + cq.Cohort.Usage[fr] += after } } } @@ -533,14 +530,9 @@ func (q *queue) resetFlavorsAndResources(cqUsage resources.FlavorResourceQuantit } func resetUsage(lqUsage resources.FlavorResourceQuantities, cqUsage resources.FlavorResourceQuantities) resources.FlavorResourceQuantities { - usedFlavorResources := make(resources.FlavorResourceQuantities) - for cqFlv, cqRes := range cqUsage { - existingUsedResources := lqUsage[cqFlv] - usedResources := make(map[corev1.ResourceName]int64, len(cqRes)) - for rName := range cqRes { - usedResources[rName] = existingUsedResources[rName] - } - usedFlavorResources[cqFlv] = usedResources + usedFlavorResources := make(resources.FlavorResourceQuantities, len(cqUsage)) + for fr := range cqUsage { + usedFlavorResources[fr] = lqUsage[fr] } return usedFlavorResources } @@ -553,48 +545,48 @@ func workloadBelongsToLocalQueue(wl *kueue.Workload, q *kueue.LocalQueue) bool { // LendingLimit will also be counted here if feature LendingLimit enabled. // Please note that for different clusterQueues, the requestable quota is different, // they should be calculated dynamically. -func (c *ClusterQueueSnapshot) RequestableCohortQuota(fName kueue.ResourceFlavorReference, rName corev1.ResourceName) (val int64) { - if c.Cohort.RequestableResources == nil || c.Cohort.RequestableResources[fName] == nil { +func (c *ClusterQueueSnapshot) RequestableCohortQuota(fr resources.FlavorResource) (val int64) { + if _, ok := c.Cohort.RequestableResources[fr]; !ok { return 0 } - requestableCohortQuota := c.Cohort.RequestableResources[fName][rName] + requestableCohortQuota := c.Cohort.RequestableResources[fr] // When feature LendingLimit enabled, cohort.requestableResource accumulated the lendingLimit if not null // rather than the flavor's quota, then the total available quota should include its own guaranteed resources. - requestableCohortQuota += c.guaranteedQuota(fName, rName) + requestableCohortQuota += c.guaranteedQuota(fr) return requestableCohortQuota } -func (c *ClusterQueueSnapshot) guaranteedQuota(fName kueue.ResourceFlavorReference, rName corev1.ResourceName) (val int64) { +func (c *ClusterQueueSnapshot) guaranteedQuota(fr resources.FlavorResource) (val int64) { if !features.Enabled(features.LendingLimit) { return 0 } - if c.GuaranteedQuota == nil || c.GuaranteedQuota[fName] == nil { + if _, ok := c.GuaranteedQuota[fr]; !ok { return 0 } - return c.GuaranteedQuota[fName][rName] + return c.GuaranteedQuota[fr] } // UsedCohortQuota returns the used quota by the flavor and resource name in the cohort. // Note that when LendingLimit enabled, the usage is not equal to the total used quota but the one // minus the guaranteed resources, this is only for judging whether workloads fit in the cohort. -func (c *ClusterQueueSnapshot) UsedCohortQuota(fName kueue.ResourceFlavorReference, rName corev1.ResourceName) (val int64) { - if c.Cohort.Usage == nil || c.Cohort.Usage[fName] == nil { +func (c *ClusterQueueSnapshot) UsedCohortQuota(fr resources.FlavorResource) (val int64) { + if _, ok := c.Cohort.Usage[fr]; !ok { return 0 } - cohortUsage := c.Cohort.Usage[fName][rName] + cohortUsage := c.Cohort.Usage[fr] // When feature LendingLimit enabled, cohortUsage is the sum of usage in LendingLimit. // If cqUsage < c.guaranteedQuota, it means the cq is not using all its guaranteedQuota, // need to count the cqUsage in, otherwise need to count the guaranteedQuota in. if features.Enabled(features.LendingLimit) { - cqUsage := c.Usage[fName][rName] - if cqUsage < c.guaranteedQuota(fName, rName) { + cqUsage := c.Usage[fr] + if cqUsage < c.guaranteedQuota(fr) { cohortUsage += cqUsage } else { - cohortUsage += c.guaranteedQuota(fName, rName) + cohortUsage += c.guaranteedQuota(fr) } } @@ -617,7 +609,7 @@ func (c *clusterQueue) lendableResourcesInCohort() map[corev1.ResourceName]int64 } func (c *clusterQueue) usageFor(fr resources.FlavorResource) int64 { - return c.Usage.For(fr) + return c.Usage[fr] } func (c *clusterQueue) QuotaFor(fr resources.FlavorResource) *ResourceQuota { @@ -638,11 +630,11 @@ func (c *ClusterQueueSnapshot) DominantResourceShare() (int, corev1.ResourceName return dominantResourceShare(c, nil, 0) } -func (c *ClusterQueueSnapshot) DominantResourceShareWith(wlReq resources.FlavorResourceQuantitiesFlat) (int, corev1.ResourceName) { +func (c *ClusterQueueSnapshot) DominantResourceShareWith(wlReq resources.FlavorResourceQuantities) (int, corev1.ResourceName) { return dominantResourceShare(c, wlReq, 1) } -func (c *ClusterQueueSnapshot) DominantResourceShareWithout(wlReq resources.FlavorResourceQuantitiesFlat) (int, corev1.ResourceName) { +func (c *ClusterQueueSnapshot) DominantResourceShareWithout(wlReq resources.FlavorResourceQuantities) (int, corev1.ResourceName) { return dominantResourceShare(c, wlReq, -1) } @@ -654,7 +646,7 @@ type dominantResourceShareNode interface { netQuotaNode } -func dominantResourceShare(node dominantResourceShareNode, wlReq resources.FlavorResourceQuantitiesFlat, m int64) (int, corev1.ResourceName) { +func dominantResourceShare(node dominantResourceShareNode, wlReq resources.FlavorResourceQuantities, m int64) (int, corev1.ResourceName) { if !node.hasCohort() { return 0, "" } diff --git a/pkg/cache/clusterqueue_snapshot.go b/pkg/cache/clusterqueue_snapshot.go index 7080962beed..d039435974c 100644 --- a/pkg/cache/clusterqueue_snapshot.go +++ b/pkg/cache/clusterqueue_snapshot.go @@ -48,11 +48,11 @@ func (c *ClusterQueueSnapshot) RGByResource(resource corev1.ResourceName) *Resou return nil } -func (c *ClusterQueueSnapshot) AddUsage(frq resources.FlavorResourceQuantitiesFlat) { +func (c *ClusterQueueSnapshot) AddUsage(frq resources.FlavorResourceQuantities) { c.addOrRemoveUsage(frq, 1) } -func (c *ClusterQueueSnapshot) Fits(frq resources.FlavorResourceQuantitiesFlat) bool { +func (c *ClusterQueueSnapshot) Fits(frq resources.FlavorResourceQuantities) bool { for fr, q := range frq { if c.Available(fr) < q { return false @@ -70,18 +70,18 @@ func (c *ClusterQueueSnapshot) Borrowing(fr resources.FlavorResource) bool { } func (c *ClusterQueueSnapshot) BorrowingWith(fr resources.FlavorResource, val int64) bool { - return c.usageFor(fr)+val > c.nominal(fr) + return c.Usage[fr]+val > c.nominal(fr) } func (c *ClusterQueueSnapshot) Available(fr resources.FlavorResource) int64 { if c.Cohort == nil { - return max(0, c.nominal(fr)-c.usageFor(fr)) + return max(0, c.nominal(fr)-c.Usage[fr]) } - capacityAvailable := c.RequestableCohortQuota(fr.Flavor, fr.Resource) - c.UsedCohortQuota(fr.Flavor, fr.Resource) + capacityAvailable := c.RequestableCohortQuota(fr) - c.UsedCohortQuota(fr) // if the borrowing limit exists, we cap our available capacity by the borrowing limit. if borrowingLimit := c.borrowingLimit(fr); borrowingLimit != nil { - withBorrowingRemaining := c.nominal(fr) + *borrowingLimit - c.usageFor(fr) + withBorrowingRemaining := c.nominal(fr) + *borrowingLimit - c.Usage[fr] capacityAvailable = min(capacityAvailable, withBorrowingRemaining) } return max(0, capacityAvailable) @@ -115,7 +115,7 @@ func (c *ClusterQueueSnapshot) lendableResourcesInCohort() map[corev1.ResourceNa } func (c *ClusterQueueSnapshot) usageFor(fr resources.FlavorResource) int64 { - return c.Usage.For(fr) + return c.Usage[fr] } func (c *ClusterQueueSnapshot) resourceGroups() []ResourceGroup { diff --git a/pkg/cache/clusterqueue_test.go b/pkg/cache/clusterqueue_test.go index f86168b1505..873653ecec6 100644 --- a/pkg/cache/clusterqueue_test.go +++ b/pkg/cache/clusterqueue_test.go @@ -96,16 +96,16 @@ func TestClusterQueueUpdateWithFlavors(t *testing.T) { func TestFitInCohort(t *testing.T) { cases := map[string]struct { - request resources.FlavorResourceQuantitiesFlat + request resources.FlavorResourceQuantities wantFit bool - usage resources.FlavorResourceQuantitiesFlat + usage resources.FlavorResourceQuantities clusterQueue []*kueue.ClusterQueue enableLendingLimit bool }{ "full cohort, empty request": { - request: resources.FlavorResourceQuantitiesFlat{}, + request: resources.FlavorResourceQuantities{}, wantFit: true, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 5_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, {Flavor: "f2", Resource: corev1.ResourceCPU}: 5_000, @@ -129,12 +129,12 @@ func TestFitInCohort(t *testing.T) { }, }, "can fit": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "f2", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f2", Resource: corev1.ResourceMemory}: 1, }, wantFit: true, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 5_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, {Flavor: "f2", Resource: corev1.ResourceCPU}: 4_000, @@ -158,14 +158,14 @@ func TestFitInCohort(t *testing.T) { }, }, "full cohort, none fit": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 1, {Flavor: "f2", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f2", Resource: corev1.ResourceMemory}: 1, }, wantFit: false, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 5_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, {Flavor: "f2", Resource: corev1.ResourceCPU}: 5_000, @@ -189,14 +189,14 @@ func TestFitInCohort(t *testing.T) { }, }, "one cannot fit": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 1, {Flavor: "f2", Resource: corev1.ResourceCPU}: 2_000, {Flavor: "f2", Resource: corev1.ResourceMemory}: 1, }, wantFit: false, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 4_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 4, {Flavor: "f2", Resource: corev1.ResourceCPU}: 4_000, @@ -220,12 +220,12 @@ func TestFitInCohort(t *testing.T) { }, }, "missing flavor": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "non-existent-flavor", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "non-existent-flavor", Resource: corev1.ResourceMemory}: 1, }, wantFit: false, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 5_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, }, @@ -243,12 +243,12 @@ func TestFitInCohort(t *testing.T) { }, }, "missing resource": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 1, }, wantFit: false, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 3_000, }, clusterQueue: []*kueue.ClusterQueue{ @@ -264,11 +264,11 @@ func TestFitInCohort(t *testing.T) { }, }, "lendingLimit enabled can't fit": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 3_000, }, wantFit: false, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 2_000, }, clusterQueue: []*kueue.ClusterQueue{ @@ -299,11 +299,11 @@ func TestFitInCohort(t *testing.T) { enableLendingLimit: true, }, "lendingLimit enabled can fit": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 3_000, }, wantFit: true, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 1_000, }, clusterQueue: []*kueue.ClusterQueue{ @@ -761,15 +761,15 @@ func TestClusterQueueUpdateWithAdmissionCheck(t *testing.T) { func TestDominantResourceShare(t *testing.T) { cases := map[string]struct { - usage resources.FlavorResourceQuantitiesFlat + usage resources.FlavorResourceQuantities clusterQueue *kueue.ClusterQueue lendingClusterQueue *kueue.ClusterQueue - flvResQ resources.FlavorResourceQuantitiesFlat + flvResQ resources.FlavorResourceQuantities wantDRValue int wantDRName corev1.ResourceName }{ "no cohort": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "default", Resource: "example.com/gpu"}: 2, }, @@ -782,7 +782,7 @@ func TestDominantResourceShare(t *testing.T) { ).Obj(), }, "usage below nominal": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "default", Resource: "example.com/gpu"}: 2, }, @@ -806,7 +806,7 @@ func TestDominantResourceShare(t *testing.T) { ).Obj(), }, "usage above nominal": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "default", Resource: "example.com/gpu"}: 7, }, @@ -832,7 +832,7 @@ func TestDominantResourceShare(t *testing.T) { wantDRValue: 200, // (7-5)*1000/10 }, "one resource above nominal": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "default", Resource: "example.com/gpu"}: 3, }, @@ -858,7 +858,7 @@ func TestDominantResourceShare(t *testing.T) { wantDRValue: 100, // (3-2)*1000/10 }, "usage with workload above nominal": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "default", Resource: "example.com/gpu"}: 2, }, @@ -880,7 +880,7 @@ func TestDominantResourceShare(t *testing.T) { ResourceQuotaWrapper("example.com/gpu").NominalQuota("5").Append(). FlavorQuotas, ).Obj(), - flvResQ: resources.FlavorResourceQuantitiesFlat{ + flvResQ: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, {Flavor: "default", Resource: "example.com/gpu"}: 4, }, @@ -888,7 +888,7 @@ func TestDominantResourceShare(t *testing.T) { wantDRValue: 300, // (1+4-2)*1000/10 }, "A resource with zero lendable": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "default", Resource: "example.com/gpu"}: 1, }, @@ -910,7 +910,7 @@ func TestDominantResourceShare(t *testing.T) { ResourceQuotaWrapper("example.com/gpu").NominalQuota("64").LendingLimit("0").Append(). FlavorQuotas, ).Obj(), - flvResQ: resources.FlavorResourceQuantitiesFlat{ + flvResQ: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, {Flavor: "default", Resource: "example.com/gpu"}: 4, }, @@ -918,7 +918,7 @@ func TestDominantResourceShare(t *testing.T) { wantDRValue: 300, // (1+4-2)*1000/10 }, "multiple flavors": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 15_000, {Flavor: "spot", Resource: corev1.ResourceCPU}: 5_000, }, @@ -941,14 +941,14 @@ func TestDominantResourceShare(t *testing.T) { ResourceQuotaWrapper("cpu").NominalQuota("100").Append(). FlavorQuotas, ).Obj(), - flvResQ: resources.FlavorResourceQuantitiesFlat{ + flvResQ: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10_000, }, wantDRName: corev1.ResourceCPU, wantDRValue: 25, // ((15+10-20)+0)*1000/200 (spot under nominal) }, "above nominal with integer weight": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: "example.com/gpu"}: 7, }, clusterQueue: utiltesting.MakeClusterQueue("cq"). @@ -971,7 +971,7 @@ func TestDominantResourceShare(t *testing.T) { wantDRValue: 100, // ((7-5)*1000/10)/2 }, "above nominal with decimal weight": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: "example.com/gpu"}: 7, }, clusterQueue: utiltesting.MakeClusterQueue("cq"). @@ -994,7 +994,7 @@ func TestDominantResourceShare(t *testing.T) { wantDRValue: 400, // ((7-5)*1000/10)/(1/2) }, "above nominal with zero weight": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: "example.com/gpu"}: 7, }, clusterQueue: utiltesting.MakeClusterQueue("cq"). diff --git a/pkg/cache/resource.go b/pkg/cache/resource.go index ca940a02660..d081f7c24ca 100644 --- a/pkg/cache/resource.go +++ b/pkg/cache/resource.go @@ -8,13 +8,16 @@ type resourceGroupNode interface { resourceGroups() []ResourceGroup } -func flavorResources(r resourceGroupNode) []resources.FlavorResource { - flavorResourceCount := 0 - for _, rg := range r.resourceGroups() { - flavorResourceCount += len(rg.Flavors) * len(rg.CoveredResources) +func flavorResourceCount(rgs []ResourceGroup) int { + count := 0 + for _, rg := range rgs { + count += len(rg.Flavors) * len(rg.CoveredResources) } + return count +} - frs := make([]resources.FlavorResource, 0, flavorResourceCount) +func flavorResources(r resourceGroupNode) []resources.FlavorResource { + frs := make([]resources.FlavorResource, 0, flavorResourceCount(r.resourceGroups())) for _, rg := range r.resourceGroups() { for _, f := range rg.Flavors { for r := range rg.CoveredResources { @@ -34,8 +37,8 @@ type netQuotaNode interface { // remainingQuota computes the remaining quota for each FlavorResource. A // negative value implies that the node is borrowing. -func remainingQuota(node netQuotaNode) resources.FlavorResourceQuantitiesFlat { - remainingQuota := make(resources.FlavorResourceQuantitiesFlat) +func remainingQuota(node netQuotaNode) resources.FlavorResourceQuantities { + remainingQuota := make(resources.FlavorResourceQuantities) for _, fr := range flavorResources(node) { remainingQuota[fr] += node.QuotaFor(fr).Nominal - node.usageFor(fr) } diff --git a/pkg/cache/snapshot.go b/pkg/cache/snapshot.go index 6c4d4c87318..884f7efca4c 100644 --- a/pkg/cache/snapshot.go +++ b/pkg/cache/snapshot.go @@ -52,7 +52,7 @@ func (s *Snapshot) AddWorkload(wl *workload.Info) { cq.addOrRemoveUsage(wl.FlavorResourceUsage(), 1) } -func (c *ClusterQueueSnapshot) addOrRemoveUsage(usage resources.FlavorResourceQuantitiesFlat, m int64) { +func (c *ClusterQueueSnapshot) addOrRemoveUsage(usage resources.FlavorResourceQuantities, m int64) { updateFlavorUsage(usage, c.Usage, m) if c.Cohort != nil { if features.Enabled(features.LendingLimit) { @@ -124,7 +124,7 @@ func (c *clusterQueue) snapshot() *ClusterQueueSnapshot { FlavorFungibility: c.FlavorFungibility, FairWeight: c.FairWeight, AllocatableResourceGeneration: c.AllocatableResourceGeneration, - Usage: make(resources.FlavorResourceQuantities, len(c.Usage)), + Usage: maps.Clone(c.Usage), Workloads: maps.Clone(c.Workloads), Preemption: c.Preemption, NamespaceSelector: c.NamespaceSelector, @@ -133,9 +133,6 @@ func (c *clusterQueue) snapshot() *ClusterQueueSnapshot { Quotas: c.quotas, } - for fName, rUsage := range c.Usage { - cc.Usage[fName] = maps.Clone(rUsage) - } if features.Enabled(features.LendingLimit) { cc.GuaranteedQuota = c.GuaranteedQuota } @@ -145,9 +142,11 @@ func (c *clusterQueue) snapshot() *ClusterQueueSnapshot { func (c *cohort) snapshotInto(cqs map[string]*ClusterQueueSnapshot) { cohortSnap := &CohortSnapshot{ - Name: c.Name, - Members: make(sets.Set[*ClusterQueueSnapshot], c.Members.Len()), - Lendable: c.CalculateLendable(), + Name: c.Name, + Members: make(sets.Set[*ClusterQueueSnapshot], c.Members.Len()), + Lendable: c.CalculateLendable(), + Usage: make(resources.FlavorResourceQuantities), + RequestableResources: make(resources.FlavorResourceQuantities), } cohortSnap.AllocatableResourceGeneration = 0 for cq := range c.Members { @@ -162,9 +161,6 @@ func (c *cohort) snapshotInto(cqs map[string]*ClusterQueueSnapshot) { } func (c *ClusterQueueSnapshot) accumulateResources(cohort *CohortSnapshot) { - if cohort.RequestableResources == nil { - cohort.RequestableResources = make(resources.FlavorResourceQuantities, len(c.ResourceGroups)) - } for _, rg := range c.ResourceGroups { for _, fName := range rg.Flavors { for rName := range rg.CoveredResources { @@ -175,23 +171,21 @@ func (c *ClusterQueueSnapshot) accumulateResources(cohort *CohortSnapshot) { // If LendingLimit is not nil, we should count the lendingLimit as the requestable // resource because we can't borrow more quota than lendingLimit. if features.Enabled(features.LendingLimit) && rQuota.LendingLimit != nil { - cohort.RequestableResources.Add(fr, *rQuota.LendingLimit) + cohort.RequestableResources[fr] += *rQuota.LendingLimit } else { - cohort.RequestableResources.Add(fr, rQuota.Nominal) + cohort.RequestableResources[fr] += rQuota.Nominal } } } } - for fName, resUsages := range c.Usage { - for res, val := range resUsages { - // Similar to cohort.RequestableResources, we accumulate the usage above the guaranteed resources, - // here we should remove the guaranteed quota as well for that part can not be borrowed. - val -= c.guaranteedQuota(fName, res) - // if val < 0, it means the cq is not using any quota belongs to LendingLimit - if val < 0 { - val = 0 - } - cohort.Usage.Add(resources.FlavorResource{Flavor: fName, Resource: res}, val) + for fr, val := range c.Usage { + // Similar to cohort.RequestableResources, we accumulate the usage above the guaranteed resources, + // here we should remove the guaranteed quota as well for that part can not be borrowed. + val -= c.guaranteedQuota(fr) + // if val < 0, it means the cq is not using any quota belongs to LendingLimit + if val < 0 { + val = 0 } + cohort.Usage[fr] += val } } diff --git a/pkg/cache/snapshot_test.go b/pkg/cache/snapshot_test.go index 4131607258c..cb26f812031 100644 --- a/pkg/cache/snapshot_test.go +++ b/pkg/cache/snapshot_test.go @@ -203,16 +203,16 @@ func TestSnapshot(t *testing.T) { cohort := &CohortSnapshot{ Name: "borrowing", AllocatableResourceGeneration: 2, - RequestableResources: resources.FlavorResourceQuantitiesFlat{ + RequestableResources: resources.FlavorResourceQuantities{ {Flavor: "demand", Resource: corev1.ResourceCPU}: 100_000, {Flavor: "spot", Resource: corev1.ResourceCPU}: 300_000, {Flavor: "default", Resource: "example.com/gpu"}: 50, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ + }, + Usage: resources.FlavorResourceQuantities{ {Flavor: "demand", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "spot", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "default", Resource: "example.com/gpu"}: 15, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 400_000, "example.com/gpu": 50, @@ -236,10 +236,10 @@ func TestSnapshot(t *testing.T) { {Flavor: "spot", Resource: corev1.ResourceCPU}: {Nominal: 200_000}, }, FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "demand", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Workloads: map[string]*workload.Info{ "/alpha": workload.NewInfo(utiltesting.MakeWorkload("alpha", ""). PodSets(*utiltesting.MakePodSet("main", 5). @@ -275,10 +275,10 @@ func TestSnapshot(t *testing.T) { {Flavor: "default", Resource: "example.com/gpu"}: {Nominal: 50}, }, FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "default", Resource: "example.com/gpu"}: 15, - }.Unflatten(), + }, Workloads: map[string]*workload.Info{ "/beta": workload.NewInfo(utiltesting.MakeWorkload("beta", ""). PodSets(*utiltesting.MakePodSet("main", 5). @@ -322,9 +322,9 @@ func TestSnapshot(t *testing.T) { {Flavor: "default", Resource: corev1.ResourceCPU}: {Nominal: 100_000}, }, FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Preemption: defaultPreemption, FairWeight: oneQuantity, NamespaceSelector: labels.Everything(), @@ -433,14 +433,14 @@ func TestSnapshot(t *testing.T) { cohort := &CohortSnapshot{ Name: "lending", AllocatableResourceGeneration: 2, - RequestableResources: resources.FlavorResourceQuantitiesFlat{ + RequestableResources: resources.FlavorResourceQuantities{ {Flavor: "arm", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "x86", Resource: corev1.ResourceCPU}: 20_000, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ + }, + Usage: resources.FlavorResourceQuantities{ {Flavor: "arm", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "x86", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 30_000, }, @@ -464,10 +464,10 @@ func TestSnapshot(t *testing.T) { }, FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "arm", Resource: corev1.ResourceCPU}: 15_000, {Flavor: "x86", Resource: corev1.ResourceCPU}: 10_000, - }.Unflatten(), + }, Workloads: map[string]*workload.Info{ "/alpha": workload.NewInfo(utiltesting.MakeWorkload("alpha", ""). PodSets(*utiltesting.MakePodSet("main", 5). @@ -494,10 +494,10 @@ func TestSnapshot(t *testing.T) { Preemption: defaultPreemption, NamespaceSelector: labels.Everything(), Status: active, - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "arm", Resource: corev1.ResourceCPU}: 5_000, {Flavor: "x86", Resource: corev1.ResourceCPU}: 10_000, - }.Unflatten(), + }, }, "b": { Name: "b", @@ -516,17 +516,17 @@ func TestSnapshot(t *testing.T) { }, FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "arm", Resource: corev1.ResourceCPU}: 0, {Flavor: "x86", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Preemption: defaultPreemption, NamespaceSelector: labels.Everything(), Status: active, - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "arm", Resource: corev1.ResourceCPU}: 5_000, {Flavor: "x86", Resource: corev1.ResourceCPU}: 10_000, - }.Unflatten(), + }, }, }, ResourceFlavors: map[kueue.ResourceFlavorReference]*kueue.ResourceFlavor{ @@ -658,11 +658,11 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { Name: "cohort", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, {Flavor: "alpha", Resource: corev1.ResourceMemory}: 0, {Flavor: "beta", Resource: corev1.ResourceMemory}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 12_000, corev1.ResourceMemory: 12 * utiltesting.Gi, @@ -678,11 +678,11 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, {Flavor: "alpha", Resource: corev1.ResourceMemory}: 0, {Flavor: "beta", Resource: corev1.ResourceMemory}: 0, - }.Unflatten(), + }, }, "c2": { Name: "c2", @@ -692,9 +692,9 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, } @@ -707,11 +707,11 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { Name: "cohort", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 2_000, {Flavor: "alpha", Resource: corev1.ResourceMemory}: utiltesting.Gi, {Flavor: "beta", Resource: corev1.ResourceMemory}: utiltesting.Gi, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 12_000, corev1.ResourceMemory: 12 * utiltesting.Gi, @@ -730,11 +730,11 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { ResourceGroups: cqCache.clusterQueues["c1"].ResourceGroups, FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, {Flavor: "alpha", Resource: corev1.ResourceMemory}: utiltesting.Gi, {Flavor: "beta", Resource: corev1.ResourceMemory}: utiltesting.Gi, - }.Unflatten(), + }, }, "c2": { Name: "c2", @@ -747,9 +747,9 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 2_000, - }.Unflatten(), + }, }, }, } @@ -762,11 +762,11 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { Name: "cohort", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "alpha", Resource: corev1.ResourceMemory}: 0, {Flavor: "beta", Resource: corev1.ResourceMemory}: utiltesting.Gi, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 12_000, corev1.ResourceMemory: 12 * utiltesting.Gi, @@ -785,11 +785,11 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { ResourceGroups: cqCache.clusterQueues["c1"].ResourceGroups, FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "alpha", Resource: corev1.ResourceMemory}: 0, {Flavor: "beta", Resource: corev1.ResourceMemory}: utiltesting.Gi, - }.Unflatten(), + }, }, "c2": { Name: "c2", @@ -802,9 +802,9 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { ResourceGroups: cqCache.clusterQueues["c2"].ResourceGroups, FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 2_000, - }.Unflatten(), + }, }, }, } @@ -914,9 +914,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -931,12 +931,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -946,12 +946,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } @@ -964,9 +964,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -981,12 +981,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 7_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -996,12 +996,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } @@ -1014,9 +1014,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -1031,12 +1031,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -1046,12 +1046,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } @@ -1064,9 +1064,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -1081,12 +1081,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -1096,12 +1096,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } @@ -1115,9 +1115,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -1132,12 +1132,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -1147,12 +1147,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } @@ -1166,9 +1166,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -1183,12 +1183,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -1198,12 +1198,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } @@ -1217,9 +1217,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -1234,12 +1234,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 9_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -1249,12 +1249,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } diff --git a/pkg/resources/resource.go b/pkg/resources/resource.go index 6bddd55ac89..d95d06ffca6 100644 --- a/pkg/resources/resource.go +++ b/pkg/resources/resource.go @@ -27,33 +27,4 @@ type FlavorResource struct { Resource corev1.ResourceName } -type FlavorResourceQuantities map[kueue.ResourceFlavorReference]Requests -type FlavorResourceQuantitiesFlat map[FlavorResource]int64 - -func (f FlavorResourceQuantitiesFlat) Unflatten() FlavorResourceQuantities { - out := make(FlavorResourceQuantities) - for flavorResource, value := range f { - if _, ok := out[flavorResource.Flavor]; !ok { - out[flavorResource.Flavor] = make(Requests) - } - out[flavorResource.Flavor][flavorResource.Resource] = value - } - return out -} - -// For attempts to access nested value, returning 0 if absent. -func (f FlavorResourceQuantities) For(fr FlavorResource) int64 { - return f[fr.Flavor][fr.Resource] -} - -// Add adds the Quantity v for the FlavorResource fr, allocating -// as needed. -func (f *FlavorResourceQuantities) Add(fr FlavorResource, v int64) { - if *f == nil { - *f = make(FlavorResourceQuantities) - } - if (*f)[fr.Flavor] == nil { - (*f)[fr.Flavor] = make(Requests) - } - (*f)[fr.Flavor][fr.Resource] += v -} +type FlavorResourceQuantities map[FlavorResource]int64 diff --git a/pkg/resources/resource_test.go b/pkg/resources/resource_test.go deleted file mode 100644 index 57ed9fa2430..00000000000 --- a/pkg/resources/resource_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package resources - -import ( - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestSafeAdd(t *testing.T) { - var a FlavorResourceQuantities = nil - fr := FlavorResource{Flavor: "Hello", Resource: "World"} - - defer func() { - if r := recover(); r != nil { - t.Errorf("Panic while assigning to nil map") - } - }() - a.Add(fr, 5) - a.Add(fr, -7) - - expected := FlavorResourceQuantitiesFlat{fr: -2}.Unflatten() - if diff := cmp.Diff(a, expected); diff != "" { - t.Fatalf("Unexpected diff %s", diff) - } -} - -func TestFor(t *testing.T) { - var a FlavorResourceQuantities = nil - - defer func() { - if r := recover(); r != nil { - t.Errorf("Panic while accessing nil map") - } - }() - result := a.For(FlavorResource{Flavor: "Hello", Resource: "World"}) - - if result != 0 { - t.Fatalf("Unexpected result: %d != 0", result) - } -} diff --git a/pkg/scheduler/flavorassigner/flavorassigner.go b/pkg/scheduler/flavorassigner/flavorassigner.go index df26a0f070b..1d7204c35b4 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner.go +++ b/pkg/scheduler/flavorassigner/flavorassigner.go @@ -46,7 +46,7 @@ type Assignment struct { // Usage is the accumulated Usage of resources as pod sets get // flavors assigned. - Usage resources.FlavorResourceQuantitiesFlat + Usage resources.FlavorResourceQuantities // representativeMode is the cached representative mode for this assignment. representativeMode *FlavorAssignmentMode @@ -106,8 +106,8 @@ func (a *Assignment) ToAPI() []kueue.PodSetAssignment { return psFlavors } -func (a *Assignment) TotalRequestsFor(wl *workload.Info) resources.FlavorResourceQuantitiesFlat { - usage := make(resources.FlavorResourceQuantitiesFlat) +func (a *Assignment) TotalRequestsFor(wl *workload.Info) resources.FlavorResourceQuantities { + usage := make(resources.FlavorResourceQuantities) for i, ps := range wl.TotalRequests { for res, q := range ps.Requests { flv := a.PodSets[i].Flavors[res].Name @@ -296,7 +296,7 @@ func (a *FlavorAssigner) Assign(log logr.Logger, counts []int32) Assignment { func (a *FlavorAssigner) assignFlavors(log logr.Logger, requests []workload.PodSetResources) Assignment { assignment := Assignment{ PodSets: make([]PodSetAssignment, 0, len(requests)), - Usage: make(resources.FlavorResourceQuantitiesFlat), + Usage: make(resources.FlavorResourceQuantities), LastState: workload.AssignmentClusterQueueState{ LastTriedFlavorIdx: make([]map[corev1.ResourceName]int, 0, len(requests)), CohortGeneration: 0, @@ -377,7 +377,7 @@ func (a *FlavorAssigner) findFlavorForPodSetResource( psID int, requests resources.Requests, resName corev1.ResourceName, - assignmentUsage resources.FlavorResourceQuantitiesFlat, + assignmentUsage resources.FlavorResourceQuantities, ) (ResourceAssignment, *Status) { resourceGroup := a.cq.RGByResource(resName) if resourceGroup == nil { @@ -429,7 +429,7 @@ func (a *FlavorAssigner) findFlavorForPodSetResource( resQuota := a.cq.QuotaFor(resources.FlavorResource{Flavor: fName, Resource: rName}) // Check considering the flavor usage by previous pod sets. fr := resources.FlavorResource{Flavor: fName, Resource: rName} - mode, borrow, s := a.fitsResourceQuota(fName, rName, val+assignmentUsage[fr], resQuota) + mode, borrow, s := a.fitsResourceQuota(fr, val+assignmentUsage[fr], resQuota) if s != nil { status.reasons = append(status.reasons, s.reasons...) } @@ -557,10 +557,10 @@ func flavorSelector(spec *corev1.PodSpec, allowedKeys sets.Set[string]) nodeaffi // if borrowing is required when preempting. // If the flavor doesn't satisfy limits immediately (when waiting or preemption // could help), it returns a Status with reasons. -func (a *FlavorAssigner) fitsResourceQuota(fName kueue.ResourceFlavorReference, rName corev1.ResourceName, val int64, rQuota *cache.ResourceQuota) (FlavorAssignmentMode, bool, *Status) { +func (a *FlavorAssigner) fitsResourceQuota(fr resources.FlavorResource, val int64, rQuota *cache.ResourceQuota) (FlavorAssignmentMode, bool, *Status) { var status Status var borrow bool - used := a.cq.Usage[fName][rName] + used := a.cq.Usage[fr] mode := NoFit if val <= rQuota.Nominal { // The request can be satisfied by the nominal quota, assuming quota is @@ -570,7 +570,7 @@ func (a *FlavorAssigner) fitsResourceQuota(fName kueue.ResourceFlavorReference, } cohortAvailable := rQuota.Nominal if a.cq.Cohort != nil { - cohortAvailable = a.cq.RequestableCohortQuota(fName, rName) + cohortAvailable = a.cq.RequestableCohortQuota(fr) } if a.canPreemptWhileBorrowing() { @@ -582,13 +582,13 @@ func (a *FlavorAssigner) fitsResourceQuota(fName kueue.ResourceFlavorReference, } } if rQuota.BorrowingLimit != nil && used+val > rQuota.Nominal+*rQuota.BorrowingLimit { - status.append(fmt.Sprintf("borrowing limit for %s in flavor %s exceeded", rName, fName)) + status.append(fmt.Sprintf("borrowing limit for %s in flavor %s exceeded", fr.Resource, fr.Flavor)) return mode, borrow, &status } cohortUsed := used if a.cq.Cohort != nil { - cohortUsed = a.cq.UsedCohortQuota(fName, rName) + cohortUsed = a.cq.UsedCohortQuota(fr) } lack := cohortUsed + val - cohortAvailable @@ -596,13 +596,13 @@ func (a *FlavorAssigner) fitsResourceQuota(fName kueue.ResourceFlavorReference, return Fit, used+val > rQuota.Nominal, nil } - lackQuantity := resources.ResourceQuantity(rName, lack) - msg := fmt.Sprintf("insufficient unused quota in cohort for %s in flavor %s, %s more needed", rName, fName, &lackQuantity) + lackQuantity := resources.ResourceQuantity(fr.Resource, lack) + msg := fmt.Sprintf("insufficient unused quota in cohort for %s in flavor %s, %s more needed", fr.Resource, fr.Flavor, &lackQuantity) if a.cq.Cohort == nil { if mode == NoFit { - msg = fmt.Sprintf("insufficient quota for %s in flavor %s in ClusterQueue", rName, fName) + msg = fmt.Sprintf("insufficient quota for %s in flavor %s in ClusterQueue", fr.Resource, fr.Flavor) } else { - msg = fmt.Sprintf("insufficient unused quota for %s in flavor %s, %s more needed", rName, fName, &lackQuantity) + msg = fmt.Sprintf("insufficient unused quota for %s in flavor %s, %s more needed", fr.Resource, fr.Flavor, &lackQuantity) } } status.append(msg) diff --git a/pkg/scheduler/flavorassigner/flavorassigner_test.go b/pkg/scheduler/flavorassigner/flavorassigner_test.go index 83516797fb0..105f364265b 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner_test.go +++ b/pkg/scheduler/flavorassigner/flavorassigner_test.go @@ -37,8 +37,8 @@ import ( ) type cohortResources struct { - requestableResources resources.FlavorResourceQuantitiesFlat - usage resources.FlavorResourceQuantitiesFlat + requestableResources resources.FlavorResourceQuantities + usage resources.FlavorResourceQuantities } func TestAssignFlavors(t *testing.T) { @@ -62,7 +62,7 @@ func TestAssignFlavors(t *testing.T) { wlPods []kueue.PodSet wlReclaimablePods []kueue.ReclaimablePod clusterQueue kueue.ClusterQueue - clusterQueueUsage resources.FlavorResourceQuantitiesFlat + clusterQueueUsage resources.FlavorResourceQuantities cohortResources *cohortResources wantRepMode FlavorAssignmentMode wantAssignment Assignment @@ -99,7 +99,7 @@ func TestAssignFlavors(t *testing.T) { Count: 1, }, }, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "default", Resource: corev1.ResourceMemory}: utiltesting.Mi, }, @@ -136,7 +136,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "tainted", Resource: corev1.ResourceCPU}: 1_000, }, }, @@ -153,7 +153,7 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "4"). FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, }, wantRepMode: Preempt, @@ -171,7 +171,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 2_000, }, }, @@ -215,7 +215,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "b_one", Resource: corev1.ResourceMemory}: 10 * utiltesting.Mi, }, @@ -239,7 +239,7 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 1_000, }, @@ -257,7 +257,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, "multiple resource groups with multiple resources, fits": { @@ -303,7 +303,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "two", Resource: corev1.ResourceMemory}: 10 * utiltesting.Mi, {Flavor: "b_one", Resource: "example.com/gpu"}: 3, @@ -334,18 +334,18 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceMemory}: 10 * utiltesting.Mi, }, cohortResources: &cohortResources{ - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, {Flavor: "one", Resource: corev1.ResourceMemory}: utiltesting.Gi, {Flavor: "two", Resource: corev1.ResourceCPU}: 4_000, {Flavor: "two", Resource: corev1.ResourceMemory}: 15 * utiltesting.Mi, {Flavor: "b_one", Resource: "example.com/gpu"}: 4, }, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceMemory}: 10 * utiltesting.Mi, {Flavor: "b_one", Resource: "example.com/gpu"}: 2, }, @@ -373,7 +373,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "two", Resource: corev1.ResourceMemory}: 10 * utiltesting.Mi, {Flavor: "b_one", Resource: "example.com/gpu"}: 3, @@ -413,7 +413,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, "multiple flavors, fits while skipping tainted flavor": { @@ -443,7 +443,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 3_000, }, }, @@ -501,7 +501,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 1_000, }, }, @@ -563,7 +563,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "two", Resource: corev1.ResourceMemory}: utiltesting.Mi, }, @@ -633,7 +633,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 1_000, }, }, @@ -691,7 +691,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, "multiple specs, fit different flavors": { @@ -737,7 +737,7 @@ func TestAssignFlavors(t *testing.T) { Count: 1, }, }, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 5_000, }, @@ -764,7 +764,7 @@ func TestAssignFlavors(t *testing.T) { ClusterQueue, cohortResources: &cohortResources{ - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 200_000, {Flavor: "default", Resource: corev1.ResourceMemory}: 200 * utiltesting.Gi, }, @@ -798,7 +798,7 @@ func TestAssignFlavors(t *testing.T) { }, }, Borrowing: true, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "default", Resource: corev1.ResourceMemory}: 5 * utiltesting.Gi, }, @@ -818,10 +818,10 @@ func TestAssignFlavors(t *testing.T) { ).Cohort("test-cohort").ClusterQueue, cohortResources: &cohortResources{ - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 9_000, }, }, @@ -836,7 +836,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, "past max, but can preempt in ClusterQueue": { @@ -852,14 +852,14 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 9_000, }, cohortResources: &cohortResources{ - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 100_000, }, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 9_000, }, }, @@ -879,7 +879,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, }, @@ -896,7 +896,7 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "2"). FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 1_000, }, wantRepMode: Preempt, @@ -914,7 +914,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, }, @@ -931,14 +931,14 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "3"). FlavorQuotas, ).Cohort("test-cohort").ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, }, @@ -957,7 +957,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, }, @@ -986,7 +986,7 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "4"). FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 3_000, }, @@ -1008,7 +1008,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 2_000, }, }, @@ -1037,7 +1037,7 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "10"). FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "tainted", Resource: corev1.ResourceCPU}: 3_000, }, @@ -1077,7 +1077,7 @@ func TestAssignFlavors(t *testing.T) { Count: 10, }, }, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, {Flavor: "tainted", Resource: corev1.ResourceCPU}: 10_000, }, @@ -1106,7 +1106,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, "num pods fit": { @@ -1137,7 +1137,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 3, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourcePods}: 3, {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, }, @@ -1170,7 +1170,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 3, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, "with reclaimable pods": { @@ -1206,7 +1206,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 3, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourcePods}: 3, {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, }, @@ -1231,7 +1231,7 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "10"). FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, wantRepMode: Preempt, @@ -1251,7 +1251,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: "cpu"}: 9_000, {Flavor: "one", Resource: "pods"}: 1, }, @@ -1274,7 +1274,7 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "10"). FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, wantRepMode: Fit, @@ -1291,7 +1291,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: "cpu"}: 9_000, {Flavor: "two", Resource: "pods"}: 1, }, @@ -1316,14 +1316,14 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 11_000, {Flavor: "one", Resource: corev1.ResourcePods}: 10, {Flavor: "two", Resource: corev1.ResourceCPU}: 1_000, @@ -1345,7 +1345,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 9_000, {Flavor: "one", Resource: corev1.ResourcePods}: 1, }, @@ -1370,15 +1370,15 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 11_000, {Flavor: "one", Resource: corev1.ResourcePods}: 10, {Flavor: "two", Resource: corev1.ResourceCPU}: 10_000, @@ -1399,7 +1399,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 9_000, {Flavor: "two", Resource: corev1.ResourcePods}: 1, }, @@ -1423,15 +1423,15 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 11_000, {Flavor: "one", Resource: corev1.ResourcePods}: 10, {Flavor: "two", Resource: corev1.ResourceCPU}: 10_000, @@ -1453,7 +1453,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: "cpu"}: 9_000, {Flavor: "one", Resource: "pods"}: 1, }, @@ -1486,10 +1486,10 @@ func TestAssignFlavors(t *testing.T) { ).Cohort("test-cohort").ClusterQueue, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, @@ -1510,7 +1510,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, }, }, @@ -1541,10 +1541,10 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort").ClusterQueue, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, @@ -1565,7 +1565,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, }, }, @@ -1597,10 +1597,10 @@ func TestAssignFlavors(t *testing.T) { ).Cohort("test-cohort"). ClusterQueue, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, @@ -1617,7 +1617,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, }, @@ -1641,10 +1641,10 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort").ClusterQueue, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ { // below the borrowingLimit required to admit Flavor: "one", Resource: corev1.ResourceCPU}: 11_000, @@ -1652,7 +1652,7 @@ func TestAssignFlavors(t *testing.T) { }, wantRepMode: NoFit, wantAssignment: Assignment{ - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, PodSets: []PodSetAssignment{ { Name: "main", @@ -1686,14 +1686,14 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 11_000, {Flavor: "one", Resource: corev1.ResourcePods}: 10, {Flavor: "two", Resource: corev1.ResourceCPU}: 10_000, @@ -1714,7 +1714,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 9_000, {Flavor: "two", Resource: corev1.ResourcePods}: 1, }, @@ -1740,14 +1740,14 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 11_000, {Flavor: "one", Resource: corev1.ResourcePods}: 10, {Flavor: "two", Resource: corev1.ResourceCPU}: 1_000, @@ -1769,7 +1769,7 @@ func TestAssignFlavors(t *testing.T) { Count: 1, }}, Borrowing: true, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 9_000, {Flavor: "one", Resource: corev1.ResourcePods}: 1, }, @@ -1789,14 +1789,14 @@ func TestAssignFlavors(t *testing.T) { ResourceQuotaWrapper(corev1.ResourceCPU).NominalQuota("10").LendingLimit("0").Append(). FlavorQuotas, ).Cohort("test-cohort").ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "one", Resource: corev1.ResourcePods}: 10, }, @@ -1818,7 +1818,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 9_000, {Flavor: "one", Resource: corev1.ResourcePods}: 1, }, @@ -1844,10 +1844,10 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort").ClusterQueue, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, @@ -1868,7 +1868,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, }, }, @@ -1892,10 +1892,10 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort").ClusterQueue, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, @@ -1912,7 +1912,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, }, @@ -1951,10 +1951,10 @@ func TestAssignFlavors(t *testing.T) { if clusterQueue.Cohort == nil { t.Fatalf("Test case has cohort resources, but cluster queue doesn't have cohort") } - clusterQueue.Cohort.Usage = tc.cohortResources.usage.Unflatten() - clusterQueue.Cohort.RequestableResources = tc.cohortResources.requestableResources.Unflatten() + clusterQueue.Cohort.Usage = tc.cohortResources.usage + clusterQueue.Cohort.RequestableResources = tc.cohortResources.requestableResources } - clusterQueue.Usage = tc.clusterQueueUsage.Unflatten() + clusterQueue.Usage = tc.clusterQueueUsage flvAssigner := New(wlInfo, clusterQueue, resourceFlavors, tc.enableFairSharing) assignment := flvAssigner.Assign(log, nil) @@ -2006,7 +2006,7 @@ func TestDeletedFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "flavor", Resource: corev1.ResourceCPU}: 3_000, }, }, @@ -2034,7 +2034,7 @@ func TestDeletedFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, } diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go index cd0b3ae30b7..219c706ce2a 100644 --- a/pkg/scheduler/preemption/preemption.go +++ b/pkg/scheduler/preemption/preemption.go @@ -231,7 +231,7 @@ func (p *Preemptor) applyPreemptionWithSSA(ctx context.Context, w *kueue.Workloa // Once the Workload fits, the heuristic tries to add Workloads back, in the // reverse order in which they were removed, while the incoming Workload still // fits. -func minimalPreemptions(log logr.Logger, wlReq resources.FlavorResourceQuantitiesFlat, cq *cache.ClusterQueueSnapshot, snapshot *cache.Snapshot, resPerFlv resourcesPerFlavor, candidates []*workload.Info, allowBorrowing bool, allowBorrowingBelowPriority *int32) []*Target { +func minimalPreemptions(log logr.Logger, wlReq resources.FlavorResourceQuantities, cq *cache.ClusterQueueSnapshot, snapshot *cache.Snapshot, resPerFlv resourcesPerFlavor, candidates []*workload.Info, allowBorrowing bool, allowBorrowingBelowPriority *int32) []*Target { if logV := log.V(5); logV.Enabled() { logV.Info("Simulating preemption", "candidates", workload.References(candidates), "resourcesRequiringPreemption", resPerFlv, "allowBorrowing", allowBorrowing, "allowBorrowingBelowPriority", allowBorrowingBelowPriority) } @@ -286,7 +286,7 @@ func minimalPreemptions(log logr.Logger, wlReq resources.FlavorResourceQuantitie return targets } -func fillBackWorkloads(targets []*Target, wlReq resources.FlavorResourceQuantitiesFlat, cq *cache.ClusterQueueSnapshot, snapshot *cache.Snapshot, allowBorrowing bool) []*Target { +func fillBackWorkloads(targets []*Target, wlReq resources.FlavorResourceQuantities, cq *cache.ClusterQueueSnapshot, snapshot *cache.Snapshot, allowBorrowing bool) []*Target { // In the reverse order, check if any of the workloads can be added back. for i := len(targets) - 2; i >= 0; i-- { snapshot.AddWorkload(targets[i].WorkloadInfo) @@ -563,7 +563,7 @@ func workloadUsesResources(wl *workload.Info, resPerFlv resourcesPerFlavor) bool // workloadFits determines if the workload requests would fit given the // requestable resources and simulated usage of the ClusterQueue and its cohort, // if it belongs to one. -func workloadFits(wlReq resources.FlavorResourceQuantitiesFlat, cq *cache.ClusterQueueSnapshot, allowBorrowing bool) bool { +func workloadFits(wlReq resources.FlavorResourceQuantities, cq *cache.ClusterQueueSnapshot, allowBorrowing bool) bool { for fr, v := range wlReq { if !allowBorrowing && cq.BorrowingWith(fr, v) { return false @@ -577,7 +577,7 @@ func workloadFits(wlReq resources.FlavorResourceQuantitiesFlat, cq *cache.Cluste func queueUnderNominalInResourcesNeedingPreemption(resPerFlv resourcesPerFlavor, cq *cache.ClusterQueueSnapshot) bool { for fr := range resPerFlv { - if cq.Usage.For(fr) >= cq.QuotaFor(fr).Nominal { + if cq.Usage[fr] >= cq.QuotaFor(fr).Nominal { return false } } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index f4eab64aa6a..1c256c9cb69 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -144,11 +144,11 @@ func (s *Scheduler) setAdmissionRoutineWrapper(wrapper routine.Wrapper) { s.admissionRoutineWrapper = wrapper } -type cohortsUsage map[string]resources.FlavorResourceQuantitiesFlat +type cohortsUsage map[string]resources.FlavorResourceQuantities -func (cu cohortsUsage) add(cohort string, assignment resources.FlavorResourceQuantitiesFlat) { +func (cu cohortsUsage) add(cohort string, assignment resources.FlavorResourceQuantities) { if cu[cohort] == nil { - cu[cohort] = make(resources.FlavorResourceQuantitiesFlat, len(assignment)) + cu[cohort] = make(resources.FlavorResourceQuantities, len(assignment)) } for fr, v := range assignment { @@ -156,11 +156,11 @@ func (cu cohortsUsage) add(cohort string, assignment resources.FlavorResourceQua } } -func (cu cohortsUsage) totalUsageForCommonFlavorResources(cohort string, assignment resources.FlavorResourceQuantitiesFlat) resources.FlavorResourceQuantitiesFlat { +func (cu cohortsUsage) totalUsageForCommonFlavorResources(cohort string, assignment resources.FlavorResourceQuantities) resources.FlavorResourceQuantities { return utilmaps.Intersect(cu[cohort], assignment, func(a, b int64) int64 { return a + b }) } -func (cu cohortsUsage) hasCommonFlavorResources(cohort string, assignment resources.FlavorResourceQuantitiesFlat) bool { +func (cu cohortsUsage) hasCommonFlavorResources(cohort string, assignment resources.FlavorResourceQuantities) bool { cohortUsage, cohortFound := cu[cohort] if !cohortFound { return false @@ -364,7 +364,7 @@ type entry struct { // netUsage returns how much capacity this entry will require from the ClusterQueue/Cohort. // When a workload is preempting, it subtracts the preempted resources from the resources // required, as the remaining quota is all we need from the CQ/Cohort. -func (e *entry) netUsage() resources.FlavorResourceQuantitiesFlat { +func (e *entry) netUsage() resources.FlavorResourceQuantities { if e.assignment.RepresentativeMode() == flavorassigner.Fit { return e.assignment.Usage } @@ -423,21 +423,21 @@ func (s *Scheduler) nominate(ctx context.Context, workloads []workload.Info, sna } // resourcesToReserve calculates how much of the available resources in cq/cohort assignment should be reserved. -func resourcesToReserve(e *entry, cq *cache.ClusterQueueSnapshot) resources.FlavorResourceQuantitiesFlat { +func resourcesToReserve(e *entry, cq *cache.ClusterQueueSnapshot) resources.FlavorResourceQuantities { if e.assignment.RepresentativeMode() != flavorassigner.Preempt { return e.assignment.Usage } - reservedUsage := make(resources.FlavorResourceQuantitiesFlat) + reservedUsage := make(resources.FlavorResourceQuantities) for fr, usage := range e.assignment.Usage { cqQuota := cq.QuotaFor(fr) if e.assignment.Borrowing { if cqQuota.BorrowingLimit == nil { reservedUsage[fr] = usage } else { - reservedUsage[fr] = min(usage, cqQuota.Nominal+*cqQuota.BorrowingLimit-cq.Usage.For(fr)) + reservedUsage[fr] = min(usage, cqQuota.Nominal+*cqQuota.BorrowingLimit-cq.Usage[fr]) } } else { - reservedUsage[fr] = max(0, min(usage, cqQuota.Nominal-cq.Usage.For(fr))) + reservedUsage[fr] = max(0, min(usage, cqQuota.Nominal-cq.Usage[fr])) } } return reservedUsage diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index cd6180ce506..fccedddabc2 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -3281,24 +3281,24 @@ func TestResourcesToReserve(t *testing.T) { name string assignmentMode flavorassigner.FlavorAssignmentMode borrowing bool - assignmentUsage resources.FlavorResourceQuantitiesFlat - cqUsage resources.FlavorResourceQuantitiesFlat - wantReserved resources.FlavorResourceQuantitiesFlat + assignmentUsage resources.FlavorResourceQuantities + cqUsage resources.FlavorResourceQuantities + wantReserved resources.FlavorResourceQuantities }{ { name: "Reserved memory and gpu less than assignment usage, assignment preempts", assignmentMode: flavorassigner.Preempt, - assignmentUsage: resources.FlavorResourceQuantitiesFlat{ + assignmentUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 6, }, - cqUsage: resources.FlavorResourceQuantitiesFlat{ + cqUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 6, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, - wantReserved: resources.FlavorResourceQuantitiesFlat{ + wantReserved: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 40, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 4, }, @@ -3306,17 +3306,17 @@ func TestResourcesToReserve(t *testing.T) { { name: "Reserved memory equal assignment usage, assignment preempts", assignmentMode: flavorassigner.Preempt, - assignmentUsage: resources.FlavorResourceQuantitiesFlat{ + assignmentUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 30, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, }, - cqUsage: resources.FlavorResourceQuantitiesFlat{ + cqUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, - wantReserved: resources.FlavorResourceQuantitiesFlat{ + wantReserved: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 30, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, }, @@ -3324,17 +3324,17 @@ func TestResourcesToReserve(t *testing.T) { { name: "Reserved memory equal assignment usage, assignment fits", assignmentMode: flavorassigner.Fit, - assignmentUsage: resources.FlavorResourceQuantitiesFlat{ + assignmentUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, }, - cqUsage: resources.FlavorResourceQuantitiesFlat{ + cqUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, - wantReserved: resources.FlavorResourceQuantitiesFlat{ + wantReserved: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, }, @@ -3342,17 +3342,17 @@ func TestResourcesToReserve(t *testing.T) { { name: "Reserved memory is 0, CQ is borrowing, assignment preempts without borrowing", assignmentMode: flavorassigner.Preempt, - assignmentUsage: resources.FlavorResourceQuantitiesFlat{ + assignmentUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, - cqUsage: resources.FlavorResourceQuantitiesFlat{ + cqUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 10, }, - wantReserved: resources.FlavorResourceQuantitiesFlat{ + wantReserved: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 0, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 0, }, @@ -3361,17 +3361,17 @@ func TestResourcesToReserve(t *testing.T) { name: "Reserved memory cut by nominal+borrowing quota, assignment preempts and borrows", assignmentMode: flavorassigner.Preempt, borrowing: true, - assignmentUsage: resources.FlavorResourceQuantitiesFlat{ + assignmentUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, - cqUsage: resources.FlavorResourceQuantitiesFlat{ + cqUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 10, }, - wantReserved: resources.FlavorResourceQuantitiesFlat{ + wantReserved: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 40, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, @@ -3380,17 +3380,17 @@ func TestResourcesToReserve(t *testing.T) { name: "Reserved memory equal assignment usage, CQ borrowing limit is nil", assignmentMode: flavorassigner.Preempt, borrowing: true, - assignmentUsage: resources.FlavorResourceQuantitiesFlat{ + assignmentUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, - cqUsage: resources.FlavorResourceQuantitiesFlat{ + cqUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 10, }, - wantReserved: resources.FlavorResourceQuantitiesFlat{ + wantReserved: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, diff --git a/pkg/workload/workload.go b/pkg/workload/workload.go index 9dd9da29d99..5e26391b91a 100644 --- a/pkg/workload/workload.go +++ b/pkg/workload/workload.go @@ -206,8 +206,8 @@ func (i *Info) CanBePartiallyAdmitted() bool { // FlavorResourceUsage returns the total resource usage for the workload, // per flavor (if assigned, otherwise flavor shows as empty string), per resource. -func (i *Info) FlavorResourceUsage() resources.FlavorResourceQuantitiesFlat { - total := make(resources.FlavorResourceQuantitiesFlat) +func (i *Info) FlavorResourceUsage() resources.FlavorResourceQuantities { + total := make(resources.FlavorResourceQuantities) if i == nil { return total } diff --git a/pkg/workload/workload_test.go b/pkg/workload/workload_test.go index ea5574e6c18..0a9de5da282 100644 --- a/pkg/workload/workload_test.go +++ b/pkg/workload/workload_test.go @@ -593,10 +593,10 @@ func TestIsEvictedByPodsReadyTimeout(t *testing.T) { func TestFlavorResourceUsage(t *testing.T) { cases := map[string]struct { info *Info - want resources.FlavorResourceQuantitiesFlat + want resources.FlavorResourceQuantities }{ "nil": { - want: resources.FlavorResourceQuantitiesFlat{}, + want: resources.FlavorResourceQuantities{}, }, "one podset, no flavors": { info: &Info{ @@ -607,7 +607,7 @@ func TestFlavorResourceUsage(t *testing.T) { }, }}, }, - want: resources.FlavorResourceQuantitiesFlat{ + want: resources.FlavorResourceQuantities{ {Flavor: "", Resource: "cpu"}: 1_000, {Flavor: "", Resource: "example.com/gpu"}: 3, }, @@ -625,7 +625,7 @@ func TestFlavorResourceUsage(t *testing.T) { }, }}, }, - want: resources.FlavorResourceQuantitiesFlat{ + want: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: "cpu"}: 1_000, {Flavor: "gpu", Resource: "example.com/gpu"}: 3, }, @@ -663,7 +663,7 @@ func TestFlavorResourceUsage(t *testing.T) { }, }, }, - want: resources.FlavorResourceQuantitiesFlat{ + want: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: "cpu"}: 3_000, {Flavor: "default", Resource: "memory"}: 2 * utiltesting.Gi, {Flavor: "model_a", Resource: "example.com/gpu"}: 3, From 9a2a414296f4b0a3d88ebc841bb71b1428c9c394 Mon Sep 17 00:00:00 2001 From: gabesaba <15304068+gabesaba@users.noreply.github.com> Date: Fri, 9 Aug 2024 09:37:44 +0200 Subject: [PATCH 4/6] Cleanup preemption.go (#2800) --- pkg/scheduler/preemption/preemption.go | 105 +++++++++++-------------- 1 file changed, 48 insertions(+), 57 deletions(-) diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go index 219c706ce2a..193d163305b 100644 --- a/pkg/scheduler/preemption/preemption.go +++ b/pkg/scheduler/preemption/preemption.go @@ -103,17 +103,17 @@ type Target struct { // GetTargets returns the list of workloads that should be evicted in order to make room for wl. func (p *Preemptor) GetTargets(log logr.Logger, wl workload.Info, assignment flavorassigner.Assignment, snapshot *cache.Snapshot) []*Target { - resPerFlv := resourcesRequiringPreemption(assignment) + frsNeedPreemption := flavorResourcesNeedPreemption(assignment) cq := snapshot.ClusterQueues[wl.ClusterQueue] - candidates := findCandidates(wl.Obj, p.workloadOrdering, cq, resPerFlv) + candidates := p.findCandidates(wl.Obj, cq, frsNeedPreemption) if len(candidates) == 0 { return nil } sort.Slice(candidates, candidatesOrdering(candidates, cq.Name, time.Now())) sameQueueCandidates := candidatesOnlyFromQueue(candidates, wl.ClusterQueue) - wlReq := assignment.TotalRequestsFor(&wl) + requests := assignment.TotalRequestsFor(&wl) // To avoid flapping, Kueue only allows preemption of workloads from the same // queue if borrowing. Preemption of workloads from queues can happen only @@ -123,12 +123,12 @@ func (p *Preemptor) GetTargets(log logr.Logger, wl workload.Info, assignment fla if len(sameQueueCandidates) == len(candidates) { // There is no possible preemption of workloads from other queues, // so we'll try borrowing. - return minimalPreemptions(log, wlReq, cq, snapshot, resPerFlv, candidates, true, nil) + return minimalPreemptions(log, requests, cq, snapshot, frsNeedPreemption, candidates, true, nil) } borrowWithinCohort, thresholdPrio := canBorrowWithinCohort(cq, wl.Obj) if p.enableFairSharing { - return p.fairPreemptions(log, &wl, assignment, snapshot, resPerFlv, candidates, thresholdPrio) + return p.fairPreemptions(log, wl, requests, snapshot, frsNeedPreemption, candidates, thresholdPrio) } // There is a potential of preemption of workloads from the other queue in the // cohort. We proceed with borrowing only if the dedicated policy @@ -136,24 +136,24 @@ func (p *Preemptor) GetTargets(log logr.Logger, wl workload.Info, assignment fla // have lower priority, and so they will not preempt the preemptor when // requeued. if borrowWithinCohort { - if !queueUnderNominalInResourcesNeedingPreemption(resPerFlv, cq) { + if !queueUnderNominalInResourcesNeedingPreemption(frsNeedPreemption, cq) { // It can only preempt workloads from another CQ if they are strictly under allowBorrowingBelowPriority. candidates = candidatesFromCQOrUnderThreshold(candidates, wl.ClusterQueue, *thresholdPrio) } - return minimalPreemptions(log, wlReq, cq, snapshot, resPerFlv, candidates, true, thresholdPrio) + return minimalPreemptions(log, requests, cq, snapshot, frsNeedPreemption, candidates, true, thresholdPrio) } // Only try preemptions in the cohort, without borrowing, if the target clusterqueue is still // under nominal quota for all resources. - if queueUnderNominalInResourcesNeedingPreemption(resPerFlv, cq) { - if targets := minimalPreemptions(log, wlReq, cq, snapshot, resPerFlv, candidates, false, nil); len(targets) > 0 { + if queueUnderNominalInResourcesNeedingPreemption(frsNeedPreemption, cq) { + if targets := minimalPreemptions(log, requests, cq, snapshot, frsNeedPreemption, candidates, false, nil); len(targets) > 0 { return targets } } // Final attempt. This time only candidates from the same queue, but // with borrowing. - return minimalPreemptions(log, wlReq, cq, snapshot, resPerFlv, sameQueueCandidates, true, nil) + return minimalPreemptions(log, requests, cq, snapshot, frsNeedPreemption, sameQueueCandidates, true, nil) } // canBorrowWithinCohort returns whether the behavior is enabled for the ClusterQueue and the threshold priority to use. @@ -231,21 +231,20 @@ func (p *Preemptor) applyPreemptionWithSSA(ctx context.Context, w *kueue.Workloa // Once the Workload fits, the heuristic tries to add Workloads back, in the // reverse order in which they were removed, while the incoming Workload still // fits. -func minimalPreemptions(log logr.Logger, wlReq resources.FlavorResourceQuantities, cq *cache.ClusterQueueSnapshot, snapshot *cache.Snapshot, resPerFlv resourcesPerFlavor, candidates []*workload.Info, allowBorrowing bool, allowBorrowingBelowPriority *int32) []*Target { +func minimalPreemptions(log logr.Logger, requests resources.FlavorResourceQuantities, cq *cache.ClusterQueueSnapshot, snapshot *cache.Snapshot, frsNeedPreemption sets.Set[resources.FlavorResource], candidates []*workload.Info, allowBorrowing bool, allowBorrowingBelowPriority *int32) []*Target { if logV := log.V(5); logV.Enabled() { - logV.Info("Simulating preemption", "candidates", workload.References(candidates), "resourcesRequiringPreemption", resPerFlv, "allowBorrowing", allowBorrowing, "allowBorrowingBelowPriority", allowBorrowingBelowPriority) + logV.Info("Simulating preemption", "candidates", workload.References(candidates), "resourcesRequiringPreemption", frsNeedPreemption, "allowBorrowing", allowBorrowing, "allowBorrowingBelowPriority", allowBorrowingBelowPriority) } // Simulate removing all candidates from the ClusterQueue and cohort. var targets []*Target fits := false for _, candWl := range candidates { candCQ := snapshot.ClusterQueues[candWl.ClusterQueue] - sameCq := cq == candCQ reason := InClusterQueueReason - if !sameCq && !cqIsBorrowing(candCQ, resPerFlv) { - continue - } - if !sameCq { + if cq != candCQ { + if !cqIsBorrowing(candCQ, frsNeedPreemption) { + continue + } reason = InCohortReclamationReason if allowBorrowingBelowPriority != nil { if priority.Priority(candWl.Obj) >= *allowBorrowingBelowPriority { @@ -272,7 +271,7 @@ func minimalPreemptions(log logr.Logger, wlReq resources.FlavorResourceQuantitie WorkloadInfo: candWl, Reason: reason, }) - if workloadFits(wlReq, cq, allowBorrowing) { + if workloadFits(requests, cq, allowBorrowing) { fits = true break } @@ -281,16 +280,16 @@ func minimalPreemptions(log logr.Logger, wlReq resources.FlavorResourceQuantitie restoreSnapshot(snapshot, targets) return nil } - targets = fillBackWorkloads(targets, wlReq, cq, snapshot, allowBorrowing) + targets = fillBackWorkloads(targets, requests, cq, snapshot, allowBorrowing) restoreSnapshot(snapshot, targets) return targets } -func fillBackWorkloads(targets []*Target, wlReq resources.FlavorResourceQuantities, cq *cache.ClusterQueueSnapshot, snapshot *cache.Snapshot, allowBorrowing bool) []*Target { +func fillBackWorkloads(targets []*Target, requests resources.FlavorResourceQuantities, cq *cache.ClusterQueueSnapshot, snapshot *cache.Snapshot, allowBorrowing bool) []*Target { // In the reverse order, check if any of the workloads can be added back. for i := len(targets) - 2; i >= 0; i-- { snapshot.AddWorkload(targets[i].WorkloadInfo) - if workloadFits(wlReq, cq, allowBorrowing) { + if workloadFits(requests, cq, allowBorrowing) { // O(1) deletion: copy the last element into index i and reduce size. targets[i] = targets[len(targets)-1] targets = targets[:len(targets)-1] @@ -338,14 +337,13 @@ func parseStrategies(s []config.PreemptionStrategy) []fsStrategy { return strategies } -func (p *Preemptor) fairPreemptions(log logr.Logger, wl *workload.Info, assignment flavorassigner.Assignment, snapshot *cache.Snapshot, resPerFlv resourcesPerFlavor, candidates []*workload.Info, allowBorrowingBelowPriority *int32) []*Target { +func (p *Preemptor) fairPreemptions(log logr.Logger, wl workload.Info, requests resources.FlavorResourceQuantities, snapshot *cache.Snapshot, frsNeedPreemption sets.Set[resources.FlavorResource], candidates []*workload.Info, allowBorrowingBelowPriority *int32) []*Target { if logV := log.V(5); logV.Enabled() { - logV.Info("Simulating fair preemption", "candidates", workload.References(candidates), "resourcesRequiringPreemption", resPerFlv, "allowBorrowingBelowPriority", allowBorrowingBelowPriority) + logV.Info("Simulating fair preemption", "candidates", workload.References(candidates), "resourcesRequiringPreemption", frsNeedPreemption, "allowBorrowingBelowPriority", allowBorrowingBelowPriority) } cqHeap := cqHeapFromCandidates(candidates, false, snapshot) nominatedCQ := snapshot.ClusterQueues[wl.ClusterQueue] - wlReq := assignment.TotalRequestsFor(wl) - newNominatedShareValue, _ := nominatedCQ.DominantResourceShareWith(wlReq) + newNominatedShareValue, _ := nominatedCQ.DominantResourceShareWith(requests) var targets []*Target fits := false var retryCandidates []*workload.Info @@ -359,11 +357,11 @@ func (p *Preemptor) fairPreemptions(log logr.Logger, wl *workload.Info, assignme WorkloadInfo: candWl, Reason: InClusterQueueReason, }) - if workloadFits(wlReq, nominatedCQ, true) { + if workloadFits(requests, nominatedCQ, true) { fits = true break } - newNominatedShareValue, _ = nominatedCQ.DominantResourceShareWith(wlReq) + newNominatedShareValue, _ = nominatedCQ.DominantResourceShareWith(requests) candCQ.workloads = candCQ.workloads[1:] if len(candCQ.workloads) > 0 { candCQ.share, _ = candCQ.cq.DominantResourceShare() @@ -387,12 +385,12 @@ func (p *Preemptor) fairPreemptions(log logr.Logger, wl *workload.Info, assignme WorkloadInfo: candWl, Reason: reason, }) - if workloadFits(wlReq, nominatedCQ, true) { + if workloadFits(requests, nominatedCQ, true) { fits = true break } candCQ.workloads = candCQ.workloads[i+1:] - if len(candCQ.workloads) > 0 && cqIsBorrowing(candCQ.cq, resPerFlv) { + if len(candCQ.workloads) > 0 && cqIsBorrowing(candCQ.cq, frsNeedPreemption) { candCQ.share = newCandShareVal cqHeap.PushIfNotPresent(candCQ) } @@ -419,7 +417,7 @@ func (p *Preemptor) fairPreemptions(log logr.Logger, wl *workload.Info, assignme WorkloadInfo: candWl, Reason: InCohortFairSharingReason, }) - if workloadFits(wlReq, nominatedCQ, true) { + if workloadFits(requests, nominatedCQ, true) { fits = true } // No requeueing because there doesn't seem to be an scenario where @@ -431,7 +429,7 @@ func (p *Preemptor) fairPreemptions(log logr.Logger, wl *workload.Info, assignme restoreSnapshot(snapshot, targets) return nil } - targets = fillBackWorkloads(targets, wlReq, nominatedCQ, snapshot, true) + targets = fillBackWorkloads(targets, requests, nominatedCQ, snapshot, true) restoreSnapshot(snapshot, targets) return targets } @@ -469,17 +467,13 @@ func cqHeapFromCandidates(candidates []*workload.Info, firstOnly bool, snapshot return cqHeap } -type resourcesPerFlavor = sets.Set[resources.FlavorResource] - -func resourcesRequiringPreemption(assignment flavorassigner.Assignment) resourcesPerFlavor { +func flavorResourcesNeedPreemption(assignment flavorassigner.Assignment) sets.Set[resources.FlavorResource] { resPerFlavor := sets.New[resources.FlavorResource]() for _, ps := range assignment.PodSets { for res, flvAssignment := range ps.Flavors { - // assignments with NoFit mode wouldn't enter the preemption path. - if flvAssignment.Mode != flavorassigner.Preempt { - continue + if flvAssignment.Mode == flavorassigner.Preempt { + resPerFlavor.Insert(resources.FlavorResource{Flavor: flvAssignment.Name, Resource: res}) } - resPerFlavor.Insert(resources.FlavorResource{Flavor: flvAssignment.Name, Resource: res}) } } return resPerFlavor @@ -488,13 +482,13 @@ func resourcesRequiringPreemption(assignment flavorassigner.Assignment) resource // findCandidates obtains candidates for preemption within the ClusterQueue and // cohort that respect the preemption policy and are using a resource that the // preempting workload needs. -func findCandidates(wl *kueue.Workload, wo workload.Ordering, cq *cache.ClusterQueueSnapshot, resPerFlv resourcesPerFlavor) []*workload.Info { +func (p *Preemptor) findCandidates(wl *kueue.Workload, cq *cache.ClusterQueueSnapshot, frsNeedPreemption sets.Set[resources.FlavorResource]) []*workload.Info { var candidates []*workload.Info wlPriority := priority.Priority(wl) if cq.Preemption.WithinClusterQueue != kueue.PreemptionPolicyNever { considerSamePrio := (cq.Preemption.WithinClusterQueue == kueue.PreemptionPolicyLowerOrNewerEqualPriority) - preemptorTS := wo.GetQueueOrderTimestamp(wl) + preemptorTS := p.workloadOrdering.GetQueueOrderTimestamp(wl) for _, candidateWl := range cq.Workloads { candidatePriority := priority.Priority(candidateWl.Obj) @@ -502,11 +496,11 @@ func findCandidates(wl *kueue.Workload, wo workload.Ordering, cq *cache.ClusterQ continue } - if candidatePriority == wlPriority && !(considerSamePrio && preemptorTS.Before(wo.GetQueueOrderTimestamp(candidateWl.Obj))) { + if candidatePriority == wlPriority && !(considerSamePrio && preemptorTS.Before(p.workloadOrdering.GetQueueOrderTimestamp(candidateWl.Obj))) { continue } - if !workloadUsesResources(candidateWl, resPerFlv) { + if !workloadUsesResources(candidateWl, frsNeedPreemption) { continue } candidates = append(candidates, candidateWl) @@ -514,20 +508,17 @@ func findCandidates(wl *kueue.Workload, wo workload.Ordering, cq *cache.ClusterQ } if cq.Cohort != nil && cq.Preemption.ReclaimWithinCohort != kueue.PreemptionPolicyNever { + onlyLowerPriority := cq.Preemption.ReclaimWithinCohort != kueue.PreemptionPolicyAny for cohortCQ := range cq.Cohort.Members { - if cq == cohortCQ || !cqIsBorrowing(cohortCQ, resPerFlv) { + if cq == cohortCQ || !cqIsBorrowing(cohortCQ, frsNeedPreemption) { // Can't reclaim quota from itself or ClusterQueues that are not borrowing. continue } - onlyLowerPrio := true - if cq.Preemption.ReclaimWithinCohort == kueue.PreemptionPolicyAny { - onlyLowerPrio = false - } for _, candidateWl := range cohortCQ.Workloads { - if onlyLowerPrio && priority.Priority(candidateWl.Obj) >= priority.Priority(wl) { + if onlyLowerPriority && priority.Priority(candidateWl.Obj) >= priority.Priority(wl) { continue } - if !workloadUsesResources(candidateWl, resPerFlv) { + if !workloadUsesResources(candidateWl, frsNeedPreemption) { continue } candidates = append(candidates, candidateWl) @@ -537,11 +528,11 @@ func findCandidates(wl *kueue.Workload, wo workload.Ordering, cq *cache.ClusterQ return candidates } -func cqIsBorrowing(cq *cache.ClusterQueueSnapshot, resPerFlv resourcesPerFlavor) bool { +func cqIsBorrowing(cq *cache.ClusterQueueSnapshot, frsNeedPreemption sets.Set[resources.FlavorResource]) bool { if cq.Cohort == nil { return false } - for fr := range resPerFlv { + for fr := range frsNeedPreemption { if cq.Borrowing(fr) { return true } @@ -549,10 +540,10 @@ func cqIsBorrowing(cq *cache.ClusterQueueSnapshot, resPerFlv resourcesPerFlavor) return false } -func workloadUsesResources(wl *workload.Info, resPerFlv resourcesPerFlavor) bool { +func workloadUsesResources(wl *workload.Info, frsNeedPreemption sets.Set[resources.FlavorResource]) bool { for _, ps := range wl.TotalRequests { for res, flv := range ps.Flavors { - if resPerFlv.Has(resources.FlavorResource{Flavor: flv, Resource: res}) { + if frsNeedPreemption.Has(resources.FlavorResource{Flavor: flv, Resource: res}) { return true } } @@ -563,8 +554,8 @@ func workloadUsesResources(wl *workload.Info, resPerFlv resourcesPerFlavor) bool // workloadFits determines if the workload requests would fit given the // requestable resources and simulated usage of the ClusterQueue and its cohort, // if it belongs to one. -func workloadFits(wlReq resources.FlavorResourceQuantities, cq *cache.ClusterQueueSnapshot, allowBorrowing bool) bool { - for fr, v := range wlReq { +func workloadFits(requests resources.FlavorResourceQuantities, cq *cache.ClusterQueueSnapshot, allowBorrowing bool) bool { + for fr, v := range requests { if !allowBorrowing && cq.BorrowingWith(fr, v) { return false } @@ -575,8 +566,8 @@ func workloadFits(wlReq resources.FlavorResourceQuantities, cq *cache.ClusterQue return true } -func queueUnderNominalInResourcesNeedingPreemption(resPerFlv resourcesPerFlavor, cq *cache.ClusterQueueSnapshot) bool { - for fr := range resPerFlv { +func queueUnderNominalInResourcesNeedingPreemption(frsNeedPreemption sets.Set[resources.FlavorResource], cq *cache.ClusterQueueSnapshot) bool { + for fr := range frsNeedPreemption { if cq.Usage[fr] >= cq.QuotaFor(fr).Nominal { return false } From f62571d80d6c2caaa69c3c32e6cca44027e6d541 Mon Sep 17 00:00:00 2001 From: gabesaba <15304068+gabesaba@users.noreply.github.com> Date: Fri, 9 Aug 2024 14:23:45 +0200 Subject: [PATCH 5/6] [Partial Admission] Check Mode before attempting Preemption (#2809) --- pkg/scheduler/scheduler.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 1c256c9cb69..4483c841a62 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -473,10 +473,11 @@ func (s *Scheduler) getAssignments(log logr.Logger, wl *workload.Info, snap *cac assignment := flvAssigner.Assign(log, nextCounts) if assignment.RepresentativeMode() == flavorassigner.Fit { return &partialAssignment{assignment: assignment}, true - } - preemptionTargets := s.preemptor.GetTargets(log, *wl, assignment, snap) - if len(preemptionTargets) > 0 { - return &partialAssignment{assignment: assignment, preemptionTargets: preemptionTargets}, true + } else if assignment.RepresentativeMode() == flavorassigner.Preempt { + preemptionTargets := s.preemptor.GetTargets(log, *wl, assignment, snap) + if len(preemptionTargets) > 0 { + return &partialAssignment{assignment: assignment, preemptionTargets: preemptionTargets}, true + } } return nil, false }) From 8f78e36d6f3c0e7d2a8a0cc705ee3f0b137e79ae Mon Sep 17 00:00:00 2001 From: gabesaba <15304068+gabesaba@users.noreply.github.com> Date: Tue, 13 Aug 2024 14:01:03 +0200 Subject: [PATCH 6/6] Prefer Reclamation to Priority Based Preemption (#2811) --- .../flavorassigner/flavorassigner.go | 71 +++++-- .../flavorassigner/flavorassigner_test.go | 164 ++++++++++++++- pkg/scheduler/preemption/preemption.go | 11 +- pkg/scheduler/preemption/preemption_oracle.go | 35 ++++ pkg/scheduler/scheduler.go | 4 +- pkg/scheduler/scheduler_test.go | 189 ++++++++++++++++++ 6 files changed, 449 insertions(+), 25 deletions(-) create mode 100644 pkg/scheduler/preemption/preemption_oracle.go diff --git a/pkg/scheduler/flavorassigner/flavorassigner.go b/pkg/scheduler/flavorassigner/flavorassigner.go index 1d7204c35b4..0c69ec615ca 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner.go +++ b/pkg/scheduler/flavorassigner/flavorassigner.go @@ -232,6 +232,31 @@ func (m FlavorAssignmentMode) String() string { return "Unknown" } +// granularMode is the FlavorAssignmentMode internal to +// FlavorAssigner, which lets us distinguish priority based preemption, +// and reclamation within Cohort. +type granularMode int + +const ( + noFit granularMode = iota + preempt + reclaim + fit +) + +func (mode granularMode) flavorAssignmentMode() FlavorAssignmentMode { + if mode == fit { + return Fit + } else if mode.isPreemptMode() { + return Preempt + } + return NoFit +} + +func (mode granularMode) isPreemptMode() bool { + return mode == preempt || mode == reclaim +} + type FlavorAssignment struct { Name kueue.ResourceFlavorReference Mode FlavorAssignmentMode @@ -239,19 +264,25 @@ type FlavorAssignment struct { borrow bool } +type preemptionOracle interface { + IsReclaimPossible(log logr.Logger, cq *cache.ClusterQueueSnapshot, wl workload.Info, fr resources.FlavorResource, quantity int64) bool +} + type FlavorAssigner struct { wl *workload.Info cq *cache.ClusterQueueSnapshot resourceFlavors map[kueue.ResourceFlavorReference]*kueue.ResourceFlavor enableFairSharing bool + oracle preemptionOracle } -func New(wl *workload.Info, cq *cache.ClusterQueueSnapshot, resourceFlavors map[kueue.ResourceFlavorReference]*kueue.ResourceFlavor, enableFairSharing bool) *FlavorAssigner { +func New(wl *workload.Info, cq *cache.ClusterQueueSnapshot, resourceFlavors map[kueue.ResourceFlavorReference]*kueue.ResourceFlavor, enableFairSharing bool, oracle preemptionOracle) *FlavorAssigner { return &FlavorAssigner{ wl: wl, cq: cq, resourceFlavors: resourceFlavors, enableFairSharing: enableFairSharing, + oracle: oracle, } } @@ -391,7 +422,7 @@ func (a *FlavorAssigner) findFlavorForPodSetResource( podSpec := &a.wl.Obj.Spec.PodSets[psID].Template.Spec var bestAssignment ResourceAssignment - bestAssignmentMode := NoFit + bestAssignmentMode := noFit // We will only check against the flavors' labels for the resource. selector := flavorSelector(podSpec, resourceGroup.LabelKeys) @@ -424,12 +455,12 @@ func (a *FlavorAssigner) findFlavorForPodSetResource( needsBorrowing := false assignments := make(ResourceAssignment, len(requests)) // Calculate representativeMode for this assignment as the worst mode among all requests. - representativeMode := Fit + representativeMode := fit for rName, val := range requests { resQuota := a.cq.QuotaFor(resources.FlavorResource{Flavor: fName, Resource: rName}) // Check considering the flavor usage by previous pod sets. fr := resources.FlavorResource{Flavor: fName, Resource: rName} - mode, borrow, s := a.fitsResourceQuota(fr, val+assignmentUsage[fr], resQuota) + mode, borrow, s := a.fitsResourceQuota(log, fr, val+assignmentUsage[fr], resQuota) if s != nil { status.reasons = append(status.reasons, s.reasons...) } @@ -437,14 +468,14 @@ func (a *FlavorAssigner) findFlavorForPodSetResource( representativeMode = mode } needsBorrowing = needsBorrowing || borrow - if representativeMode == NoFit { + if representativeMode == noFit { // The flavor doesn't fit, no need to check other resources. break } assignments[rName] = &FlavorAssignment{ Name: fName, - Mode: mode, + Mode: mode.flavorAssignmentMode(), borrow: borrow, } } @@ -462,7 +493,7 @@ func (a *FlavorAssigner) findFlavorForPodSetResource( } else if representativeMode > bestAssignmentMode { bestAssignment = assignments bestAssignmentMode = representativeMode - if bestAssignmentMode == Fit { + if bestAssignmentMode == fit { // All the resources fit in the cohort, no need to check more flavors. return bestAssignment, nil } @@ -478,27 +509,27 @@ func (a *FlavorAssigner) findFlavorForPodSetResource( assignment.TriedFlavorIdx = attemptedFlavorIdx } } - if bestAssignmentMode == Fit { + if bestAssignmentMode == fit { return bestAssignment, nil } } return bestAssignment, status } -func shouldTryNextFlavor(representativeMode FlavorAssignmentMode, flavorFungibility kueue.FlavorFungibility, needsBorrowing bool) bool { +func shouldTryNextFlavor(representativeMode granularMode, flavorFungibility kueue.FlavorFungibility, needsBorrowing bool) bool { policyPreempt := flavorFungibility.WhenCanPreempt policyBorrow := flavorFungibility.WhenCanBorrow - if representativeMode == Preempt && policyPreempt == kueue.Preempt { + if representativeMode.isPreemptMode() && policyPreempt == kueue.Preempt { if !needsBorrowing || policyBorrow == kueue.Borrow { return false } } - if representativeMode == Fit && needsBorrowing && policyBorrow == kueue.Borrow { + if representativeMode == fit && needsBorrowing && policyBorrow == kueue.Borrow { return false } - if representativeMode == Fit && !needsBorrowing { + if representativeMode == fit && !needsBorrowing { return false } @@ -557,16 +588,16 @@ func flavorSelector(spec *corev1.PodSpec, allowedKeys sets.Set[string]) nodeaffi // if borrowing is required when preempting. // If the flavor doesn't satisfy limits immediately (when waiting or preemption // could help), it returns a Status with reasons. -func (a *FlavorAssigner) fitsResourceQuota(fr resources.FlavorResource, val int64, rQuota *cache.ResourceQuota) (FlavorAssignmentMode, bool, *Status) { +func (a *FlavorAssigner) fitsResourceQuota(log logr.Logger, fr resources.FlavorResource, val int64, rQuota *cache.ResourceQuota) (granularMode, bool, *Status) { var status Status var borrow bool used := a.cq.Usage[fr] - mode := NoFit + mode := noFit if val <= rQuota.Nominal { // The request can be satisfied by the nominal quota, assuming quota is // reclaimed from the cohort or assuming all active workloads in the // ClusterQueue are preempted. - mode = Preempt + mode = preempt } cohortAvailable := rQuota.Nominal if a.cq.Cohort != nil { @@ -577,7 +608,7 @@ func (a *FlavorAssigner) fitsResourceQuota(fr resources.FlavorResource, val int6 // when preemption with borrowing is enabled, we can succeed to admit the // workload if preemption is used. if (rQuota.BorrowingLimit == nil || val <= rQuota.Nominal+*rQuota.BorrowingLimit) && val <= cohortAvailable { - mode = Preempt + mode = preempt borrow = val > rQuota.Nominal } } @@ -586,6 +617,10 @@ func (a *FlavorAssigner) fitsResourceQuota(fr resources.FlavorResource, val int6 return mode, borrow, &status } + if a.oracle.IsReclaimPossible(log, a.cq, *a.wl, fr, val) { + mode = reclaim + } + cohortUsed := used if a.cq.Cohort != nil { cohortUsed = a.cq.UsedCohortQuota(fr) @@ -593,13 +628,13 @@ func (a *FlavorAssigner) fitsResourceQuota(fr resources.FlavorResource, val int6 lack := cohortUsed + val - cohortAvailable if lack <= 0 { - return Fit, used+val > rQuota.Nominal, nil + return fit, used+val > rQuota.Nominal, nil } lackQuantity := resources.ResourceQuantity(fr.Resource, lack) msg := fmt.Sprintf("insufficient unused quota in cohort for %s in flavor %s, %s more needed", fr.Resource, fr.Flavor, &lackQuantity) if a.cq.Cohort == nil { - if mode == NoFit { + if mode == noFit { msg = fmt.Sprintf("insufficient quota for %s in flavor %s in ClusterQueue", fr.Resource, fr.Flavor) } else { msg = fmt.Sprintf("insufficient unused quota for %s in flavor %s, %s more needed", fr.Resource, fr.Flavor, &lackQuantity) diff --git a/pkg/scheduler/flavorassigner/flavorassigner_test.go b/pkg/scheduler/flavorassigner/flavorassigner_test.go index 105f364265b..96c604800c6 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner_test.go +++ b/pkg/scheduler/flavorassigner/flavorassigner_test.go @@ -21,6 +21,7 @@ import ( "fmt" "testing" + "github.com/go-logr/logr" "github.com/go-logr/logr/testr" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -41,6 +42,12 @@ type cohortResources struct { usage resources.FlavorResourceQuantities } +type testOracle struct{} + +func (f *testOracle) IsReclaimPossible(log logr.Logger, cq *cache.ClusterQueueSnapshot, wl workload.Info, fr resources.FlavorResource, quantity int64) bool { + return cq.QuotaFor(fr).Nominal >= quantity+cq.Usage[fr] +} + func TestAssignFlavors(t *testing.T) { resourceFlavors := map[kueue.ResourceFlavorReference]*kueue.ResourceFlavor{ "default": { @@ -1956,7 +1963,7 @@ func TestAssignFlavors(t *testing.T) { } clusterQueue.Usage = tc.clusterQueueUsage - flvAssigner := New(wlInfo, clusterQueue, resourceFlavors, tc.enableFairSharing) + flvAssigner := New(wlInfo, clusterQueue, resourceFlavors, tc.enableFairSharing, &testOracle{}) assignment := flvAssigner.Assign(log, nil) if repMode := assignment.RepresentativeMode(); repMode != tc.wantRepMode { t.Errorf("e.assignFlavors(_).RepresentativeMode()=%s, want %s", repMode, tc.wantRepMode) @@ -1969,6 +1976,159 @@ func TestAssignFlavors(t *testing.T) { } } +// We have 3 flavors: uno, due, tre. Each has 10 compute and 10 gpu. +// These FlavorResources are provided by test-clusterqueue, and made +// available to its Cohort. +func TestReclaimBeforePriorityPreemption(t *testing.T) { + type rfMap = map[corev1.ResourceName]kueue.ResourceFlavorReference + cases := map[string]struct { + workloadRequests *utiltesting.PodSetWrapper + testClusterQueueUsage resources.FlavorResourceQuantities + otherClusterQueueUsage resources.FlavorResourceQuantities + flavorFungibility *kueue.FlavorFungibility + wantMode FlavorAssignmentMode + wantAssigment rfMap + }{ + "Select first flavor which fits": { + workloadRequests: utiltesting.MakePodSet("main", 1).Request("gpu", "10"), + testClusterQueueUsage: resources.FlavorResourceQuantities{ + {Flavor: "uno", Resource: "gpu"}: 1, + }, + otherClusterQueueUsage: resources.FlavorResourceQuantities{ + {Flavor: "due", Resource: "gpu"}: 1, + }, + wantMode: Fit, + wantAssigment: rfMap{"gpu": "tre"}, + }, + "Select first flavor where gpu reclamation is possible": { + workloadRequests: utiltesting.MakePodSet("main", 1).Request("gpu", "10"), + testClusterQueueUsage: resources.FlavorResourceQuantities{ + {Flavor: "uno", Resource: "gpu"}: 1, + }, + otherClusterQueueUsage: resources.FlavorResourceQuantities{ + {Flavor: "due", Resource: "gpu"}: 1, + {Flavor: "tre", Resource: "gpu"}: 1, + }, + wantMode: Preempt, + wantAssigment: rfMap{"gpu": "due"}, + }, + "Select first flavor when flavor fungibility is disabled": { + workloadRequests: utiltesting.MakePodSet("main", 1).Request("gpu", "10"), + testClusterQueueUsage: resources.FlavorResourceQuantities{ + {Flavor: "uno", Resource: "gpu"}: 1, + }, + otherClusterQueueUsage: resources.FlavorResourceQuantities{ + {Flavor: "due", Resource: "gpu"}: 1, + {Flavor: "tre", Resource: "gpu"}: 1, + }, + flavorFungibility: &kueue.FlavorFungibility{ + WhenCanPreempt: kueue.Preempt, + }, + wantMode: Preempt, + wantAssigment: rfMap{"gpu": "uno"}, + }, + "Select first flavor where priority based preemption is possible": { + workloadRequests: utiltesting.MakePodSet("main", 1).Request("gpu", "10"), + testClusterQueueUsage: resources.FlavorResourceQuantities{ + {Flavor: "uno", Resource: "gpu"}: 1, + {Flavor: "due", Resource: "gpu"}: 1, + {Flavor: "tre", Resource: "gpu"}: 1, + }, + wantMode: Preempt, + wantAssigment: rfMap{"gpu": "uno"}, + }, + "Select second flavor where gpu reclamation is possible, as compute Fits": { + workloadRequests: utiltesting.MakePodSet("main", 1).Request("gpu", "10").Request("compute", "10"), + testClusterQueueUsage: resources.FlavorResourceQuantities{ + {Flavor: "uno", Resource: "gpu"}: 1, + {Flavor: "uno", Resource: "compute"}: 1, + {Flavor: "due", Resource: "compute"}: 1, + }, + otherClusterQueueUsage: resources.FlavorResourceQuantities{ + {Flavor: "due", Resource: "gpu"}: 1, + {Flavor: "tre", Resource: "gpu"}: 1, + }, + wantMode: Preempt, + wantAssigment: rfMap{"gpu": "tre", "compute": "tre"}, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + resourceFlavors := map[kueue.ResourceFlavorReference]*kueue.ResourceFlavor{ + "uno": utiltesting.MakeResourceFlavor("uno").Obj(), + "due": utiltesting.MakeResourceFlavor("due").Obj(), + "tre": utiltesting.MakeResourceFlavor("tre").Obj(), + } + testCq := utiltesting.MakeClusterQueue("test-clusterqueue"). + Cohort("cohort"). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, + }). + FlavorFungibility(kueue.FlavorFungibility{ + WhenCanPreempt: kueue.TryNextFlavor, + }). + ResourceGroup( + utiltesting.MakeFlavorQuotas("uno").Resource("compute", "10").Resource("gpu", "10").FlavorQuotas, + utiltesting.MakeFlavorQuotas("due").Resource("compute", "10").Resource("gpu", "10").FlavorQuotas, + utiltesting.MakeFlavorQuotas("tre").Resource("compute", "10").Resource("gpu", "10").FlavorQuotas, + ).ClusterQueue + otherCq := utiltesting.MakeClusterQueue("other-clusterqueue"). + Cohort("cohort"). + ResourceGroup( + utiltesting.MakeFlavorQuotas("uno").Resource("compute", "0").Resource("gpu", "0").FlavorQuotas, + utiltesting.MakeFlavorQuotas("due").Resource("compute", "0").Resource("gpu", "0").FlavorQuotas, + utiltesting.MakeFlavorQuotas("tre").Resource("compute", "0").Resource("gpu", "0").FlavorQuotas, + ).ClusterQueue + + wlInfo := workload.NewInfo(&kueue.Workload{ + Spec: kueue.WorkloadSpec{ + PodSets: []kueue.PodSet{ + tc.workloadRequests.PodSet, + }, + }, + }) + + if tc.flavorFungibility != nil { + testCq.Spec.FlavorFungibility = tc.flavorFungibility + } + + cache := cache.New(utiltesting.NewFakeClient()) + if err := cache.AddClusterQueue(context.Background(), &testCq); err != nil { + t.Fatalf("Failed to add CQ to cache") + } + if err := cache.AddClusterQueue(context.Background(), &otherCq); err != nil { + t.Fatalf("Failed to add CQ to cache") + } + for _, rf := range resourceFlavors { + cache.AddOrUpdateResourceFlavor(rf) + } + + snapshot := cache.Snapshot() + otherClusterQueue := snapshot.ClusterQueues["other-clusterqueue"] + otherClusterQueue.AddUsage(tc.otherClusterQueueUsage) + + testClusterQueue := snapshot.ClusterQueues["test-clusterqueue"] + testClusterQueue.AddUsage(tc.testClusterQueueUsage) + + flvAssigner := New(wlInfo, testClusterQueue, resourceFlavors, false, &testOracle{}) + log := testr.NewWithOptions(t, testr.Options{Verbosity: 2}) + assignment := flvAssigner.Assign(log, nil) + if gotRepMode := assignment.RepresentativeMode(); gotRepMode != tc.wantMode { + t.Errorf("Unexpected RepresentativeMode. got %s, want %s", gotRepMode, tc.wantMode) + } + if len(assignment.PodSets[0].Flavors) != len(tc.wantAssigment) { + t.Errorf("Wrong number of flavors. got %d, want %d", len(assignment.PodSets[0].Flavors), len(tc.wantAssigment)) + } + for resourceName, wantFlavor := range tc.wantAssigment { + if gotFlavor := assignment.PodSets[0].Flavors[resourceName].Name; gotFlavor != wantFlavor { + t.Errorf("Unexpected flavor. got %s, want %s", gotFlavor, wantFlavor) + } + } + }) + } +} + // Tests the case where the Cache's flavors and CQs flavors // fall out of sync, so that the CQ has flavors which no-longer exist. func TestDeletedFlavors(t *testing.T) { @@ -2077,7 +2237,7 @@ func TestDeletedFlavors(t *testing.T) { cache.DeleteResourceFlavor(flavorMap["deleted-flavor"]) delete(flavorMap, "deleted-flavor") - flvAssigner := New(wlInfo, clusterQueue, flavorMap, false) + flvAssigner := New(wlInfo, clusterQueue, flavorMap, false, &testOracle{}) assignment := flvAssigner.Assign(log, nil) if repMode := assignment.RepresentativeMode(); repMode != tc.wantRepMode { diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go index 193d163305b..2fbbf2bfde2 100644 --- a/pkg/scheduler/preemption/preemption.go +++ b/pkg/scheduler/preemption/preemption.go @@ -101,11 +101,17 @@ type Target struct { Reason string } -// GetTargets returns the list of workloads that should be evicted in order to make room for wl. +// GetTargets returns the list of workloads that should be evicted in +// order to make room for wl. func (p *Preemptor) GetTargets(log logr.Logger, wl workload.Info, assignment flavorassigner.Assignment, snapshot *cache.Snapshot) []*Target { frsNeedPreemption := flavorResourcesNeedPreemption(assignment) - cq := snapshot.ClusterQueues[wl.ClusterQueue] + requests := assignment.TotalRequestsFor(&wl) + return p.getTargets(log, wl, requests, frsNeedPreemption, snapshot) +} +func (p *Preemptor) getTargets(log logr.Logger, wl workload.Info, requests resources.FlavorResourceQuantities, + frsNeedPreemption sets.Set[resources.FlavorResource], snapshot *cache.Snapshot) []*Target { + cq := snapshot.ClusterQueues[wl.ClusterQueue] candidates := p.findCandidates(wl.Obj, cq, frsNeedPreemption) if len(candidates) == 0 { return nil @@ -113,7 +119,6 @@ func (p *Preemptor) GetTargets(log logr.Logger, wl workload.Info, assignment fla sort.Slice(candidates, candidatesOrdering(candidates, cq.Name, time.Now())) sameQueueCandidates := candidatesOnlyFromQueue(candidates, wl.ClusterQueue) - requests := assignment.TotalRequestsFor(&wl) // To avoid flapping, Kueue only allows preemption of workloads from the same // queue if borrowing. Preemption of workloads from queues can happen only diff --git a/pkg/scheduler/preemption/preemption_oracle.go b/pkg/scheduler/preemption/preemption_oracle.go new file mode 100644 index 00000000000..e1efc20493f --- /dev/null +++ b/pkg/scheduler/preemption/preemption_oracle.go @@ -0,0 +1,35 @@ +package preemption + +import ( + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/sets" + + "sigs.k8s.io/kueue/pkg/cache" + "sigs.k8s.io/kueue/pkg/resources" + "sigs.k8s.io/kueue/pkg/workload" +) + +func NewOracle(preemptor *Preemptor, snapshot *cache.Snapshot) *PreemptionOracle { + return &PreemptionOracle{preemptor, snapshot} +} + +type PreemptionOracle struct { + preemptor *Preemptor + snapshot *cache.Snapshot +} + +// IsReclaimPossible determines if a ClusterQueue can fit this +// FlavorResource by reclaiming its nominal quota which it lent to its +// Cohort. +func (p *PreemptionOracle) IsReclaimPossible(log logr.Logger, cq *cache.ClusterQueueSnapshot, wl workload.Info, fr resources.FlavorResource, quantity int64) bool { + if cq.Usage[fr]+quantity > cq.QuotaFor(fr).Nominal { + return false + } + + for _, candidate := range p.preemptor.getTargets(log, wl, resources.FlavorResourceQuantities{fr: quantity}, sets.New(fr), p.snapshot) { + if candidate.WorkloadInfo.ClusterQueue == cq.Name { + return false + } + } + return true +} diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 4483c841a62..55f4aac0687 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -268,7 +268,7 @@ func (s *Scheduler) schedule(ctx context.Context) wait.SpeedSignal { continue } if mode == flavorassigner.Preempt && cycleCohortsSkipPreemption.Has(cq.Cohort.Name) { - setSkipped(e, "Workload skipped because its premption calculations were invalidated by another workload") + setSkipped(e, "Workload skipped because its preemption calculations were invalidated by another workload") continue } } @@ -450,7 +450,7 @@ type partialAssignment struct { func (s *Scheduler) getAssignments(log logr.Logger, wl *workload.Info, snap *cache.Snapshot) (flavorassigner.Assignment, []*preemption.Target) { cq := snap.ClusterQueues[wl.ClusterQueue] - flvAssigner := flavorassigner.New(wl, cq, snap.ResourceFlavors, s.fairSharing.Enable) + flvAssigner := flavorassigner.New(wl, cq, snap.ResourceFlavors, s.fairSharing.Enable, preemption.NewOracle(s.preemptor, snap)) fullAssignment := flvAssigner.Assign(log, nil) var faPreemtionTargets []*preemption.Target diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index fccedddabc2..dc161d597f2 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -2313,6 +2313,195 @@ func TestSchedule(t *testing.T) { "eng-beta/b1": *utiltesting.MakeAdmission("other-beta").Assignment(corev1.ResourceCPU, "default", "2").Obj(), }, }, + "prefer reclamation over cq priority based preemption": { + // Flavor 1, on-demand, requires preemption of workload in CQ. + // Flavor 2, spot, requires preemption of workload in Cohort which + // is borrowing from CQ. + // Flavor 2 is a better assignment, so we preempt in it. + additionalClusterQueues: []kueue.ClusterQueue{ + *utiltesting.MakeClusterQueue("other-alpha"). + Cohort("other"). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, + }). + ResourceGroup( + utiltesting.MakeFlavorQuotas("on-demand").Resource("gpu", "10").FlavorQuotas, + utiltesting.MakeFlavorQuotas("spot").Resource("gpu", "10").FlavorQuotas, + ). + Obj(), + *utiltesting.MakeClusterQueue("other-beta"). + Cohort("other"). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, + }). + ResourceGroup( + utiltesting.MakeFlavorQuotas("on-demand").Resource("gpu", "0").FlavorQuotas, + utiltesting.MakeFlavorQuotas("spot").Resource("gpu", "0").FlavorQuotas, + ). + Obj(), + }, + additionalLocalQueues: []kueue.LocalQueue{ + *utiltesting.MakeLocalQueue("other", "eng-alpha").ClusterQueue("other-alpha").Obj(), + *utiltesting.MakeLocalQueue("other", "eng-beta").ClusterQueue("other-beta").Obj(), + }, + workloads: []kueue.Workload{ + *utiltesting.MakeWorkload("a1", "eng-alpha"). + Priority(50). + Queue("other"). + Request("gpu", "5"). + SimpleReserveQuota("other-alpha", "on-demand", now). + Obj(), + *utiltesting.MakeWorkload("b1", "eng-beta"). + Priority(50). + Queue("other"). + Request("gpu", "5"). + SimpleReserveQuota("other-beta", "spot", now). + Obj(), + *utiltesting.MakeWorkload("preemptor", "eng-alpha"). + Priority(100). + Queue("other"). + Request("gpu", "6"). + Obj(), + }, + wantPreempted: sets.New("eng-beta/b1"), + wantLeft: map[string][]string{ + "other-alpha": {"eng-alpha/preemptor"}, + }, + wantAssignments: map[string]kueue.Admission{ + "eng-alpha/a1": *utiltesting.MakeAdmission("other-alpha").Assignment("gpu", "on-demand", "5").Obj(), + "eng-beta/b1": *utiltesting.MakeAdmission("other-beta").Assignment("gpu", "spot", "5").Obj(), + }, + }, + "prefer first preemption flavor when second flavor requires both reclaim and cq priority preemption": { + // Flavor 1, on-demand, requires preemption of workload in CQ. + // Flavor 2, spot, requires preemption of workload in Cohort and CQ + // Since Flavor 2 doesn't improve the assignment, we prefer Flavor 1. + additionalClusterQueues: []kueue.ClusterQueue{ + *utiltesting.MakeClusterQueue("other-alpha"). + Cohort("other"). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, + }). + ResourceGroup( + utiltesting.MakeFlavorQuotas("on-demand").Resource("gpu", "10").FlavorQuotas, + utiltesting.MakeFlavorQuotas("spot").Resource("gpu", "10").FlavorQuotas, + ). + Obj(), + *utiltesting.MakeClusterQueue("other-beta"). + Cohort("other"). + ResourceGroup( + utiltesting.MakeFlavorQuotas("on-demand").Resource("gpu", "0").FlavorQuotas, + utiltesting.MakeFlavorQuotas("spot").Resource("gpu", "0").FlavorQuotas, + ). + Obj(), + }, + additionalLocalQueues: []kueue.LocalQueue{ + *utiltesting.MakeLocalQueue("other", "eng-alpha").ClusterQueue("other-alpha").Obj(), + *utiltesting.MakeLocalQueue("other", "eng-beta").ClusterQueue("other-beta").Obj(), + }, + workloads: []kueue.Workload{ + *utiltesting.MakeWorkload("a1", "eng-alpha"). + Priority(50). + Queue("other"). + Request("gpu", "5"). + SimpleReserveQuota("other-alpha", "on-demand", now). + Obj(), + *utiltesting.MakeWorkload("a2", "eng-alpha"). + Priority(50). + Queue("other"). + Request("gpu", "5"). + SimpleReserveQuota("other-alpha", "spot", now). + Obj(), + *utiltesting.MakeWorkload("b1", "eng-beta"). + Priority(50). + Queue("other"). + Request("gpu", "5"). + SimpleReserveQuota("other-beta", "spot", now). + Obj(), + *utiltesting.MakeWorkload("preemptor", "eng-alpha"). + Priority(100). + Queue("other"). + Request("gpu", "6"). + Obj(), + }, + wantPreempted: sets.New("eng-alpha/a1"), + wantLeft: map[string][]string{ + "other-alpha": {"eng-alpha/preemptor"}, + }, + wantAssignments: map[string]kueue.Admission{ + "eng-alpha/a1": *utiltesting.MakeAdmission("other-alpha").Assignment("gpu", "on-demand", "5").Obj(), + "eng-alpha/a2": *utiltesting.MakeAdmission("other-alpha").Assignment("gpu", "spot", "5").Obj(), + "eng-beta/b1": *utiltesting.MakeAdmission("other-beta").Assignment("gpu", "spot", "5").Obj(), + }, + }, + "prefer first preemption flavor when second flavor also requires cq preemption": { + // Flavor 1, on-demand, requires preemption of workload in CQ + // Flavor 2, spot, also requires preemption of workload in CQ, + // since the borrowing workload in Cohort is too high priority. + // Therefore, we choose Flavor 1. + additionalClusterQueues: []kueue.ClusterQueue{ + *utiltesting.MakeClusterQueue("other-alpha"). + Cohort("other"). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, + }). + ResourceGroup( + utiltesting.MakeFlavorQuotas("on-demand").Resource("gpu", "10").FlavorQuotas, + utiltesting.MakeFlavorQuotas("spot").Resource("gpu", "10").FlavorQuotas, + ). + Obj(), + *utiltesting.MakeClusterQueue("other-beta"). + Cohort("other"). + ResourceGroup( + utiltesting.MakeFlavorQuotas("on-demand").Resource("gpu", "0").FlavorQuotas, + utiltesting.MakeFlavorQuotas("spot").Resource("gpu", "0").FlavorQuotas, + ). + Obj(), + }, + additionalLocalQueues: []kueue.LocalQueue{ + *utiltesting.MakeLocalQueue("other", "eng-alpha").ClusterQueue("other-alpha").Obj(), + *utiltesting.MakeLocalQueue("other", "eng-beta").ClusterQueue("other-beta").Obj(), + }, + workloads: []kueue.Workload{ + *utiltesting.MakeWorkload("a1", "eng-alpha"). + Priority(50). + Queue("other"). + Request("gpu", "6"). + SimpleReserveQuota("other-alpha", "on-demand", now). + Obj(), + *utiltesting.MakeWorkload("a2", "eng-alpha"). + Priority(50). + Queue("other"). + Request("gpu", "5"). + SimpleReserveQuota("other-alpha", "spot", now). + Obj(), + *utiltesting.MakeWorkload("b1", "eng-beta"). + // b1 is too high priority for preemptor. + Priority(9001). + Queue("other"). + Request("gpu", "5"). + SimpleReserveQuota("other-beta", "spot", now). + Obj(), + *utiltesting.MakeWorkload("preemptor", "eng-alpha"). + Priority(100). + Queue("other"). + Request("gpu", "5"). + Obj(), + }, + wantPreempted: sets.New("eng-alpha/a1"), + wantLeft: map[string][]string{ + "other-alpha": {"eng-alpha/preemptor"}, + }, + wantAssignments: map[string]kueue.Admission{ + "eng-alpha/a1": *utiltesting.MakeAdmission("other-alpha").Assignment("gpu", "on-demand", "6").Obj(), + "eng-alpha/a2": *utiltesting.MakeAdmission("other-alpha").Assignment("gpu", "spot", "5").Obj(), + "eng-beta/b1": *utiltesting.MakeAdmission("other-beta").Assignment("gpu", "spot", "5").Obj(), + }, + }, } for name, tc := range cases {