diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index a2ac90f87f3..460b2a3a40a 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -619,14 +619,14 @@ func getUsage(frq resources.FlavorResourceQuantities, cq *clusterQueue, cohort * usage := make([]kueue.FlavorUsage, 0, len(frq)) for _, rg := range cq.ResourceGroups { for _, fName := range rg.Flavors { - flvUsage := frq[fName] outFlvUsage := kueue.FlavorUsage{ Name: fName, Resources: make([]kueue.ResourceUsage, 0, len(rg.CoveredResources)), } for rName := range rg.CoveredResources { - rQuota := cq.QuotaFor(resources.FlavorResource{Flavor: fName, Resource: rName}) - used := flvUsage[rName] + fr := resources.FlavorResource{Flavor: fName, Resource: rName} + rQuota := cq.QuotaFor(fr) + used := frq[fr] rUsage := kueue.ResourceUsage{ Name: rName, Total: resources.ResourceQuantity(rName, used), @@ -682,15 +682,15 @@ func filterLocalQueueUsage(orig resources.FlavorResourceQuantities, resourceGrou qFlvUsages := make([]kueue.LocalQueueFlavorUsage, 0, len(orig)) for _, rg := range resourceGroups { for _, fName := range rg.Flavors { - flvUsage := orig[fName] outFlvUsage := kueue.LocalQueueFlavorUsage{ Name: fName, Resources: make([]kueue.LocalQueueResourceUsage, 0, len(rg.CoveredResources)), } for rName := range rg.CoveredResources { + fr := resources.FlavorResource{Flavor: fName, Resource: rName} outFlvUsage.Resources = append(outFlvUsage.Resources, kueue.LocalQueueResourceUsage{ Name: rName, - Total: resources.ResourceQuantity(rName, flvUsage[rName]), + Total: resources.ResourceQuantity(rName, orig[fr]), }) } // The resourceUsages should be in a stable order to avoid endless creation of update events. diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index 17a9582e499..68468c15bde 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -110,12 +110,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -125,12 +125,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -160,12 +160,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: pending, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -257,12 +257,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, FlavorFungibility: defaultFlavorFungibility, NamespaceSelector: labels.Nothing(), - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -272,12 +272,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, FlavorFungibility: defaultFlavorFungibility, NamespaceSelector: labels.Nothing(), - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -307,12 +307,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: pending, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -379,12 +379,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 2, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -422,17 +422,17 @@ func TestCacheClusterQueueOperations(t *testing.T) { "e": { Name: "e", AllocatableResourceGeneration: 2, - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: "cpu"}: 1_000, - }.Unflatten(), + }, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -479,12 +479,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -504,12 +504,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: pending, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -551,12 +551,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -566,12 +566,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -601,12 +601,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, NamespaceSelector: labels.Nothing(), FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "nonexistent-flavor", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Status: active, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -661,22 +661,22 @@ func TestCacheClusterQueueOperations(t *testing.T) { NamespaceSelector: labels.Everything(), AllocatableResourceGeneration: 1, FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "foo", Resource: "cpu"}: 0, {Flavor: "foo", Resource: "memory"}: 0, {Flavor: "bar", Resource: "cpu"}: 0, {Flavor: "bar", Resource: "memory"}: 0, {Flavor: "theta", Resource: "example.com/gpu"}: 0, {Flavor: "gamma", Resource: "example.com/gpu"}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "foo", Resource: "cpu"}: 0, {Flavor: "foo", Resource: "memory"}: 0, {Flavor: "bar", Resource: "cpu"}: 0, {Flavor: "bar", Resource: "memory"}: 0, {Flavor: "theta", Resource: "example.com/gpu"}: 0, {Flavor: "gamma", Resource: "example.com/gpu"}: 0, - }.Unflatten(), + }, Status: pending, Preemption: defaultPreemption, FairWeight: oneQuantity, @@ -852,12 +852,12 @@ func TestCacheClusterQueueOperations(t *testing.T) { Preemption: defaultPreemption, AllocatableResourceGeneration: 1, FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 2000, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 1000, - }.Unflatten(), + }, FairWeight: oneQuantity, Workloads: map[string]*workload.Info{ "ns/reserving": { @@ -952,27 +952,27 @@ func TestCacheClusterQueueOperations(t *testing.T) { AllocatableResourceGeneration: 1, FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 2_000, {Flavor: "on-demand", Resource: corev1.ResourceMemory}: 2 * utiltesting.Gi, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceMemory}: 0, {Flavor: "license", Resource: "license"}: 4, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ + }, + Usage: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "on-demand", Resource: corev1.ResourceMemory}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceMemory}: 0, {Flavor: "license", Resource: "license"}: 0, - }.Unflatten(), - AdmittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + AdmittedUsage: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "on-demand", Resource: corev1.ResourceMemory}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceMemory}: 0, {Flavor: "license", Resource: "license"}: 0, - }.Unflatten(), + }, }, }, }, @@ -1099,17 +1099,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c", "/d"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1128,17 +1128,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1156,17 +1156,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1185,17 +1185,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/a", "/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, }, }, @@ -1214,17 +1214,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1243,17 +1243,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1271,17 +1271,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c", "/d"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1296,17 +1296,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1319,17 +1319,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1343,17 +1343,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1369,17 +1369,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1394,17 +1394,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1431,17 +1431,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b", "/d"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 20, {Flavor: "spot", Resource: corev1.ResourceCPU}: 30, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c", "/e"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, }, wantAssumedWorkloads: map[string]string{ @@ -1464,17 +1464,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, wantAssumedWorkloads: map[string]string{}, @@ -1504,17 +1504,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c", "/e"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, }, wantAssumedWorkloads: map[string]string{ @@ -1536,17 +1536,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 0, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, }, @@ -1578,17 +1578,17 @@ func TestCacheWorkloadOperations(t *testing.T) { wantResults: map[string]result{ "one": { Workloads: sets.New("/a", "/b", "/d"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 20, {Flavor: "spot", Resource: corev1.ResourceCPU}: 30, - }.Unflatten(), + }, }, "two": { Workloads: sets.New("/c", "/e"), - UsedResources: resources.FlavorResourceQuantitiesFlat{ + UsedResources: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10, {Flavor: "spot", Resource: corev1.ResourceCPU}: 15, - }.Unflatten(), + }, }, }, wantAssumedWorkloads: map[string]string{ @@ -2243,46 +2243,46 @@ func TestCacheQueueOperations(t *testing.T) { key: "ns1/alpha", reservingWorkloads: 1, admittedWorkloads: 1, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("2")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("8Gi")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), - admittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + admittedUsage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("2")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("8Gi")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), + }, }, "ns2/beta": { key: "ns2/beta", reservingWorkloads: 2, admittedWorkloads: 1, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("7")), - }.Unflatten(), - admittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + admittedUsage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("2")), - }.Unflatten(), + }, }, "ns1/gamma": { key: "ns1/gamma", reservingWorkloads: 1, admittedWorkloads: 0, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "ondemand", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("5")), {Flavor: "ondemand", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("16Gi")), {Flavor: "model-b", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), - admittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + admittedUsage: resources.FlavorResourceQuantities{ {Flavor: "ondemand", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "ondemand", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-b", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), + }, }, } cacheLocalQueuesAfterInsertingCqAndQ := map[string]*queue{ @@ -2290,46 +2290,46 @@ func TestCacheQueueOperations(t *testing.T) { key: "ns1/alpha", reservingWorkloads: 0, admittedWorkloads: 0, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), - admittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + admittedUsage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), + }, }, "ns2/beta": { key: "ns2/beta", reservingWorkloads: 0, admittedWorkloads: 0, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), - admittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + admittedUsage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "spot", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-a", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), + }, }, "ns1/gamma": { key: "ns1/gamma", reservingWorkloads: 0, admittedWorkloads: 0, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "ondemand", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "ondemand", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-b", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), - admittedUsage: resources.FlavorResourceQuantitiesFlat{ + }, + admittedUsage: resources.FlavorResourceQuantities{ {Flavor: "ondemand", Resource: corev1.ResourceCPU}: resources.ResourceValue(corev1.ResourceCPU, resource.MustParse("0")), {Flavor: "ondemand", Resource: corev1.ResourceMemory}: resources.ResourceValue(corev1.ResourceMemory, resource.MustParse("0")), {Flavor: "model-b", Resource: "example.com/gpu"}: resources.ResourceValue("example.com/gpu", resource.MustParse("0")), - }.Unflatten(), + }, }, } cases := map[string]struct { diff --git a/pkg/cache/clusterqueue.go b/pkg/cache/clusterqueue.go index f2fc3b3e4ae..b9ab3ad74a1 100644 --- a/pkg/cache/clusterqueue.go +++ b/pkg/cache/clusterqueue.go @@ -134,9 +134,9 @@ func (c *cohort) CalculateLendable() map[corev1.ResourceName]int64 { return lendable } -func (c *ClusterQueueSnapshot) FitInCohort(q resources.FlavorResourceQuantitiesFlat) bool { +func (c *ClusterQueueSnapshot) FitInCohort(q resources.FlavorResourceQuantities) bool { for fr, value := range q { - available := c.RequestableCohortQuota(fr.Flavor, fr.Resource) - c.UsedCohortQuota(fr.Flavor, fr.Resource) + available := c.RequestableCohortQuota(fr) - c.UsedCohortQuota(fr) if available < value { return false } @@ -196,34 +196,31 @@ func (c *clusterQueue) update(in *kueue.ClusterQueue, resourceFlavors map[kueue. } if features.Enabled(features.LendingLimit) { - var guaranteedQuota resources.FlavorResourceQuantities + c.GuaranteedQuota = make(resources.FlavorResourceQuantities) for _, rg := range c.ResourceGroups { for _, fName := range rg.Flavors { for rName := range rg.CoveredResources { fr := resources.FlavorResource{Flavor: fName, Resource: rName} rQuota := c.QuotaFor(fr) if rQuota.LendingLimit != nil { - guaranteedQuota.Add(fr, rQuota.Nominal-*rQuota.LendingLimit) + c.GuaranteedQuota[fr] += rQuota.Nominal - *rQuota.LendingLimit } } } } - c.GuaranteedQuota = guaranteedQuota } return nil } func filterFlavorQuantities(orig resources.FlavorResourceQuantities, resourceGroups []kueue.ResourceGroup) resources.FlavorResourceQuantities { - ret := make(resources.FlavorResourceQuantities) + ret := make(resources.FlavorResourceQuantities, len(orig)) for _, rg := range resourceGroups { for _, f := range rg.Flavors { - existingUsedResources := orig[f.Name] - usedResources := make(map[corev1.ResourceName]int64, len(f.Resources)) for _, r := range f.Resources { - usedResources[r.Name] = existingUsedResources[r.Name] + fr := resources.FlavorResource{Flavor: f.Name, Resource: r.Name} + ret[fr] = orig[fr] } - ret[f.Name] = usedResources } } return ret @@ -458,23 +455,23 @@ func (c *clusterQueue) updateWorkloadUsage(wi *workload.Info, m int64) { } } -func updateFlavorUsage(newUsage resources.FlavorResourceQuantitiesFlat, oldUsage resources.FlavorResourceQuantities, m int64) { +func updateFlavorUsage(newUsage resources.FlavorResourceQuantities, oldUsage resources.FlavorResourceQuantities, m int64) { for fr, q := range newUsage { - oldUsage.Add(fr, q*m) + oldUsage[fr] += q * m } } -func updateCohortUsage(newUsage resources.FlavorResourceQuantitiesFlat, cq *ClusterQueueSnapshot, m int64) { +func updateCohortUsage(newUsage resources.FlavorResourceQuantities, cq *ClusterQueueSnapshot, m int64) { for fr, v := range newUsage { - after := cq.Usage.For(fr) - cq.guaranteedQuota(fr.Flavor, fr.Resource) + after := cq.Usage[fr] - cq.guaranteedQuota(fr) // rollback update cq.Usage before := after - v*m if before > 0 { - cq.Cohort.Usage.Add(fr, -before) + cq.Cohort.Usage[fr] -= before } // simulate updating cq.Usage if after > 0 { - cq.Cohort.Usage.Add(fr, after) + cq.Cohort.Usage[fr] += after } } } @@ -533,14 +530,9 @@ func (q *queue) resetFlavorsAndResources(cqUsage resources.FlavorResourceQuantit } func resetUsage(lqUsage resources.FlavorResourceQuantities, cqUsage resources.FlavorResourceQuantities) resources.FlavorResourceQuantities { - usedFlavorResources := make(resources.FlavorResourceQuantities) - for cqFlv, cqRes := range cqUsage { - existingUsedResources := lqUsage[cqFlv] - usedResources := make(map[corev1.ResourceName]int64, len(cqRes)) - for rName := range cqRes { - usedResources[rName] = existingUsedResources[rName] - } - usedFlavorResources[cqFlv] = usedResources + usedFlavorResources := make(resources.FlavorResourceQuantities, len(cqUsage)) + for fr := range cqUsage { + usedFlavorResources[fr] = lqUsage[fr] } return usedFlavorResources } @@ -553,48 +545,48 @@ func workloadBelongsToLocalQueue(wl *kueue.Workload, q *kueue.LocalQueue) bool { // LendingLimit will also be counted here if feature LendingLimit enabled. // Please note that for different clusterQueues, the requestable quota is different, // they should be calculated dynamically. -func (c *ClusterQueueSnapshot) RequestableCohortQuota(fName kueue.ResourceFlavorReference, rName corev1.ResourceName) (val int64) { - if c.Cohort.RequestableResources == nil || c.Cohort.RequestableResources[fName] == nil { +func (c *ClusterQueueSnapshot) RequestableCohortQuota(fr resources.FlavorResource) (val int64) { + if _, ok := c.Cohort.RequestableResources[fr]; !ok { return 0 } - requestableCohortQuota := c.Cohort.RequestableResources[fName][rName] + requestableCohortQuota := c.Cohort.RequestableResources[fr] // When feature LendingLimit enabled, cohort.requestableResource accumulated the lendingLimit if not null // rather than the flavor's quota, then the total available quota should include its own guaranteed resources. - requestableCohortQuota += c.guaranteedQuota(fName, rName) + requestableCohortQuota += c.guaranteedQuota(fr) return requestableCohortQuota } -func (c *ClusterQueueSnapshot) guaranteedQuota(fName kueue.ResourceFlavorReference, rName corev1.ResourceName) (val int64) { +func (c *ClusterQueueSnapshot) guaranteedQuota(fr resources.FlavorResource) (val int64) { if !features.Enabled(features.LendingLimit) { return 0 } - if c.GuaranteedQuota == nil || c.GuaranteedQuota[fName] == nil { + if _, ok := c.GuaranteedQuota[fr]; !ok { return 0 } - return c.GuaranteedQuota[fName][rName] + return c.GuaranteedQuota[fr] } // UsedCohortQuota returns the used quota by the flavor and resource name in the cohort. // Note that when LendingLimit enabled, the usage is not equal to the total used quota but the one // minus the guaranteed resources, this is only for judging whether workloads fit in the cohort. -func (c *ClusterQueueSnapshot) UsedCohortQuota(fName kueue.ResourceFlavorReference, rName corev1.ResourceName) (val int64) { - if c.Cohort.Usage == nil || c.Cohort.Usage[fName] == nil { +func (c *ClusterQueueSnapshot) UsedCohortQuota(fr resources.FlavorResource) (val int64) { + if _, ok := c.Cohort.Usage[fr]; !ok { return 0 } - cohortUsage := c.Cohort.Usage[fName][rName] + cohortUsage := c.Cohort.Usage[fr] // When feature LendingLimit enabled, cohortUsage is the sum of usage in LendingLimit. // If cqUsage < c.guaranteedQuota, it means the cq is not using all its guaranteedQuota, // need to count the cqUsage in, otherwise need to count the guaranteedQuota in. if features.Enabled(features.LendingLimit) { - cqUsage := c.Usage[fName][rName] - if cqUsage < c.guaranteedQuota(fName, rName) { + cqUsage := c.Usage[fr] + if cqUsage < c.guaranteedQuota(fr) { cohortUsage += cqUsage } else { - cohortUsage += c.guaranteedQuota(fName, rName) + cohortUsage += c.guaranteedQuota(fr) } } @@ -617,7 +609,7 @@ func (c *clusterQueue) lendableResourcesInCohort() map[corev1.ResourceName]int64 } func (c *clusterQueue) usageFor(fr resources.FlavorResource) int64 { - return c.Usage.For(fr) + return c.Usage[fr] } func (c *clusterQueue) QuotaFor(fr resources.FlavorResource) *ResourceQuota { @@ -638,11 +630,11 @@ func (c *ClusterQueueSnapshot) DominantResourceShare() (int, corev1.ResourceName return dominantResourceShare(c, nil, 0) } -func (c *ClusterQueueSnapshot) DominantResourceShareWith(wlReq resources.FlavorResourceQuantitiesFlat) (int, corev1.ResourceName) { +func (c *ClusterQueueSnapshot) DominantResourceShareWith(wlReq resources.FlavorResourceQuantities) (int, corev1.ResourceName) { return dominantResourceShare(c, wlReq, 1) } -func (c *ClusterQueueSnapshot) DominantResourceShareWithout(wlReq resources.FlavorResourceQuantitiesFlat) (int, corev1.ResourceName) { +func (c *ClusterQueueSnapshot) DominantResourceShareWithout(wlReq resources.FlavorResourceQuantities) (int, corev1.ResourceName) { return dominantResourceShare(c, wlReq, -1) } @@ -654,7 +646,7 @@ type dominantResourceShareNode interface { netQuotaNode } -func dominantResourceShare(node dominantResourceShareNode, wlReq resources.FlavorResourceQuantitiesFlat, m int64) (int, corev1.ResourceName) { +func dominantResourceShare(node dominantResourceShareNode, wlReq resources.FlavorResourceQuantities, m int64) (int, corev1.ResourceName) { if !node.hasCohort() { return 0, "" } diff --git a/pkg/cache/clusterqueue_snapshot.go b/pkg/cache/clusterqueue_snapshot.go index 7080962beed..d039435974c 100644 --- a/pkg/cache/clusterqueue_snapshot.go +++ b/pkg/cache/clusterqueue_snapshot.go @@ -48,11 +48,11 @@ func (c *ClusterQueueSnapshot) RGByResource(resource corev1.ResourceName) *Resou return nil } -func (c *ClusterQueueSnapshot) AddUsage(frq resources.FlavorResourceQuantitiesFlat) { +func (c *ClusterQueueSnapshot) AddUsage(frq resources.FlavorResourceQuantities) { c.addOrRemoveUsage(frq, 1) } -func (c *ClusterQueueSnapshot) Fits(frq resources.FlavorResourceQuantitiesFlat) bool { +func (c *ClusterQueueSnapshot) Fits(frq resources.FlavorResourceQuantities) bool { for fr, q := range frq { if c.Available(fr) < q { return false @@ -70,18 +70,18 @@ func (c *ClusterQueueSnapshot) Borrowing(fr resources.FlavorResource) bool { } func (c *ClusterQueueSnapshot) BorrowingWith(fr resources.FlavorResource, val int64) bool { - return c.usageFor(fr)+val > c.nominal(fr) + return c.Usage[fr]+val > c.nominal(fr) } func (c *ClusterQueueSnapshot) Available(fr resources.FlavorResource) int64 { if c.Cohort == nil { - return max(0, c.nominal(fr)-c.usageFor(fr)) + return max(0, c.nominal(fr)-c.Usage[fr]) } - capacityAvailable := c.RequestableCohortQuota(fr.Flavor, fr.Resource) - c.UsedCohortQuota(fr.Flavor, fr.Resource) + capacityAvailable := c.RequestableCohortQuota(fr) - c.UsedCohortQuota(fr) // if the borrowing limit exists, we cap our available capacity by the borrowing limit. if borrowingLimit := c.borrowingLimit(fr); borrowingLimit != nil { - withBorrowingRemaining := c.nominal(fr) + *borrowingLimit - c.usageFor(fr) + withBorrowingRemaining := c.nominal(fr) + *borrowingLimit - c.Usage[fr] capacityAvailable = min(capacityAvailable, withBorrowingRemaining) } return max(0, capacityAvailable) @@ -115,7 +115,7 @@ func (c *ClusterQueueSnapshot) lendableResourcesInCohort() map[corev1.ResourceNa } func (c *ClusterQueueSnapshot) usageFor(fr resources.FlavorResource) int64 { - return c.Usage.For(fr) + return c.Usage[fr] } func (c *ClusterQueueSnapshot) resourceGroups() []ResourceGroup { diff --git a/pkg/cache/clusterqueue_test.go b/pkg/cache/clusterqueue_test.go index f86168b1505..873653ecec6 100644 --- a/pkg/cache/clusterqueue_test.go +++ b/pkg/cache/clusterqueue_test.go @@ -96,16 +96,16 @@ func TestClusterQueueUpdateWithFlavors(t *testing.T) { func TestFitInCohort(t *testing.T) { cases := map[string]struct { - request resources.FlavorResourceQuantitiesFlat + request resources.FlavorResourceQuantities wantFit bool - usage resources.FlavorResourceQuantitiesFlat + usage resources.FlavorResourceQuantities clusterQueue []*kueue.ClusterQueue enableLendingLimit bool }{ "full cohort, empty request": { - request: resources.FlavorResourceQuantitiesFlat{}, + request: resources.FlavorResourceQuantities{}, wantFit: true, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 5_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, {Flavor: "f2", Resource: corev1.ResourceCPU}: 5_000, @@ -129,12 +129,12 @@ func TestFitInCohort(t *testing.T) { }, }, "can fit": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "f2", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f2", Resource: corev1.ResourceMemory}: 1, }, wantFit: true, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 5_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, {Flavor: "f2", Resource: corev1.ResourceCPU}: 4_000, @@ -158,14 +158,14 @@ func TestFitInCohort(t *testing.T) { }, }, "full cohort, none fit": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 1, {Flavor: "f2", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f2", Resource: corev1.ResourceMemory}: 1, }, wantFit: false, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 5_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, {Flavor: "f2", Resource: corev1.ResourceCPU}: 5_000, @@ -189,14 +189,14 @@ func TestFitInCohort(t *testing.T) { }, }, "one cannot fit": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 1, {Flavor: "f2", Resource: corev1.ResourceCPU}: 2_000, {Flavor: "f2", Resource: corev1.ResourceMemory}: 1, }, wantFit: false, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 4_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 4, {Flavor: "f2", Resource: corev1.ResourceCPU}: 4_000, @@ -220,12 +220,12 @@ func TestFitInCohort(t *testing.T) { }, }, "missing flavor": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "non-existent-flavor", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "non-existent-flavor", Resource: corev1.ResourceMemory}: 1, }, wantFit: false, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 5_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 5, }, @@ -243,12 +243,12 @@ func TestFitInCohort(t *testing.T) { }, }, "missing resource": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "f1", Resource: corev1.ResourceMemory}: 1, }, wantFit: false, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 3_000, }, clusterQueue: []*kueue.ClusterQueue{ @@ -264,11 +264,11 @@ func TestFitInCohort(t *testing.T) { }, }, "lendingLimit enabled can't fit": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 3_000, }, wantFit: false, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 2_000, }, clusterQueue: []*kueue.ClusterQueue{ @@ -299,11 +299,11 @@ func TestFitInCohort(t *testing.T) { enableLendingLimit: true, }, "lendingLimit enabled can fit": { - request: resources.FlavorResourceQuantitiesFlat{ + request: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 3_000, }, wantFit: true, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "f1", Resource: corev1.ResourceCPU}: 1_000, }, clusterQueue: []*kueue.ClusterQueue{ @@ -761,15 +761,15 @@ func TestClusterQueueUpdateWithAdmissionCheck(t *testing.T) { func TestDominantResourceShare(t *testing.T) { cases := map[string]struct { - usage resources.FlavorResourceQuantitiesFlat + usage resources.FlavorResourceQuantities clusterQueue *kueue.ClusterQueue lendingClusterQueue *kueue.ClusterQueue - flvResQ resources.FlavorResourceQuantitiesFlat + flvResQ resources.FlavorResourceQuantities wantDRValue int wantDRName corev1.ResourceName }{ "no cohort": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "default", Resource: "example.com/gpu"}: 2, }, @@ -782,7 +782,7 @@ func TestDominantResourceShare(t *testing.T) { ).Obj(), }, "usage below nominal": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "default", Resource: "example.com/gpu"}: 2, }, @@ -806,7 +806,7 @@ func TestDominantResourceShare(t *testing.T) { ).Obj(), }, "usage above nominal": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "default", Resource: "example.com/gpu"}: 7, }, @@ -832,7 +832,7 @@ func TestDominantResourceShare(t *testing.T) { wantDRValue: 200, // (7-5)*1000/10 }, "one resource above nominal": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "default", Resource: "example.com/gpu"}: 3, }, @@ -858,7 +858,7 @@ func TestDominantResourceShare(t *testing.T) { wantDRValue: 100, // (3-2)*1000/10 }, "usage with workload above nominal": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "default", Resource: "example.com/gpu"}: 2, }, @@ -880,7 +880,7 @@ func TestDominantResourceShare(t *testing.T) { ResourceQuotaWrapper("example.com/gpu").NominalQuota("5").Append(). FlavorQuotas, ).Obj(), - flvResQ: resources.FlavorResourceQuantitiesFlat{ + flvResQ: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, {Flavor: "default", Resource: "example.com/gpu"}: 4, }, @@ -888,7 +888,7 @@ func TestDominantResourceShare(t *testing.T) { wantDRValue: 300, // (1+4-2)*1000/10 }, "A resource with zero lendable": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "default", Resource: "example.com/gpu"}: 1, }, @@ -910,7 +910,7 @@ func TestDominantResourceShare(t *testing.T) { ResourceQuotaWrapper("example.com/gpu").NominalQuota("64").LendingLimit("0").Append(). FlavorQuotas, ).Obj(), - flvResQ: resources.FlavorResourceQuantitiesFlat{ + flvResQ: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, {Flavor: "default", Resource: "example.com/gpu"}: 4, }, @@ -918,7 +918,7 @@ func TestDominantResourceShare(t *testing.T) { wantDRValue: 300, // (1+4-2)*1000/10 }, "multiple flavors": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 15_000, {Flavor: "spot", Resource: corev1.ResourceCPU}: 5_000, }, @@ -941,14 +941,14 @@ func TestDominantResourceShare(t *testing.T) { ResourceQuotaWrapper("cpu").NominalQuota("100").Append(). FlavorQuotas, ).Obj(), - flvResQ: resources.FlavorResourceQuantitiesFlat{ + flvResQ: resources.FlavorResourceQuantities{ {Flavor: "on-demand", Resource: corev1.ResourceCPU}: 10_000, }, wantDRName: corev1.ResourceCPU, wantDRValue: 25, // ((15+10-20)+0)*1000/200 (spot under nominal) }, "above nominal with integer weight": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: "example.com/gpu"}: 7, }, clusterQueue: utiltesting.MakeClusterQueue("cq"). @@ -971,7 +971,7 @@ func TestDominantResourceShare(t *testing.T) { wantDRValue: 100, // ((7-5)*1000/10)/2 }, "above nominal with decimal weight": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: "example.com/gpu"}: 7, }, clusterQueue: utiltesting.MakeClusterQueue("cq"). @@ -994,7 +994,7 @@ func TestDominantResourceShare(t *testing.T) { wantDRValue: 400, // ((7-5)*1000/10)/(1/2) }, "above nominal with zero weight": { - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: "example.com/gpu"}: 7, }, clusterQueue: utiltesting.MakeClusterQueue("cq"). diff --git a/pkg/cache/resource.go b/pkg/cache/resource.go index ca940a02660..d081f7c24ca 100644 --- a/pkg/cache/resource.go +++ b/pkg/cache/resource.go @@ -8,13 +8,16 @@ type resourceGroupNode interface { resourceGroups() []ResourceGroup } -func flavorResources(r resourceGroupNode) []resources.FlavorResource { - flavorResourceCount := 0 - for _, rg := range r.resourceGroups() { - flavorResourceCount += len(rg.Flavors) * len(rg.CoveredResources) +func flavorResourceCount(rgs []ResourceGroup) int { + count := 0 + for _, rg := range rgs { + count += len(rg.Flavors) * len(rg.CoveredResources) } + return count +} - frs := make([]resources.FlavorResource, 0, flavorResourceCount) +func flavorResources(r resourceGroupNode) []resources.FlavorResource { + frs := make([]resources.FlavorResource, 0, flavorResourceCount(r.resourceGroups())) for _, rg := range r.resourceGroups() { for _, f := range rg.Flavors { for r := range rg.CoveredResources { @@ -34,8 +37,8 @@ type netQuotaNode interface { // remainingQuota computes the remaining quota for each FlavorResource. A // negative value implies that the node is borrowing. -func remainingQuota(node netQuotaNode) resources.FlavorResourceQuantitiesFlat { - remainingQuota := make(resources.FlavorResourceQuantitiesFlat) +func remainingQuota(node netQuotaNode) resources.FlavorResourceQuantities { + remainingQuota := make(resources.FlavorResourceQuantities) for _, fr := range flavorResources(node) { remainingQuota[fr] += node.QuotaFor(fr).Nominal - node.usageFor(fr) } diff --git a/pkg/cache/snapshot.go b/pkg/cache/snapshot.go index 6c4d4c87318..884f7efca4c 100644 --- a/pkg/cache/snapshot.go +++ b/pkg/cache/snapshot.go @@ -52,7 +52,7 @@ func (s *Snapshot) AddWorkload(wl *workload.Info) { cq.addOrRemoveUsage(wl.FlavorResourceUsage(), 1) } -func (c *ClusterQueueSnapshot) addOrRemoveUsage(usage resources.FlavorResourceQuantitiesFlat, m int64) { +func (c *ClusterQueueSnapshot) addOrRemoveUsage(usage resources.FlavorResourceQuantities, m int64) { updateFlavorUsage(usage, c.Usage, m) if c.Cohort != nil { if features.Enabled(features.LendingLimit) { @@ -124,7 +124,7 @@ func (c *clusterQueue) snapshot() *ClusterQueueSnapshot { FlavorFungibility: c.FlavorFungibility, FairWeight: c.FairWeight, AllocatableResourceGeneration: c.AllocatableResourceGeneration, - Usage: make(resources.FlavorResourceQuantities, len(c.Usage)), + Usage: maps.Clone(c.Usage), Workloads: maps.Clone(c.Workloads), Preemption: c.Preemption, NamespaceSelector: c.NamespaceSelector, @@ -133,9 +133,6 @@ func (c *clusterQueue) snapshot() *ClusterQueueSnapshot { Quotas: c.quotas, } - for fName, rUsage := range c.Usage { - cc.Usage[fName] = maps.Clone(rUsage) - } if features.Enabled(features.LendingLimit) { cc.GuaranteedQuota = c.GuaranteedQuota } @@ -145,9 +142,11 @@ func (c *clusterQueue) snapshot() *ClusterQueueSnapshot { func (c *cohort) snapshotInto(cqs map[string]*ClusterQueueSnapshot) { cohortSnap := &CohortSnapshot{ - Name: c.Name, - Members: make(sets.Set[*ClusterQueueSnapshot], c.Members.Len()), - Lendable: c.CalculateLendable(), + Name: c.Name, + Members: make(sets.Set[*ClusterQueueSnapshot], c.Members.Len()), + Lendable: c.CalculateLendable(), + Usage: make(resources.FlavorResourceQuantities), + RequestableResources: make(resources.FlavorResourceQuantities), } cohortSnap.AllocatableResourceGeneration = 0 for cq := range c.Members { @@ -162,9 +161,6 @@ func (c *cohort) snapshotInto(cqs map[string]*ClusterQueueSnapshot) { } func (c *ClusterQueueSnapshot) accumulateResources(cohort *CohortSnapshot) { - if cohort.RequestableResources == nil { - cohort.RequestableResources = make(resources.FlavorResourceQuantities, len(c.ResourceGroups)) - } for _, rg := range c.ResourceGroups { for _, fName := range rg.Flavors { for rName := range rg.CoveredResources { @@ -175,23 +171,21 @@ func (c *ClusterQueueSnapshot) accumulateResources(cohort *CohortSnapshot) { // If LendingLimit is not nil, we should count the lendingLimit as the requestable // resource because we can't borrow more quota than lendingLimit. if features.Enabled(features.LendingLimit) && rQuota.LendingLimit != nil { - cohort.RequestableResources.Add(fr, *rQuota.LendingLimit) + cohort.RequestableResources[fr] += *rQuota.LendingLimit } else { - cohort.RequestableResources.Add(fr, rQuota.Nominal) + cohort.RequestableResources[fr] += rQuota.Nominal } } } } - for fName, resUsages := range c.Usage { - for res, val := range resUsages { - // Similar to cohort.RequestableResources, we accumulate the usage above the guaranteed resources, - // here we should remove the guaranteed quota as well for that part can not be borrowed. - val -= c.guaranteedQuota(fName, res) - // if val < 0, it means the cq is not using any quota belongs to LendingLimit - if val < 0 { - val = 0 - } - cohort.Usage.Add(resources.FlavorResource{Flavor: fName, Resource: res}, val) + for fr, val := range c.Usage { + // Similar to cohort.RequestableResources, we accumulate the usage above the guaranteed resources, + // here we should remove the guaranteed quota as well for that part can not be borrowed. + val -= c.guaranteedQuota(fr) + // if val < 0, it means the cq is not using any quota belongs to LendingLimit + if val < 0 { + val = 0 } + cohort.Usage[fr] += val } } diff --git a/pkg/cache/snapshot_test.go b/pkg/cache/snapshot_test.go index 4131607258c..cb26f812031 100644 --- a/pkg/cache/snapshot_test.go +++ b/pkg/cache/snapshot_test.go @@ -203,16 +203,16 @@ func TestSnapshot(t *testing.T) { cohort := &CohortSnapshot{ Name: "borrowing", AllocatableResourceGeneration: 2, - RequestableResources: resources.FlavorResourceQuantitiesFlat{ + RequestableResources: resources.FlavorResourceQuantities{ {Flavor: "demand", Resource: corev1.ResourceCPU}: 100_000, {Flavor: "spot", Resource: corev1.ResourceCPU}: 300_000, {Flavor: "default", Resource: "example.com/gpu"}: 50, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ + }, + Usage: resources.FlavorResourceQuantities{ {Flavor: "demand", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "spot", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "default", Resource: "example.com/gpu"}: 15, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 400_000, "example.com/gpu": 50, @@ -236,10 +236,10 @@ func TestSnapshot(t *testing.T) { {Flavor: "spot", Resource: corev1.ResourceCPU}: {Nominal: 200_000}, }, FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "demand", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "spot", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Workloads: map[string]*workload.Info{ "/alpha": workload.NewInfo(utiltesting.MakeWorkload("alpha", ""). PodSets(*utiltesting.MakePodSet("main", 5). @@ -275,10 +275,10 @@ func TestSnapshot(t *testing.T) { {Flavor: "default", Resource: "example.com/gpu"}: {Nominal: 50}, }, FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "spot", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "default", Resource: "example.com/gpu"}: 15, - }.Unflatten(), + }, Workloads: map[string]*workload.Info{ "/beta": workload.NewInfo(utiltesting.MakeWorkload("beta", ""). PodSets(*utiltesting.MakePodSet("main", 5). @@ -322,9 +322,9 @@ func TestSnapshot(t *testing.T) { {Flavor: "default", Resource: corev1.ResourceCPU}: {Nominal: 100_000}, }, FlavorFungibility: defaultFlavorFungibility, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Preemption: defaultPreemption, FairWeight: oneQuantity, NamespaceSelector: labels.Everything(), @@ -433,14 +433,14 @@ func TestSnapshot(t *testing.T) { cohort := &CohortSnapshot{ Name: "lending", AllocatableResourceGeneration: 2, - RequestableResources: resources.FlavorResourceQuantitiesFlat{ + RequestableResources: resources.FlavorResourceQuantities{ {Flavor: "arm", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "x86", Resource: corev1.ResourceCPU}: 20_000, - }.Unflatten(), - Usage: resources.FlavorResourceQuantitiesFlat{ + }, + Usage: resources.FlavorResourceQuantities{ {Flavor: "arm", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "x86", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 30_000, }, @@ -464,10 +464,10 @@ func TestSnapshot(t *testing.T) { }, FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "arm", Resource: corev1.ResourceCPU}: 15_000, {Flavor: "x86", Resource: corev1.ResourceCPU}: 10_000, - }.Unflatten(), + }, Workloads: map[string]*workload.Info{ "/alpha": workload.NewInfo(utiltesting.MakeWorkload("alpha", ""). PodSets(*utiltesting.MakePodSet("main", 5). @@ -494,10 +494,10 @@ func TestSnapshot(t *testing.T) { Preemption: defaultPreemption, NamespaceSelector: labels.Everything(), Status: active, - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "arm", Resource: corev1.ResourceCPU}: 5_000, {Flavor: "x86", Resource: corev1.ResourceCPU}: 10_000, - }.Unflatten(), + }, }, "b": { Name: "b", @@ -516,17 +516,17 @@ func TestSnapshot(t *testing.T) { }, FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "arm", Resource: corev1.ResourceCPU}: 0, {Flavor: "x86", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Preemption: defaultPreemption, NamespaceSelector: labels.Everything(), Status: active, - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "arm", Resource: corev1.ResourceCPU}: 5_000, {Flavor: "x86", Resource: corev1.ResourceCPU}: 10_000, - }.Unflatten(), + }, }, }, ResourceFlavors: map[kueue.ResourceFlavorReference]*kueue.ResourceFlavor{ @@ -658,11 +658,11 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { Name: "cohort", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, {Flavor: "alpha", Resource: corev1.ResourceMemory}: 0, {Flavor: "beta", Resource: corev1.ResourceMemory}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 12_000, corev1.ResourceMemory: 12 * utiltesting.Gi, @@ -678,11 +678,11 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, {Flavor: "alpha", Resource: corev1.ResourceMemory}: 0, {Flavor: "beta", Resource: corev1.ResourceMemory}: 0, - }.Unflatten(), + }, }, "c2": { Name: "c2", @@ -692,9 +692,9 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, }, }, } @@ -707,11 +707,11 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { Name: "cohort", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 2_000, {Flavor: "alpha", Resource: corev1.ResourceMemory}: utiltesting.Gi, {Flavor: "beta", Resource: corev1.ResourceMemory}: utiltesting.Gi, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 12_000, corev1.ResourceMemory: 12 * utiltesting.Gi, @@ -730,11 +730,11 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { ResourceGroups: cqCache.clusterQueues["c1"].ResourceGroups, FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, {Flavor: "alpha", Resource: corev1.ResourceMemory}: utiltesting.Gi, {Flavor: "beta", Resource: corev1.ResourceMemory}: utiltesting.Gi, - }.Unflatten(), + }, }, "c2": { Name: "c2", @@ -747,9 +747,9 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 2_000, - }.Unflatten(), + }, }, }, } @@ -762,11 +762,11 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { Name: "cohort", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "alpha", Resource: corev1.ResourceMemory}: 0, {Flavor: "beta", Resource: corev1.ResourceMemory}: utiltesting.Gi, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 12_000, corev1.ResourceMemory: 12 * utiltesting.Gi, @@ -785,11 +785,11 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { ResourceGroups: cqCache.clusterQueues["c1"].ResourceGroups, FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "alpha", Resource: corev1.ResourceMemory}: 0, {Flavor: "beta", Resource: corev1.ResourceMemory}: utiltesting.Gi, - }.Unflatten(), + }, }, "c2": { Name: "c2", @@ -802,9 +802,9 @@ func TestSnapshotAddRemoveWorkload(t *testing.T) { ResourceGroups: cqCache.clusterQueues["c2"].ResourceGroups, FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 2_000, - }.Unflatten(), + }, }, }, } @@ -914,9 +914,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -931,12 +931,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -946,12 +946,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } @@ -964,9 +964,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -981,12 +981,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 7_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -996,12 +996,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } @@ -1014,9 +1014,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -1031,12 +1031,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -1046,12 +1046,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } @@ -1064,9 +1064,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -1081,12 +1081,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -1096,12 +1096,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } @@ -1115,9 +1115,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -1132,12 +1132,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -1147,12 +1147,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } @@ -1166,9 +1166,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -1183,12 +1183,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -1198,12 +1198,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } @@ -1217,9 +1217,9 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { Name: "lend", AllocatableResourceGeneration: 2, RequestableResources: initialCohortResources, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, - }.Unflatten(), + }, Lendable: map[corev1.ResourceName]int64{ corev1.ResourceCPU: 10_000, }, @@ -1234,12 +1234,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 9_000, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 6_000, - }.Unflatten(), + }, }, "lend-b": { Name: "lend-b", @@ -1249,12 +1249,12 @@ func TestSnapshotAddRemoveWorkloadWithLendingLimit(t *testing.T) { FlavorFungibility: defaultFlavorFungibility, FairWeight: oneQuantity, AllocatableResourceGeneration: 1, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 0, - }.Unflatten(), - GuaranteedQuota: resources.FlavorResourceQuantitiesFlat{ + }, + GuaranteedQuota: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 4_000, - }.Unflatten(), + }, }, }, } diff --git a/pkg/resources/resource.go b/pkg/resources/resource.go index 6bddd55ac89..d95d06ffca6 100644 --- a/pkg/resources/resource.go +++ b/pkg/resources/resource.go @@ -27,33 +27,4 @@ type FlavorResource struct { Resource corev1.ResourceName } -type FlavorResourceQuantities map[kueue.ResourceFlavorReference]Requests -type FlavorResourceQuantitiesFlat map[FlavorResource]int64 - -func (f FlavorResourceQuantitiesFlat) Unflatten() FlavorResourceQuantities { - out := make(FlavorResourceQuantities) - for flavorResource, value := range f { - if _, ok := out[flavorResource.Flavor]; !ok { - out[flavorResource.Flavor] = make(Requests) - } - out[flavorResource.Flavor][flavorResource.Resource] = value - } - return out -} - -// For attempts to access nested value, returning 0 if absent. -func (f FlavorResourceQuantities) For(fr FlavorResource) int64 { - return f[fr.Flavor][fr.Resource] -} - -// Add adds the Quantity v for the FlavorResource fr, allocating -// as needed. -func (f *FlavorResourceQuantities) Add(fr FlavorResource, v int64) { - if *f == nil { - *f = make(FlavorResourceQuantities) - } - if (*f)[fr.Flavor] == nil { - (*f)[fr.Flavor] = make(Requests) - } - (*f)[fr.Flavor][fr.Resource] += v -} +type FlavorResourceQuantities map[FlavorResource]int64 diff --git a/pkg/resources/resource_test.go b/pkg/resources/resource_test.go deleted file mode 100644 index 57ed9fa2430..00000000000 --- a/pkg/resources/resource_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package resources - -import ( - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestSafeAdd(t *testing.T) { - var a FlavorResourceQuantities = nil - fr := FlavorResource{Flavor: "Hello", Resource: "World"} - - defer func() { - if r := recover(); r != nil { - t.Errorf("Panic while assigning to nil map") - } - }() - a.Add(fr, 5) - a.Add(fr, -7) - - expected := FlavorResourceQuantitiesFlat{fr: -2}.Unflatten() - if diff := cmp.Diff(a, expected); diff != "" { - t.Fatalf("Unexpected diff %s", diff) - } -} - -func TestFor(t *testing.T) { - var a FlavorResourceQuantities = nil - - defer func() { - if r := recover(); r != nil { - t.Errorf("Panic while accessing nil map") - } - }() - result := a.For(FlavorResource{Flavor: "Hello", Resource: "World"}) - - if result != 0 { - t.Fatalf("Unexpected result: %d != 0", result) - } -} diff --git a/pkg/scheduler/flavorassigner/flavorassigner.go b/pkg/scheduler/flavorassigner/flavorassigner.go index df26a0f070b..1d7204c35b4 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner.go +++ b/pkg/scheduler/flavorassigner/flavorassigner.go @@ -46,7 +46,7 @@ type Assignment struct { // Usage is the accumulated Usage of resources as pod sets get // flavors assigned. - Usage resources.FlavorResourceQuantitiesFlat + Usage resources.FlavorResourceQuantities // representativeMode is the cached representative mode for this assignment. representativeMode *FlavorAssignmentMode @@ -106,8 +106,8 @@ func (a *Assignment) ToAPI() []kueue.PodSetAssignment { return psFlavors } -func (a *Assignment) TotalRequestsFor(wl *workload.Info) resources.FlavorResourceQuantitiesFlat { - usage := make(resources.FlavorResourceQuantitiesFlat) +func (a *Assignment) TotalRequestsFor(wl *workload.Info) resources.FlavorResourceQuantities { + usage := make(resources.FlavorResourceQuantities) for i, ps := range wl.TotalRequests { for res, q := range ps.Requests { flv := a.PodSets[i].Flavors[res].Name @@ -296,7 +296,7 @@ func (a *FlavorAssigner) Assign(log logr.Logger, counts []int32) Assignment { func (a *FlavorAssigner) assignFlavors(log logr.Logger, requests []workload.PodSetResources) Assignment { assignment := Assignment{ PodSets: make([]PodSetAssignment, 0, len(requests)), - Usage: make(resources.FlavorResourceQuantitiesFlat), + Usage: make(resources.FlavorResourceQuantities), LastState: workload.AssignmentClusterQueueState{ LastTriedFlavorIdx: make([]map[corev1.ResourceName]int, 0, len(requests)), CohortGeneration: 0, @@ -377,7 +377,7 @@ func (a *FlavorAssigner) findFlavorForPodSetResource( psID int, requests resources.Requests, resName corev1.ResourceName, - assignmentUsage resources.FlavorResourceQuantitiesFlat, + assignmentUsage resources.FlavorResourceQuantities, ) (ResourceAssignment, *Status) { resourceGroup := a.cq.RGByResource(resName) if resourceGroup == nil { @@ -429,7 +429,7 @@ func (a *FlavorAssigner) findFlavorForPodSetResource( resQuota := a.cq.QuotaFor(resources.FlavorResource{Flavor: fName, Resource: rName}) // Check considering the flavor usage by previous pod sets. fr := resources.FlavorResource{Flavor: fName, Resource: rName} - mode, borrow, s := a.fitsResourceQuota(fName, rName, val+assignmentUsage[fr], resQuota) + mode, borrow, s := a.fitsResourceQuota(fr, val+assignmentUsage[fr], resQuota) if s != nil { status.reasons = append(status.reasons, s.reasons...) } @@ -557,10 +557,10 @@ func flavorSelector(spec *corev1.PodSpec, allowedKeys sets.Set[string]) nodeaffi // if borrowing is required when preempting. // If the flavor doesn't satisfy limits immediately (when waiting or preemption // could help), it returns a Status with reasons. -func (a *FlavorAssigner) fitsResourceQuota(fName kueue.ResourceFlavorReference, rName corev1.ResourceName, val int64, rQuota *cache.ResourceQuota) (FlavorAssignmentMode, bool, *Status) { +func (a *FlavorAssigner) fitsResourceQuota(fr resources.FlavorResource, val int64, rQuota *cache.ResourceQuota) (FlavorAssignmentMode, bool, *Status) { var status Status var borrow bool - used := a.cq.Usage[fName][rName] + used := a.cq.Usage[fr] mode := NoFit if val <= rQuota.Nominal { // The request can be satisfied by the nominal quota, assuming quota is @@ -570,7 +570,7 @@ func (a *FlavorAssigner) fitsResourceQuota(fName kueue.ResourceFlavorReference, } cohortAvailable := rQuota.Nominal if a.cq.Cohort != nil { - cohortAvailable = a.cq.RequestableCohortQuota(fName, rName) + cohortAvailable = a.cq.RequestableCohortQuota(fr) } if a.canPreemptWhileBorrowing() { @@ -582,13 +582,13 @@ func (a *FlavorAssigner) fitsResourceQuota(fName kueue.ResourceFlavorReference, } } if rQuota.BorrowingLimit != nil && used+val > rQuota.Nominal+*rQuota.BorrowingLimit { - status.append(fmt.Sprintf("borrowing limit for %s in flavor %s exceeded", rName, fName)) + status.append(fmt.Sprintf("borrowing limit for %s in flavor %s exceeded", fr.Resource, fr.Flavor)) return mode, borrow, &status } cohortUsed := used if a.cq.Cohort != nil { - cohortUsed = a.cq.UsedCohortQuota(fName, rName) + cohortUsed = a.cq.UsedCohortQuota(fr) } lack := cohortUsed + val - cohortAvailable @@ -596,13 +596,13 @@ func (a *FlavorAssigner) fitsResourceQuota(fName kueue.ResourceFlavorReference, return Fit, used+val > rQuota.Nominal, nil } - lackQuantity := resources.ResourceQuantity(rName, lack) - msg := fmt.Sprintf("insufficient unused quota in cohort for %s in flavor %s, %s more needed", rName, fName, &lackQuantity) + lackQuantity := resources.ResourceQuantity(fr.Resource, lack) + msg := fmt.Sprintf("insufficient unused quota in cohort for %s in flavor %s, %s more needed", fr.Resource, fr.Flavor, &lackQuantity) if a.cq.Cohort == nil { if mode == NoFit { - msg = fmt.Sprintf("insufficient quota for %s in flavor %s in ClusterQueue", rName, fName) + msg = fmt.Sprintf("insufficient quota for %s in flavor %s in ClusterQueue", fr.Resource, fr.Flavor) } else { - msg = fmt.Sprintf("insufficient unused quota for %s in flavor %s, %s more needed", rName, fName, &lackQuantity) + msg = fmt.Sprintf("insufficient unused quota for %s in flavor %s, %s more needed", fr.Resource, fr.Flavor, &lackQuantity) } } status.append(msg) diff --git a/pkg/scheduler/flavorassigner/flavorassigner_test.go b/pkg/scheduler/flavorassigner/flavorassigner_test.go index 83516797fb0..105f364265b 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner_test.go +++ b/pkg/scheduler/flavorassigner/flavorassigner_test.go @@ -37,8 +37,8 @@ import ( ) type cohortResources struct { - requestableResources resources.FlavorResourceQuantitiesFlat - usage resources.FlavorResourceQuantitiesFlat + requestableResources resources.FlavorResourceQuantities + usage resources.FlavorResourceQuantities } func TestAssignFlavors(t *testing.T) { @@ -62,7 +62,7 @@ func TestAssignFlavors(t *testing.T) { wlPods []kueue.PodSet wlReclaimablePods []kueue.ReclaimablePod clusterQueue kueue.ClusterQueue - clusterQueueUsage resources.FlavorResourceQuantitiesFlat + clusterQueueUsage resources.FlavorResourceQuantities cohortResources *cohortResources wantRepMode FlavorAssignmentMode wantAssignment Assignment @@ -99,7 +99,7 @@ func TestAssignFlavors(t *testing.T) { Count: 1, }, }, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "default", Resource: corev1.ResourceMemory}: utiltesting.Mi, }, @@ -136,7 +136,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "tainted", Resource: corev1.ResourceCPU}: 1_000, }, }, @@ -153,7 +153,7 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "4"). FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, }, wantRepMode: Preempt, @@ -171,7 +171,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 2_000, }, }, @@ -215,7 +215,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "b_one", Resource: corev1.ResourceMemory}: 10 * utiltesting.Mi, }, @@ -239,7 +239,7 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 1_000, }, @@ -257,7 +257,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, "multiple resource groups with multiple resources, fits": { @@ -303,7 +303,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "two", Resource: corev1.ResourceMemory}: 10 * utiltesting.Mi, {Flavor: "b_one", Resource: "example.com/gpu"}: 3, @@ -334,18 +334,18 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceMemory}: 10 * utiltesting.Mi, }, cohortResources: &cohortResources{ - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, {Flavor: "one", Resource: corev1.ResourceMemory}: utiltesting.Gi, {Flavor: "two", Resource: corev1.ResourceCPU}: 4_000, {Flavor: "two", Resource: corev1.ResourceMemory}: 15 * utiltesting.Mi, {Flavor: "b_one", Resource: "example.com/gpu"}: 4, }, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceMemory}: 10 * utiltesting.Mi, {Flavor: "b_one", Resource: "example.com/gpu"}: 2, }, @@ -373,7 +373,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "two", Resource: corev1.ResourceMemory}: 10 * utiltesting.Mi, {Flavor: "b_one", Resource: "example.com/gpu"}: 3, @@ -413,7 +413,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, "multiple flavors, fits while skipping tainted flavor": { @@ -443,7 +443,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 3_000, }, }, @@ -501,7 +501,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 1_000, }, }, @@ -563,7 +563,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 1_000, {Flavor: "two", Resource: corev1.ResourceMemory}: utiltesting.Mi, }, @@ -633,7 +633,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 1_000, }, }, @@ -691,7 +691,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, "multiple specs, fit different flavors": { @@ -737,7 +737,7 @@ func TestAssignFlavors(t *testing.T) { Count: 1, }, }, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 5_000, }, @@ -764,7 +764,7 @@ func TestAssignFlavors(t *testing.T) { ClusterQueue, cohortResources: &cohortResources{ - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 200_000, {Flavor: "default", Resource: corev1.ResourceMemory}: 200 * utiltesting.Gi, }, @@ -798,7 +798,7 @@ func TestAssignFlavors(t *testing.T) { }, }, Borrowing: true, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "default", Resource: corev1.ResourceMemory}: 5 * utiltesting.Gi, }, @@ -818,10 +818,10 @@ func TestAssignFlavors(t *testing.T) { ).Cohort("test-cohort").ClusterQueue, cohortResources: &cohortResources{ - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 9_000, }, }, @@ -836,7 +836,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, "past max, but can preempt in ClusterQueue": { @@ -852,14 +852,14 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 9_000, }, cohortResources: &cohortResources{ - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 100_000, }, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 9_000, }, }, @@ -879,7 +879,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, }, @@ -896,7 +896,7 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "2"). FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 1_000, }, wantRepMode: Preempt, @@ -914,7 +914,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, }, @@ -931,14 +931,14 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "3"). FlavorQuotas, ).Cohort("test-cohort").ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, }, @@ -957,7 +957,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, }, @@ -986,7 +986,7 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "4"). FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 3_000, }, @@ -1008,7 +1008,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 2_000, }, }, @@ -1037,7 +1037,7 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "10"). FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 3_000, {Flavor: "tainted", Resource: corev1.ResourceCPU}: 3_000, }, @@ -1077,7 +1077,7 @@ func TestAssignFlavors(t *testing.T) { Count: 10, }, }, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, {Flavor: "tainted", Resource: corev1.ResourceCPU}: 10_000, }, @@ -1106,7 +1106,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, "num pods fit": { @@ -1137,7 +1137,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 3, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourcePods}: 3, {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, }, @@ -1170,7 +1170,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 3, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, "with reclaimable pods": { @@ -1206,7 +1206,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 3, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: corev1.ResourcePods}: 3, {Flavor: "default", Resource: corev1.ResourceCPU}: 3_000, }, @@ -1231,7 +1231,7 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "10"). FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, wantRepMode: Preempt, @@ -1251,7 +1251,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: "cpu"}: 9_000, {Flavor: "one", Resource: "pods"}: 1, }, @@ -1274,7 +1274,7 @@ func TestAssignFlavors(t *testing.T) { Resource(corev1.ResourceCPU, "10"). FlavorQuotas, ).ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, wantRepMode: Fit, @@ -1291,7 +1291,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: "cpu"}: 9_000, {Flavor: "two", Resource: "pods"}: 1, }, @@ -1316,14 +1316,14 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 11_000, {Flavor: "one", Resource: corev1.ResourcePods}: 10, {Flavor: "two", Resource: corev1.ResourceCPU}: 1_000, @@ -1345,7 +1345,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 9_000, {Flavor: "one", Resource: corev1.ResourcePods}: 1, }, @@ -1370,15 +1370,15 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 11_000, {Flavor: "one", Resource: corev1.ResourcePods}: 10, {Flavor: "two", Resource: corev1.ResourceCPU}: 10_000, @@ -1399,7 +1399,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 9_000, {Flavor: "two", Resource: corev1.ResourcePods}: 1, }, @@ -1423,15 +1423,15 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 11_000, {Flavor: "one", Resource: corev1.ResourcePods}: 10, {Flavor: "two", Resource: corev1.ResourceCPU}: 10_000, @@ -1453,7 +1453,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: "cpu"}: 9_000, {Flavor: "one", Resource: "pods"}: 1, }, @@ -1486,10 +1486,10 @@ func TestAssignFlavors(t *testing.T) { ).Cohort("test-cohort").ClusterQueue, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, @@ -1510,7 +1510,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, }, }, @@ -1541,10 +1541,10 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort").ClusterQueue, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, @@ -1565,7 +1565,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, }, }, @@ -1597,10 +1597,10 @@ func TestAssignFlavors(t *testing.T) { ).Cohort("test-cohort"). ClusterQueue, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, @@ -1617,7 +1617,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, }, @@ -1641,10 +1641,10 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort").ClusterQueue, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ { // below the borrowingLimit required to admit Flavor: "one", Resource: corev1.ResourceCPU}: 11_000, @@ -1652,7 +1652,7 @@ func TestAssignFlavors(t *testing.T) { }, wantRepMode: NoFit, wantAssignment: Assignment{ - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, PodSets: []PodSetAssignment{ { Name: "main", @@ -1686,14 +1686,14 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 11_000, {Flavor: "one", Resource: corev1.ResourcePods}: 10, {Flavor: "two", Resource: corev1.ResourceCPU}: 10_000, @@ -1714,7 +1714,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 9_000, {Flavor: "two", Resource: corev1.ResourcePods}: 1, }, @@ -1740,14 +1740,14 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort"). ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 11_000, {Flavor: "one", Resource: corev1.ResourcePods}: 10, {Flavor: "two", Resource: corev1.ResourceCPU}: 1_000, @@ -1769,7 +1769,7 @@ func TestAssignFlavors(t *testing.T) { Count: 1, }}, Borrowing: true, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 9_000, {Flavor: "one", Resource: corev1.ResourcePods}: 1, }, @@ -1789,14 +1789,14 @@ func TestAssignFlavors(t *testing.T) { ResourceQuotaWrapper(corev1.ResourceCPU).NominalQuota("10").LendingLimit("0").Append(). FlavorQuotas, ).Cohort("test-cohort").ClusterQueue, - clusterQueueUsage: resources.FlavorResourceQuantitiesFlat{ + clusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 2_000, }, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, {Flavor: "one", Resource: corev1.ResourcePods}: 10, }, @@ -1818,7 +1818,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 9_000, {Flavor: "one", Resource: corev1.ResourcePods}: 1, }, @@ -1844,10 +1844,10 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort").ClusterQueue, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, @@ -1868,7 +1868,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, }, }, @@ -1892,10 +1892,10 @@ func TestAssignFlavors(t *testing.T) { FlavorQuotas, ).Cohort("test-cohort").ClusterQueue, cohortResources: &cohortResources{ - usage: resources.FlavorResourceQuantitiesFlat{ + usage: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 10_000, }, - requestableResources: resources.FlavorResourceQuantitiesFlat{ + requestableResources: resources.FlavorResourceQuantities{ {Flavor: "one", Resource: corev1.ResourceCPU}: 12_000, {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, @@ -1912,7 +1912,7 @@ func TestAssignFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "two", Resource: corev1.ResourceCPU}: 12_000, }, }, @@ -1951,10 +1951,10 @@ func TestAssignFlavors(t *testing.T) { if clusterQueue.Cohort == nil { t.Fatalf("Test case has cohort resources, but cluster queue doesn't have cohort") } - clusterQueue.Cohort.Usage = tc.cohortResources.usage.Unflatten() - clusterQueue.Cohort.RequestableResources = tc.cohortResources.requestableResources.Unflatten() + clusterQueue.Cohort.Usage = tc.cohortResources.usage + clusterQueue.Cohort.RequestableResources = tc.cohortResources.requestableResources } - clusterQueue.Usage = tc.clusterQueueUsage.Unflatten() + clusterQueue.Usage = tc.clusterQueueUsage flvAssigner := New(wlInfo, clusterQueue, resourceFlavors, tc.enableFairSharing) assignment := flvAssigner.Assign(log, nil) @@ -2006,7 +2006,7 @@ func TestDeletedFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{ + Usage: resources.FlavorResourceQuantities{ {Flavor: "flavor", Resource: corev1.ResourceCPU}: 3_000, }, }, @@ -2034,7 +2034,7 @@ func TestDeletedFlavors(t *testing.T) { }, Count: 1, }}, - Usage: resources.FlavorResourceQuantitiesFlat{}, + Usage: resources.FlavorResourceQuantities{}, }, }, } diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go index cd0b3ae30b7..219c706ce2a 100644 --- a/pkg/scheduler/preemption/preemption.go +++ b/pkg/scheduler/preemption/preemption.go @@ -231,7 +231,7 @@ func (p *Preemptor) applyPreemptionWithSSA(ctx context.Context, w *kueue.Workloa // Once the Workload fits, the heuristic tries to add Workloads back, in the // reverse order in which they were removed, while the incoming Workload still // fits. -func minimalPreemptions(log logr.Logger, wlReq resources.FlavorResourceQuantitiesFlat, cq *cache.ClusterQueueSnapshot, snapshot *cache.Snapshot, resPerFlv resourcesPerFlavor, candidates []*workload.Info, allowBorrowing bool, allowBorrowingBelowPriority *int32) []*Target { +func minimalPreemptions(log logr.Logger, wlReq resources.FlavorResourceQuantities, cq *cache.ClusterQueueSnapshot, snapshot *cache.Snapshot, resPerFlv resourcesPerFlavor, candidates []*workload.Info, allowBorrowing bool, allowBorrowingBelowPriority *int32) []*Target { if logV := log.V(5); logV.Enabled() { logV.Info("Simulating preemption", "candidates", workload.References(candidates), "resourcesRequiringPreemption", resPerFlv, "allowBorrowing", allowBorrowing, "allowBorrowingBelowPriority", allowBorrowingBelowPriority) } @@ -286,7 +286,7 @@ func minimalPreemptions(log logr.Logger, wlReq resources.FlavorResourceQuantitie return targets } -func fillBackWorkloads(targets []*Target, wlReq resources.FlavorResourceQuantitiesFlat, cq *cache.ClusterQueueSnapshot, snapshot *cache.Snapshot, allowBorrowing bool) []*Target { +func fillBackWorkloads(targets []*Target, wlReq resources.FlavorResourceQuantities, cq *cache.ClusterQueueSnapshot, snapshot *cache.Snapshot, allowBorrowing bool) []*Target { // In the reverse order, check if any of the workloads can be added back. for i := len(targets) - 2; i >= 0; i-- { snapshot.AddWorkload(targets[i].WorkloadInfo) @@ -563,7 +563,7 @@ func workloadUsesResources(wl *workload.Info, resPerFlv resourcesPerFlavor) bool // workloadFits determines if the workload requests would fit given the // requestable resources and simulated usage of the ClusterQueue and its cohort, // if it belongs to one. -func workloadFits(wlReq resources.FlavorResourceQuantitiesFlat, cq *cache.ClusterQueueSnapshot, allowBorrowing bool) bool { +func workloadFits(wlReq resources.FlavorResourceQuantities, cq *cache.ClusterQueueSnapshot, allowBorrowing bool) bool { for fr, v := range wlReq { if !allowBorrowing && cq.BorrowingWith(fr, v) { return false @@ -577,7 +577,7 @@ func workloadFits(wlReq resources.FlavorResourceQuantitiesFlat, cq *cache.Cluste func queueUnderNominalInResourcesNeedingPreemption(resPerFlv resourcesPerFlavor, cq *cache.ClusterQueueSnapshot) bool { for fr := range resPerFlv { - if cq.Usage.For(fr) >= cq.QuotaFor(fr).Nominal { + if cq.Usage[fr] >= cq.QuotaFor(fr).Nominal { return false } } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 049238a7063..558e8456f1c 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -144,11 +144,11 @@ func (s *Scheduler) setAdmissionRoutineWrapper(wrapper routine.Wrapper) { s.admissionRoutineWrapper = wrapper } -type cohortsUsage map[string]resources.FlavorResourceQuantitiesFlat +type cohortsUsage map[string]resources.FlavorResourceQuantities -func (cu cohortsUsage) add(cohort string, assignment resources.FlavorResourceQuantitiesFlat) { +func (cu cohortsUsage) add(cohort string, assignment resources.FlavorResourceQuantities) { if cu[cohort] == nil { - cu[cohort] = make(resources.FlavorResourceQuantitiesFlat, len(assignment)) + cu[cohort] = make(resources.FlavorResourceQuantities, len(assignment)) } for fr, v := range assignment { @@ -156,11 +156,11 @@ func (cu cohortsUsage) add(cohort string, assignment resources.FlavorResourceQua } } -func (cu cohortsUsage) totalUsageForCommonFlavorResources(cohort string, assignment resources.FlavorResourceQuantitiesFlat) resources.FlavorResourceQuantitiesFlat { +func (cu cohortsUsage) totalUsageForCommonFlavorResources(cohort string, assignment resources.FlavorResourceQuantities) resources.FlavorResourceQuantities { return utilmaps.Intersect(cu[cohort], assignment, func(a, b int64) int64 { return a + b }) } -func (cu cohortsUsage) hasCommonFlavorResources(cohort string, assignment resources.FlavorResourceQuantitiesFlat) bool { +func (cu cohortsUsage) hasCommonFlavorResources(cohort string, assignment resources.FlavorResourceQuantities) bool { cohortUsage, cohortFound := cu[cohort] if !cohortFound { return false @@ -364,7 +364,7 @@ type entry struct { // netUsage returns how much capacity this entry will require from the ClusterQueue/Cohort. // When a workload is preempting, it subtracts the preempted resources from the resources // required, as the remaining quota is all we need from the CQ/Cohort. -func (e *entry) netUsage() resources.FlavorResourceQuantitiesFlat { +func (e *entry) netUsage() resources.FlavorResourceQuantities { if e.assignment.RepresentativeMode() == flavorassigner.Fit { return e.assignment.Usage } @@ -423,21 +423,21 @@ func (s *Scheduler) nominate(ctx context.Context, workloads []workload.Info, sna } // resourcesToReserve calculates how much of the available resources in cq/cohort assignment should be reserved. -func resourcesToReserve(e *entry, cq *cache.ClusterQueueSnapshot) resources.FlavorResourceQuantitiesFlat { +func resourcesToReserve(e *entry, cq *cache.ClusterQueueSnapshot) resources.FlavorResourceQuantities { if e.assignment.RepresentativeMode() != flavorassigner.Preempt { return e.assignment.Usage } - reservedUsage := make(resources.FlavorResourceQuantitiesFlat) + reservedUsage := make(resources.FlavorResourceQuantities) for fr, usage := range e.assignment.Usage { cqQuota := cq.QuotaFor(fr) if e.assignment.Borrowing { if cqQuota.BorrowingLimit == nil { reservedUsage[fr] = usage } else { - reservedUsage[fr] = min(usage, cqQuota.Nominal+*cqQuota.BorrowingLimit-cq.Usage.For(fr)) + reservedUsage[fr] = min(usage, cqQuota.Nominal+*cqQuota.BorrowingLimit-cq.Usage[fr]) } } else { - reservedUsage[fr] = max(0, min(usage, cqQuota.Nominal-cq.Usage.For(fr))) + reservedUsage[fr] = max(0, min(usage, cqQuota.Nominal-cq.Usage[fr])) } } return reservedUsage diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index cd6180ce506..fccedddabc2 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -3281,24 +3281,24 @@ func TestResourcesToReserve(t *testing.T) { name string assignmentMode flavorassigner.FlavorAssignmentMode borrowing bool - assignmentUsage resources.FlavorResourceQuantitiesFlat - cqUsage resources.FlavorResourceQuantitiesFlat - wantReserved resources.FlavorResourceQuantitiesFlat + assignmentUsage resources.FlavorResourceQuantities + cqUsage resources.FlavorResourceQuantities + wantReserved resources.FlavorResourceQuantities }{ { name: "Reserved memory and gpu less than assignment usage, assignment preempts", assignmentMode: flavorassigner.Preempt, - assignmentUsage: resources.FlavorResourceQuantitiesFlat{ + assignmentUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 6, }, - cqUsage: resources.FlavorResourceQuantitiesFlat{ + cqUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 6, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, - wantReserved: resources.FlavorResourceQuantitiesFlat{ + wantReserved: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 40, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 4, }, @@ -3306,17 +3306,17 @@ func TestResourcesToReserve(t *testing.T) { { name: "Reserved memory equal assignment usage, assignment preempts", assignmentMode: flavorassigner.Preempt, - assignmentUsage: resources.FlavorResourceQuantitiesFlat{ + assignmentUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 30, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, }, - cqUsage: resources.FlavorResourceQuantitiesFlat{ + cqUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, - wantReserved: resources.FlavorResourceQuantitiesFlat{ + wantReserved: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 30, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, }, @@ -3324,17 +3324,17 @@ func TestResourcesToReserve(t *testing.T) { { name: "Reserved memory equal assignment usage, assignment fits", assignmentMode: flavorassigner.Fit, - assignmentUsage: resources.FlavorResourceQuantitiesFlat{ + assignmentUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, }, - cqUsage: resources.FlavorResourceQuantitiesFlat{ + cqUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, - wantReserved: resources.FlavorResourceQuantitiesFlat{ + wantReserved: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, }, @@ -3342,17 +3342,17 @@ func TestResourcesToReserve(t *testing.T) { { name: "Reserved memory is 0, CQ is borrowing, assignment preempts without borrowing", assignmentMode: flavorassigner.Preempt, - assignmentUsage: resources.FlavorResourceQuantitiesFlat{ + assignmentUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, - cqUsage: resources.FlavorResourceQuantitiesFlat{ + cqUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 10, }, - wantReserved: resources.FlavorResourceQuantitiesFlat{ + wantReserved: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 0, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 0, }, @@ -3361,17 +3361,17 @@ func TestResourcesToReserve(t *testing.T) { name: "Reserved memory cut by nominal+borrowing quota, assignment preempts and borrows", assignmentMode: flavorassigner.Preempt, borrowing: true, - assignmentUsage: resources.FlavorResourceQuantitiesFlat{ + assignmentUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, - cqUsage: resources.FlavorResourceQuantitiesFlat{ + cqUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 10, }, - wantReserved: resources.FlavorResourceQuantitiesFlat{ + wantReserved: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 40, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, @@ -3380,17 +3380,17 @@ func TestResourcesToReserve(t *testing.T) { name: "Reserved memory equal assignment usage, CQ borrowing limit is nil", assignmentMode: flavorassigner.Preempt, borrowing: true, - assignmentUsage: resources.FlavorResourceQuantitiesFlat{ + assignmentUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, - cqUsage: resources.FlavorResourceQuantitiesFlat{ + cqUsage: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("spot"), Resource: corev1.ResourceMemory}: 60, {Flavor: kueue.ResourceFlavorReference("model-a"), Resource: "gpu"}: 2, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 10, }, - wantReserved: resources.FlavorResourceQuantitiesFlat{ + wantReserved: resources.FlavorResourceQuantities{ {Flavor: kueue.ResourceFlavorReference("on-demand"), Resource: corev1.ResourceMemory}: 50, {Flavor: kueue.ResourceFlavorReference("model-b"), Resource: "gpu"}: 2, }, diff --git a/pkg/workload/workload.go b/pkg/workload/workload.go index 9dd9da29d99..5e26391b91a 100644 --- a/pkg/workload/workload.go +++ b/pkg/workload/workload.go @@ -206,8 +206,8 @@ func (i *Info) CanBePartiallyAdmitted() bool { // FlavorResourceUsage returns the total resource usage for the workload, // per flavor (if assigned, otherwise flavor shows as empty string), per resource. -func (i *Info) FlavorResourceUsage() resources.FlavorResourceQuantitiesFlat { - total := make(resources.FlavorResourceQuantitiesFlat) +func (i *Info) FlavorResourceUsage() resources.FlavorResourceQuantities { + total := make(resources.FlavorResourceQuantities) if i == nil { return total } diff --git a/pkg/workload/workload_test.go b/pkg/workload/workload_test.go index ea5574e6c18..0a9de5da282 100644 --- a/pkg/workload/workload_test.go +++ b/pkg/workload/workload_test.go @@ -593,10 +593,10 @@ func TestIsEvictedByPodsReadyTimeout(t *testing.T) { func TestFlavorResourceUsage(t *testing.T) { cases := map[string]struct { info *Info - want resources.FlavorResourceQuantitiesFlat + want resources.FlavorResourceQuantities }{ "nil": { - want: resources.FlavorResourceQuantitiesFlat{}, + want: resources.FlavorResourceQuantities{}, }, "one podset, no flavors": { info: &Info{ @@ -607,7 +607,7 @@ func TestFlavorResourceUsage(t *testing.T) { }, }}, }, - want: resources.FlavorResourceQuantitiesFlat{ + want: resources.FlavorResourceQuantities{ {Flavor: "", Resource: "cpu"}: 1_000, {Flavor: "", Resource: "example.com/gpu"}: 3, }, @@ -625,7 +625,7 @@ func TestFlavorResourceUsage(t *testing.T) { }, }}, }, - want: resources.FlavorResourceQuantitiesFlat{ + want: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: "cpu"}: 1_000, {Flavor: "gpu", Resource: "example.com/gpu"}: 3, }, @@ -663,7 +663,7 @@ func TestFlavorResourceUsage(t *testing.T) { }, }, }, - want: resources.FlavorResourceQuantitiesFlat{ + want: resources.FlavorResourceQuantities{ {Flavor: "default", Resource: "cpu"}: 3_000, {Flavor: "default", Resource: "memory"}: 2 * utiltesting.Gi, {Flavor: "model_a", Resource: "example.com/gpu"}: 3,